{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "d31d8850", "metadata": { "ExecuteTime": { "end_time": "2023-11-13T17:04:37.529008Z", "start_time": "2023-11-13T17:04:37.527215Z" } }, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "7cd4c0dd", "metadata": { "ExecuteTime": { "end_time": "2023-11-13T17:04:48.887287Z", "start_time": "2023-11-13T17:04:39.856588Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n" ] } ], "source": [ "import tqdm\n", "import math\n", "import torch\n", "import random\n", "import funcy\n", "import copy\n", "import gc\n", "import re\n", "import json\n", "import tempfile\n", "import collections\n", "import pandas as pd\n", "import numpy as np\n", "import fasttext\n", "from joblib import Parallel, delayed\n", "\n", "from suno_utils.audio import Audio\n", "from suno_utils.utils.text import (\n", " write_jsonl,\n", " read_jsonl,\n", " write_json,\n", " read_json,\n", " normalize_whitespace,\n", ")\n", "from suno_utils.utils.s3 import read_from_s3, check_s3_file_exists, open_from_s3\n", "from suno_utils.utils.tokenizers import tokenize\n", "from suno_utils.harvest.youtube.constants.text_lang import BASE_TO_FASTTEXT_REMAP\n", "\n", "LANG_ID_MODEL_FP = \"s3://suno-data/georg/trained_models/chirp_v1/lid.176.bin\"\n", "text_lang_model = read_from_s3(LANG_ID_MODEL_FP, read_f=fasttext.load_model)\n", "\n", "def _get_text_lang(text):\n", " \"\"\"get probability of input language for text\"\"\"\n", " text = text.replace(\"’\", \"'\").lower()\n", " text = re.sub(r\"\\[.+?\\]\", \" \", text)\n", " text = normalize_whitespace(text)\n", " out = text_lang_model.predict(text, k=1)\n", " lang_str = out[0][0]\n", " p_lang = out[1][0]\n", " lang = lang_str.split(\"__\")[-1]\n", " lang = BASE_TO_FASTTEXT_REMAP.get(lang, lang)\n", " # if p_lang >= 0.8:\n", " # return lang\n", " return p_lang, lang\n", "\n", "TMP_DIR = os.path.join(os.getcwd(), \"hoot\")\n", "os.makedirs(TMP_DIR, exist_ok=True)" ] }, { "cell_type": "markdown", "id": "d9d50793", "metadata": {}, "source": [ "## genius_hq" ] }, { "cell_type": "code", "execution_count": 3, "id": "8f428b8f", "metadata": { "ExecuteTime": { "end_time": "2023-11-13T17:04:49.099114Z", "start_time": "2023-11-13T17:04:48.888764Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/tony/anaconda3/envs/suno_env/lib/python3.10/site-packages/Bio/pairwise2.py:278: BiopythonDeprecationWarning: Bio.pairwise2 has been deprecated, and we intend to remove it in a future release of Biopython. As an alternative, please consider using Bio.Align.PairwiseAligner as a replacement, and contact the Biopython developers if you still need the Bio.pairwise2 module.\n", " warnings.warn(\n" ] } ], "source": [ "from Bio import pairwise2\n", "\n", "from suno_utils.utils.metrics import get_cer\n", "from suno_utils.tasks.hoot import (\n", " parse_lyrics,\n", " legacy_parse_lyrics,\n", " EMBEDDING_RATE as HOOT_EMBEDDING_RATE,\n", ")\n", "from suno_utils.utils.lyrics import remove_speakers" ] }, { "cell_type": "code", "execution_count": 4, "id": "2cd505d5", "metadata": { "ExecuteTime": { "end_time": "2023-11-13T17:08:43.196809Z", "start_time": "2023-11-13T17:07:05.388877Z" } }, "outputs": [], "source": [ "# takes ~ 2 mins\n", "base_metas = read_from_s3(\n", " \"s3://suno-data/datasets/bundles/v1/genius_hq/metas_plus.jsonl\", read_f=read_jsonl\n", ")\n", "# TODO: original IDs are already unique here, but did we resolve correctly by highest genius views?\n", "# could be a way to filter translations that don't contain the word 'translation'" ] }, { "cell_type": "code", "execution_count": 5, "id": "6901f1b9", "metadata": { "ExecuteTime": { "end_time": "2023-11-13T17:08:47.238252Z", "start_time": "2023-11-13T17:08:43.198714Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total, 2090009\n", "lang cut, 23432\n", "genius view cut, 86048\n", "youtube view cut, 96343\n", "duration cut, 53169\n", "lyrics cut, 34057\n" ] } ], "source": [ "total_base_metas = len(base_metas)\n", "n_genius_view_cut = 5\n", "n_youtube_view_cut = 50\n", "max_duration = 8 * 60\n", "min_duration = 1 * 60\n", "max_len_lyrics = 9128\n", "min_len_lyrics = 64\n", "print(f\"total, {len(base_metas)}\")\n", "print(f\"lang cut, {sum(1 if m['lang'] is None else 0 for m in base_metas)}\")\n", "print(\n", " f'genius view cut, {sum(1 if m[\"genius_views\"] < n_genius_view_cut else 0 for m in base_metas)}'\n", ")\n", "print(\n", " f'youtube view cut, {sum(1 if m[\"youtube_views\"] < n_youtube_view_cut else 0 for m in base_metas)}'\n", ")\n", "print(\n", " f'duration cut, {sum(1 if (m[\"duration_s\"] > max_duration or m[\"duration_s\"] < min_duration) else 0 for m in base_metas)}'\n", ")\n", "print(\n", " f'lyrics cut, {sum(1 if (len(m[\"lyrics\"]) > max_len_lyrics or len(m[\"lyrics\"]) < min_len_lyrics) else 0 for m in base_metas)}'\n", ")" ] }, { "cell_type": "code", "execution_count": 6, "id": "6ac295a0", "metadata": { "ExecuteTime": { "end_time": "2023-11-13T17:08:53.109856Z", "start_time": "2023-11-13T17:08:47.239491Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2090009 clips\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████| 2090009/2090009 [00:05<00:00, 390084.30it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "1836168 filtered clips\n", "107025 hours\n" ] } ], "source": [ "print(len(base_metas), \"clips\")\n", "metas = []\n", "seen_id = set()\n", "seen_path = set()\n", "for m in tqdm.tqdm(base_metas):\n", " # guessed_lang = _get_text_lang(m[\"lyrics\"])[1]\n", " if (\n", " m[\"genius_views\"] < n_genius_view_cut\n", " or m[\"youtube_views\"] < n_youtube_view_cut\n", " or m[\"lang\"] is None\n", " or \"translation\" in m[\"genius_slug\"].lower()\n", " # or m[\"lang\"].lower()[:2] != \"en\"\n", " or m[\"duration_s\"] < min_duration\n", " or m[\"duration_s\"] > max_duration\n", " or len(m[\"lyrics\"]) < min_len_lyrics\n", " or len(m[\"lyrics\"]) > max_len_lyrics\n", " or m[\"id\"] in seen_id\n", " or m[\"audio_filepath\"] in seen_path\n", " # or _get_text_lang(m[\"lyrics\"])[1] != \"en\"\n", " ):\n", " continue\n", " seen_id.add(m[\"id\"])\n", " seen_path.add(m[\"audio_filepath\"])\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"lyrics\": m[\"lyrics\"].strip(),\n", " \"duration_s\": m[\"duration_s\"],\n", " \"lang\": m[\"lang\"],\n", " # \"guess_lang\": guessed_lang,\n", " \"audio_filepath\": m[\"audio_filepath\"],\n", " \"original_id\": m[\"original_id\"],\n", " \"genius_slug\": m[\"genius_slug\"],\n", " \"genius_views\": m[\"genius_views\"],\n", " }\n", " metas.append(new_m)\n", "print(len(metas), \"filtered clips\")\n", "print(round(sum(m[\"duration_s\"] for m in metas) / 60 / 60), \"hours\")\n", "metas_map = {m[\"id\"]: m for m in metas}\n", "# 2090009 clips\n", "# 1836168 filtered clips\n", "# 107025 hours" ] }, { "cell_type": "code", "execution_count": 8, "id": "2baf4347", "metadata": { "ExecuteTime": { "end_time": "2023-11-13T18:51:46.421528Z", "start_time": "2023-11-13T18:51:46.417615Z" } }, "outputs": [ { "data": { "text/plain": [ "{'id': 'a93101d1-2e60-4c8b-906e-64e21f7f8d08',\n", " 'lyrics': '[Letra de \"Calor\" ft. Lara91k]\\n\\n[Intro: Lara91k]\\nYeah, uh\\nEy, yeah, yeah\\nUh, yeah, yeah, ey\\nEy, ey, ey\\nPréndelo, yeah\\n\\n[Estribillo: Lara91k]\\nPréndelo, se ponen en fila por un blunt (Agítalo)\\nTodo\\' venden su culo, mi amor (Acércalo)\\nEllos se prenden fuego en la Calor (Y así es como)\\nSe dan vuelta como Courtney Love (Préndelo)\\nSe ponen en fila por un blunt (Agítalo)\\nTodo\\' venden su culo, mi amor (Acércalo)\\nEllos se prenden fuego en la Calor (Y así es como)\\nVan dando vuelta\\' como Courtney Love\\nSe ponen en fila por un blunt\\nTodo\\' venden su culo, mi amor\\nEllos se prenden fuego en la Calor\\nSe dan vuelta como Courtney Love\\n\\n[Verso 1: Neo Pistéa]\\nPréndelo, mientras movémelo\\nEn lo oscuro, pegado al muro, ey\\nBeba, vos prendémelo, que me agarra la calor\\nTu culo es ciencia ficción, quiero ver tu peliculón\\nSi me acerco, prendo un blunt, no vas a decir que no\\nSi te agarra la calor, causa una buena situación\\nQué rica satisfacción\\nPrenderte aunque me meta en un lío más\\nSi eso llama mi atención\\nLa plata y tu toto, esto es amor de verdad\\nEy, sí, que\\nDame, quiero chispa, girl\\nLo voy a prender y te voy a prender también\\n¿Por qué no te prendés, bebé, y lo prendés?\\nDespués te pienso comer, te voy a comer\\nBebé, voy a comer pero-pero primero prendé\\n¿Por qué no te prendés, bebé, y lo prendés?\\nDesde que te vi bailando, la calor me está pegando\\nSi veo que nadie nos mira, bebé, meto mano\\nSi veo que nadie nos mira, traémelo\\nAcércalo, agítalo, rolámelo y\\n\\n\\n[Estribillo: Lara91k, Neo Pistéa & Ambos]\\nPréndelo, se ponen en fila por un blunt (Agítalo)\\nTodo\\' venden su culo, mi amor (Acércalo)\\nEllos se prenden fuego en la Calor (Y así es como)\\nSe dan vuelta como Courtney Love (Préndelo)\\nSe ponen en fila por un blunt (Agítalo, agítalo)\\nTodo\\' venden su culo, mi amor (Acércalo, ey)\\nEllos se prenden fuego en la Calor (Y así es como, hace calor, babe)\\nVan dando vuelta\\' como Courtney Love (Préndelo)\\nSe ponen en fila por un blunt (Agítalo, babe)\\nTodo\\' venden su culo, mi amor\\nEllos se prenden fuego en la Calor (Hace calor, babe)\\nSe dan vuelta como Courtney Love\\n\\n[Verso 2: Lara91k]\\nHace calor, subió la tensión\\nTomamo\\' vitamina\\' para sentirno\\' mejor\\nHace calor, decís que no\\nSabe lo que quiere, pero llama mi atención\\nCaigo con las pibas y siempre me voy con vos\\nYa sé que es temprano pero quiero irme con vos\\nY te lo juro, en lo oscuro, lo juro, no puedo\\nDecirte que no, aunque siempre lo dude primero\\nBaby, ya sabemo\\', esto fue siempre así\\nVos me enamoraste, por eso mentí\\nPero cuando juego pongo el on a mil\\nAnsiando con tenerte y préndelo\\n\\n\\n[Estribillo: Lara91k]\\nSe ponen en fila por un blunt\\nTodo\\' venden su culo, mi amor\\nEllos se prenden fuego en la Calor\\nSe dan vuelta como Courtney Love (Préndelo)\\nSe ponen en fila por un blunt (Agítalo)\\nTodo\\' venden su culo, mi amor (Acércalo)\\nEllos se prenden fuego en la Calor (Y así es como)\\nDando una vuelta como Courtney Love',\n", " 'duration_s': 274,\n", " 'lang': 'en',\n", " 'audio_filepath': 's3://suno-data/datasets/harvest/genius_hq/audio/OzrkFekIbxE.webm',\n", " 'original_id': 'OzrkFekIbxE',\n", " 'genius_slug': '0-600-and-neo-pistea-calor-lyrics',\n", " 'genius_views': 1802}" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "metas[0]" ] }, { "cell_type": "code", "execution_count": null, "id": "532d2ad2", "metadata": {}, "outputs": [], "source": [ "# or a shortcut\n", "# with open(\"/home/tony/Data/Hoot/metas.json\", \"r\") as fp:\n", "# metas = json.load()fp" ] }, { "cell_type": "markdown", "id": "a1cc806d", "metadata": {}, "source": [ "#### do alignment (will be cache in TMP_DIR)" ] }, { "cell_type": "code", "execution_count": 7, "id": "4d414eb9", "metadata": { "ExecuteTime": { "end_time": "2023-11-13T17:10:36.420695Z", "start_time": "2023-11-13T17:10:36.418725Z" } }, "outputs": [], "source": [ "MIN_SEGMENT_DURATION_S = 5 # 5 # TODO: should this be larger??\n", "MAX_SEGMENT_DURATION_S = 2 * 60\n", "MAX_PAD_BOUNDARY_S = 8\n", "MAX_PAD_S = 2\n", "EARLY_VERSE_TERMINATE_CHANCE = 0.9" ] }, { "cell_type": "code", "execution_count": 9, "id": "d350830c", "metadata": { "ExecuteTime": { "end_time": "2023-10-16T20:09:44.416658Z", "start_time": "2023-10-16T20:09:44.169747Z" } }, "outputs": [], "source": [ "# we need tokenzier to decode greedy hoot preds\n", "# (use a dict instead of model to allow multiprocessing)\n", "with open(\"/home/tony/Work/hoot/scripts/hoot_vocab.json\") as f:\n", " vocab = json.load(f)\n", "inv_vocab = {v: k for k, v in vocab.items()}\n", "\n", "\n", "def decode_token_ids(token_ids):\n", " prev_idx = None\n", " tokens = []\n", " for idx in token_ids:\n", " idx = int(idx)\n", " if idx == prev_idx:\n", " continue\n", " if idx != len(vocab):\n", " tokens.append(inv_vocab[idx])\n", " prev_idx = idx\n", " return \"\".join(tokens).replace(\"▁\", \" \").strip()" ] }, { "cell_type": "code", "execution_count": 10, "id": "a273ecbd", "metadata": { "ExecuteTime": { "end_time": "2023-10-16T20:11:11.701805Z", "start_time": "2023-10-16T20:11:11.678702Z" } }, "outputs": [], "source": [ "def _norm_lyrics_to_plain(text):\n", " text = text.lower()\n", " text = re.sub(r\"\\[.*?\\]\", \" \", text)\n", " text = re.sub(r\"[^a-z\\'\\s]\", \" \", text)\n", " return normalize_whitespace(text)\n", "\n", "\n", "def _get_alignable_tokens(text):\n", " # replace all metatags so we don't tokenize within\n", " tag_repl_map = {}\n", " safe_text = \"\"\n", " offs = 0\n", " for m in re.finditer(r\"\\[.*?\\]\", text):\n", " if m.start() - offs > 0:\n", " safe_text += text[offs : m.start()]\n", " uid = str(random.randint(9999999, 99999999))\n", " safe_text += uid\n", " tag_repl_map[uid] = m.group()\n", " offs = m.end()\n", " if len(text) - offs > 0:\n", " safe_text += text[offs:]\n", " # get tokens that we want to align\n", " safe_words = [s for s in re.split(\"([A-Za-z']+)\", safe_text) if len(s) > 0]\n", " words_to_align = [\n", " (n, word.lower())\n", " for n, word in enumerate(safe_words)\n", " if re.match(r\"^[a-z\\']+$\", word.lower())\n", " ]\n", " # add back in metatags\n", " words = []\n", " for safe_word in safe_words:\n", " word = safe_word\n", " for k, v in tag_repl_map.items():\n", " if k in safe_word:\n", " word = word.replace(k, v)\n", " words.append(word)\n", " return words, words_to_align\n", "\n", "\n", "def _assign_word_timings(words_to_align, aligned_words):\n", " flat_words_a = [w for _, w in words_to_align]\n", " flat_words_b = [m[\"text\"] for m in aligned_words]\n", " assert len(set(flat_words_a) & set(flat_words_b)) >= 0.75 * len(\n", " set(flat_words_a) | set(flat_words_b)\n", " )\n", " alignment = pairwise2.align.globalxx(flat_words_a, flat_words_b, gap_char=[\"*\"])[0]\n", " assert np.mean([int(e != \"*\") for e in alignment.seqA]) >= 0.75\n", " assert np.mean([int(e != \"*\") for e in alignment.seqB]) >= 0.75\n", " # assign align info to tokens in ground truth\n", " flat_words_a_timed = []\n", " idx_a = 0\n", " idx_b = 0\n", " for a, b in zip(alignment.seqA, alignment.seqB):\n", " if a != \"*\":\n", " start_s = None\n", " end_s = None\n", " p_align = None\n", " assert a == words_to_align[idx_a][1]\n", " idx = words_to_align[idx_a][0]\n", " if b != \"*\":\n", " start_s = aligned_words[idx_b][\"start_s\"]\n", " end_s = aligned_words[idx_b][\"end_s\"]\n", " p_align = aligned_words[idx_b][\"p_align\"]\n", " flat_words_a_timed.append(\n", " {\n", " \"word\": a,\n", " \"start_s\": round(start_s, 2),\n", " \"end_s\": round(end_s, 2),\n", " \"p_align\": round(p_align, 3),\n", " \"orig_idx\": idx,\n", " }\n", " )\n", " idx_a += 1\n", " if b != \"*\":\n", " idx_b += 1\n", " return flat_words_a_timed\n", "\n", "\n", "def _merge_metatags_whitespace(all_words_timed):\n", " # generally forward merge cause of \"[Chorus]\\nFirst line.\"\n", " # but also maybe backward with first part \")\\n\\n[Verse]\\n\"\n", " merged_words_timed = []\n", " n = 0\n", " while True:\n", " if n == len(all_words_timed):\n", " break\n", " if \"success\" not in all_words_timed[n]:\n", " # we have whitespace so lets see if we can merge a prefix to previous\n", " word = all_words_timed[n][\"word\"]\n", " if (\n", " \"\\n\" in word\n", " and word.index(\"\\n\") > 0\n", " and n > 0\n", " and merged_words_timed[-1].get(\"success\", False)\n", " ):\n", " merged_words_timed[-1][\"word\"] += word[: word.index(\"\\n\")]\n", " word = word[word.index(\"\\n\") :]\n", " elif n == len(all_words_timed) - 1:\n", " merged_words_timed[-1][\"word\"] += word\n", " n += 1\n", " continue\n", " # lets see if we can merge forward\n", " if len(all_words_timed) - 1 > n and all_words_timed[n + 1].get(\n", " \"success\", False\n", " ):\n", " # merge forward\n", " new_m = {\n", " \"word\": word + all_words_timed[n + 1][\"word\"],\n", " \"success\": all_words_timed[n + 1][\"success\"],\n", " \"start_s\": all_words_timed[n + 1][\"start_s\"],\n", " \"end_s\": all_words_timed[n + 1][\"end_s\"],\n", " \"p_align\": all_words_timed[n + 1][\"p_align\"],\n", " }\n", " merged_words_timed.append(new_m)\n", " n += 2\n", " continue\n", " else:\n", " merged_words_timed.append({\"word\": word})\n", " n += 1\n", " continue\n", " else:\n", " merged_words_timed.append(copy.deepcopy(all_words_timed[n]))\n", " n += 1\n", " continue\n", " return merged_words_timed\n", "\n", "\n", "# pad words\n", "def _pad_words(merged_words_timed, duration_s):\n", " # calc max duration\n", " duration_s = max(\n", " duration_s,\n", " max([m[\"end_s\"] for m in merged_words_timed if m.get(\"success\", False)]),\n", " )\n", " # make copy so we can modify in place\n", " padded_words_timed = copy.deepcopy(merged_words_timed)\n", " for n, m in enumerate(padded_words_timed):\n", " if not m.get(\"success\", False):\n", " continue\n", " # do left padding\n", " if n == 0:\n", " if m[\"start_s\"] <= MAX_PAD_BOUNDARY_S:\n", " m[\"start_s\"] = 0\n", " else:\n", " m[\"start_s\"] = m[\"start_s\"] - MAX_PAD_S\n", " else:\n", " mm = merged_words_timed[n - 1]\n", " if mm.get(\"success\", False):\n", " if m[\"start_s\"] - mm[\"end_s\"] > 0.01:\n", " m[\"start_s\"] -= min(MAX_PAD_S, (m[\"start_s\"] - mm[\"end_s\"]) / 2)\n", " m[\"start_s\"] = max(0, m[\"start_s\"])\n", " m[\"start_s\"] = min(duration_s, m[\"start_s\"])\n", " # do right padding\n", " if n == len(merged_words_timed) - 1:\n", " if duration_s - m[\"end_s\"] <= MAX_PAD_BOUNDARY_S:\n", " m[\"end_s\"] = duration_s\n", " else:\n", " m[\"end_s\"] = m[\"end_s\"] + MAX_PAD_S\n", " else:\n", " mm = merged_words_timed[n + 1]\n", " if mm.get(\"success\", False):\n", " if mm[\"start_s\"] - m[\"end_s\"] > 0.01:\n", " m[\"end_s\"] += min(MAX_PAD_S, (mm[\"start_s\"] - m[\"end_s\"]) / 2)\n", " m[\"end_s\"] = max(0, m[\"end_s\"])\n", " m[\"end_s\"] = min(duration_s, m[\"end_s\"])\n", " m[\"start_s\"] = round(m[\"start_s\"], 2)\n", " m[\"end_s\"] = round(m[\"end_s\"], 2)\n", " return padded_words_timed\n", "\n", "\n", "def _merge_into_lines(padded_words_timed):\n", " # first reconsitute lines so we don't break mid-line\n", " lines = []\n", " tmp_line = []\n", " for m in padded_words_timed:\n", " if \"\\n\" in m[\"word\"] and len(tmp_line) > 0:\n", " lines.append(tmp_line)\n", " tmp_line = []\n", " tmp_line.append(m)\n", " if len(tmp_line) > 0:\n", " lines.append(tmp_line)\n", " # simplify and classify as good breakpoints\n", " annotated_lines = []\n", " for line in lines:\n", " assert len(line) > 0\n", " text = \"\".join(m[\"word\"] for m in line)\n", " start_s = None\n", " end_s = None\n", " l = [\n", " m[\"start_s\"]\n", " for m in line[:2]\n", " if \"start_s\" in m and m[\"start_s\"] is not None\n", " ]\n", " if len(l) > 0:\n", " start_s = l[0]\n", " l = [m[\"end_s\"] for m in line[-2:] if \"end_s\" in m and m[\"end_s\"] is not None]\n", " if len(l) > 0:\n", " end_s = l[-1]\n", " success = (\n", " start_s is not None\n", " and end_s is not None\n", " and np.mean([m.get(\"success\", False) for m in line]) >= 0.75\n", " # do mean of success > 0.75\n", " )\n", " new_m = {\n", " \"text\": text,\n", " \"success\": success,\n", " }\n", " if success:\n", " new_m[\"start_s\"] = start_s\n", " new_m[\"end_s\"] = end_s\n", " annotated_lines.append(new_m)\n", " return annotated_lines\n", "\n", "\n", "def _collect_valid_segment_from_lines(annotated_lines):\n", " # group lines into segments\n", " valid_break_lines = [\n", " (n, line) for n, line in enumerate(annotated_lines) if line[\"success\"]\n", " ]\n", " segments = []\n", " tmp_segment = []\n", " for idx, line in valid_break_lines:\n", " try:\n", " # check if we wanna evict\n", " if len(tmp_segment) > 0 and (\n", " (\n", " (\n", " re.match(r\"^\\s*\\n\\s*\\n\\s*\", line[\"text\"])\n", " or re.match(r\"^\\s*\\[.*\\]\\s*$\", line[\"text\"])\n", " )\n", " and random.random() >= EARLY_VERSE_TERMINATE_CHANCE\n", " and len(normalize_whitespace(line[\"text\"])) >= 25\n", " )\n", " or line[\"end_s\"] - tmp_segment[0][-1][\"start_s\"]\n", " > MAX_SEGMENT_DURATION_S\n", " ):\n", " segments.append(tmp_segment)\n", " tmp_segment = []\n", " except:\n", " import pdb\n", "\n", " pdb.set_trace()\n", "\n", " tmp_segment.append((idx, line))\n", " if len(tmp_segment) > 0:\n", " segments.append(tmp_segment)\n", " # splice back in non-success lines\n", " all_segments = []\n", " offs = 0\n", " for seg in segments:\n", " first_idx = seg[0][0]\n", " last_idx = seg[-1][0]\n", " if first_idx > offs:\n", " all_segments.append(annotated_lines[offs:first_idx])\n", " all_segments.append(annotated_lines[first_idx : last_idx + 1])\n", " offs = last_idx + 1\n", " last_idx = segments[-1][-1][0] + 1\n", " if last_idx < len(annotated_lines):\n", " all_segments.append(annotated_lines[last_idx : len(annotated_lines)])\n", " # merge and filter segment lines\n", " # TODO: ignoring instrumental stuff here (add as empty string in future)\n", " merged_segments = []\n", " for lines in all_segments:\n", " assert len(lines) > 0\n", " text = \"\".join(m[\"text\"] for m in lines)\n", " start_s = lines[0].get(\"start_s\")\n", " end_s = lines[-1].get(\"end_s\")\n", " success = (\n", " start_s is not None\n", " and end_s is not None \n", " and np.mean([m[\"success\"] for m in lines]) >= 0.75\n", " # do mean of success > 0.75\n", " )\n", " if (\n", " not success\n", " or end_s - start_s > MAX_SEGMENT_DURATION_S\n", " or end_s - start_s < MIN_SEGMENT_DURATION_S\n", " ):\n", " continue\n", " merged_segments.append(\n", " {\n", " \"text\": text,\n", " \"start_s\": start_s,\n", " \"end_s\": end_s,\n", " }\n", " )\n", " return merged_segments\n", "\n", "\n", "def _verify_durations(segments, duration_s):\n", " assert duration_s + 0.01 > np.sum(\n", " [e[\"end_s\"] - e[\"start_s\"] for e in segments if e.get(\"start_s\") is not None]\n", " )" ] }, { "cell_type": "code", "execution_count": 11, "id": "3cd6dcbc", "metadata": { "ExecuteTime": { "end_time": "2023-10-16T20:11:12.523479Z", "start_time": "2023-10-16T20:11:12.513854Z" } }, "outputs": [], "source": [ "# arr[0]: probs.argmax(axis=-1), # decoded ID\n", "# arr[1]: probs.max(axis=-1), # prob decoded ID\n", "# arr[2]: probs[:,-1], # prob blank\n", "\n", "def _get_aligned_lyrics(align_metas, align_archive, silent=False):\n", " aligned_lyrics_map = {}\n", " tot_duration_s = 0\n", " tot_duration_silence_s = 0\n", " for m in tqdm.tqdm(align_metas, disable=silent):\n", " arr = align_archive[m[\"id\"]]\n", " if m[\"id\"] not in metas_map:\n", " continue\n", " meta = metas_map[m[\"id\"]]\n", " if meta[\"lang\"][:2].lower() != \"en\":\n", " continue\n", " if \"alignment\" not in m:\n", " continue\n", " aligned_words = m[\"alignment\"]\n", " if len(aligned_words) <= 50:\n", " continue\n", " duration_s = round(arr.shape[-1] / HOOT_EMBEDDING_RATE, 2)\n", " if duration_s < 30 or duration_s > 10*60 or np.abs(meta[\"duration_s\"] - duration_s) > 5:\n", " continue\n", " # check language and alignment quality\n", " p_silence = round(arr[2].mean(), 3)\n", " weights = 1 - arr[2]\n", " p_en = round((arr[1]*weights).sum() / weights.sum(), 3)\n", " p_align = np.mean([mm[\"p_align\"] for mm in aligned_words])\n", " if p_en < 0.65 or p_align < 0.1:\n", " continue\n", " true_text_norm = _norm_lyrics_to_plain(meta[\"lyrics\"])\n", " greedy_preds = decode_token_ids(arr[0])\n", " cer_val = round(get_cer(true_text_norm, greedy_preds), 3)\n", " if cer_val > 0.75:\n", " continue\n", " # do alignment\n", " all_words, words_to_align = _get_alignable_tokens(meta[\"lyrics\"])\n", " try:\n", " timed_words = _assign_word_timings(words_to_align, aligned_words)\n", " except:\n", " # sometimes error on np.mean with None\n", " continue\n", " # add into full words\n", " timed_words_map = {m[\"orig_idx\"]: m for m in timed_words}\n", " all_words_timed = []\n", " for n, w in enumerate(all_words):\n", " if n not in timed_words_map:\n", " all_words_timed.append({\"word\": w})\n", " continue\n", " assert(w.lower() == timed_words_map[n][\"word\"]) \n", " new_m = {\n", " \"word\": w,\n", " \"success\": timed_words_map[n][\"start_s\"] is not None,\n", " \"start_s\": timed_words_map[n][\"start_s\"],\n", " \"end_s\": timed_words_map[n][\"end_s\"],\n", " \"p_align\": timed_words_map[n][\"p_align\"],\n", " }\n", " all_words_timed.append(new_m)\n", " # merge meta tags forward for timing (eg [Verse 1])\n", " merged_words_timed = _merge_metatags_whitespace(all_words_timed)\n", " try:\n", " _verify_durations(merged_words_timed, duration_s)\n", " except:\n", " # this shouldn't happen but is super rare so fine for noq\n", " continue\n", " # pad words if possible\n", " padded_words_timed = _pad_words(merged_words_timed, duration_s)\n", " try:\n", " _verify_durations(padded_words_timed, duration_s)\n", " except:\n", " # this shouldn't happen but is super rare so fine for noq\n", " continue\n", " # merge into lines\n", " annotated_lines = _merge_into_lines(padded_words_timed)\n", " try:\n", " _verify_durations(annotated_lines, duration_s)\n", " except:\n", " # this shouldn't happen but is super rare so fine for noq\n", " continue\n", " # merge into segments and filer\n", " valid_segments = _collect_valid_segment_from_lines(annotated_lines)\n", " try:\n", " _verify_durations(valid_segments, duration_s)\n", " except:\n", " # this shouldn't happen but is super rare so fine for noq\n", " continue\n", " # if less than hald the string aligned maybe skip as well\n", " retained_text = \"\".join(m[\"text\"] for m in valid_segments)\n", " if len(retained_text) / len(meta[\"lyrics\"]) < 0.5:\n", " continue\n", " assert(all(\n", " MIN_SEGMENT_DURATION_S - 0.1 <= m[\"end_s\"] - m[\"start_s\"] <= MAX_SEGMENT_DURATION_S + 0.1 \n", " for m in valid_segments\n", " ))\n", " aligned_lyrics_map[m[\"id\"]] = valid_segments\n", " return aligned_lyrics_map\n", "\n", "def collect_lyrics(work_item):\n", " archive_nr = work_item\n", " base_url = \"s3://suno-data/datasets/bundles/v1/genius_hq\"\n", " align_metas = read_from_s3(\n", " os.path.join(base_url, \"hoot_logits\", \"metas\", f\"part_{archive_nr}.jsonl\"), read_f=read_jsonl\n", " )\n", " align_archive = read_from_s3(\n", " os.path.join(base_url, \"hoot_logits\", f\"part_{archive_nr}.npz\"), read_f=np.load\n", " )\n", " aligned_lyrics_map = _get_aligned_lyrics(align_metas, align_archive, silent=True)\n", " return [(k, v) for k, v in aligned_lyrics_map.items()]" ] }, { "cell_type": "code", "execution_count": 12, "id": "04a716c4", "metadata": { "ExecuteTime": { "end_time": "2023-10-16T20:13:46.498307Z", "start_time": "2023-10-16T20:13:28.012493Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "253/1000, retained\n" ] } ], "source": [ "# test output\n", "out = collect_lyrics(0)\n", "print(f\"{len(out)}/1000, retained\")\n", "# 255/1000, retained\n", "# should be ~500k songs retained\n", "# should be ~30k hours max (less cause segments)" ] }, { "cell_type": "code", "execution_count": 13, "id": "93991c09", "metadata": { "ExecuteTime": { "end_time": "2023-10-16T20:14:02.296780Z", "start_time": "2023-10-16T20:14:01.528581Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "12.1 hours retained\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAi6klEQVR4nO3de3BU9f3/8VcuyyYBEgyYhBQC8Ua4FSjXCOONkIB8USStoEhRGZ3SBAtpEVC5ekFQ8UIjDB2LODW2ZaYglxqICEHGABJKHZSJgFQUSGihSQzIsrLn94c/tsZwSWB3zyc5z8fMDrPnnJx9v5KwvDjn7G6YZVmWAAAADBJu9wAAAAA/RkEBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABgn0u4BroTP59PRo0fVsmVLhYWF2T0OAACoB8uy9M033yg5OVnh4Zc+RtIoC8rRo0fVvn17u8cAAABX4KuvvlK7du0uuU2jLCgtW7aU9H3A2NhYm6e5cl6vVxs3blRmZqZcLpfd44SE0zI7La/kvMxOyys5L7PT8krBy1xdXa327dv7/x2/lEZZUM6f1omNjW30BSUmJkaxsbGO+qV3Uman5ZWcl9lpeSXnZXZaXin4metzeQYXyQIAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYJ9LuAQAAQGh1nL7+kuvdEZYW9gvRMBfBERQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADBOpN0DAACAy+s4fX29tvvX88ODPElocAQFAAAYh4ICAACMQ0EBAADGoaAAAADjNKigzJ8/X3379lXLli2VkJCgkSNHqqysrNY2Z86cUU5Ojlq3bq0WLVooOztbFRUVtbY5fPiwhg8frpiYGCUkJGjq1Kn67rvvrj4NAABoEhpUUIqLi5WTk6Pt27erqKhIXq9XmZmZOnXqlH+bKVOmaO3atVq5cqWKi4t19OhRjRo1yr/+3LlzGj58uM6ePauPPvpIK1as0JtvvqlZs2YFLhUAAGjUGvQy48LCwlr333zzTSUkJKi0tFS33HKLqqqq9MYbb6igoEB33HGHJGn58uXq3Lmztm/frgEDBmjjxo367LPP9P777ysxMVE9e/bU008/rWnTpmnOnDlq1qxZ4NIBAIBG6areB6WqqkqSFB8fL0kqLS2V1+tVRkaGf5u0tDSlpKSopKREAwYMUElJibp3767ExET/NllZWZo4caI+/fRT9erV62pGAgDA0er7fimmu+KC4vP5NHnyZA0cOFDdunWTJJWXl6tZs2Zq1apVrW0TExNVXl7u3+aH5eT8+vPrLsTj8cjj8fjvV1dXS5K8Xq+8Xu+VRrDd+dkbc4aGclpmp+WVnJfZaXkl52U2Ja87wgrdY4V//1iBztyQ/V1xQcnJydHevXu1bdu2K91Fvc2fP19z586ts3zjxo2KiYkJ+uMHW1FRkd0jhJzTMjstr+S8zE7LKzkvs915F/YL/WMGOvPp06frve0VFZTc3FytW7dOW7duVbt27fzLk5KSdPbsWVVWVtY6ilJRUaGkpCT/Njt37qy1v/Ov8jm/zY/NmDFDeXl5/vvV1dVq3769MjMzFRsbeyURjOD1elVUVKQhQ4bI5XLZPU5IOC2z0/JKzsvstLyS8zKbkrfbnA0heyx3uKWn+/gCnvn8GZD6aFBBsSxLkyZN0qpVq7RlyxalpqbWWt+7d2+5XC5t2rRJ2dnZkqSysjIdPnxY6enpkqT09HQ9++yzOn78uBISEiR939BiY2PVpUuXCz6u2+2W2+2us9zlcjWJvxxNJUdDOC2z0/JKzsvstLyS8zLbnddzLizkjxnozA3ZV4MKSk5OjgoKCvTuu++qZcuW/mtG4uLiFB0drbi4OE2YMEF5eXmKj49XbGysJk2apPT0dA0YMECSlJmZqS5dumjcuHFauHChysvL9dRTTyknJ+eCJQQAADhPgwrKkiVLJEm33XZbreXLly/Xgw8+KEl6+eWXFR4eruzsbHk8HmVlZen111/3bxsREaF169Zp4sSJSk9PV/PmzTV+/HjNmzfv6pIAAIAmo8GneC4nKipK+fn5ys/Pv+g2HTp00N///veGPDQAAHAQPosHAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEa9GnGAACgYTpOX2/3CI0SR1AAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGCcSLsHAADARB2nr5ckuSMsLewndZuzQZ5zYbW2+dfzw+0YzRE4ggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMaJbOgXbN26VS+88IJKS0t17NgxrVq1SiNHjvSvf/DBB7VixYpaX5OVlaXCwkL//ZMnT2rSpElau3atwsPDlZ2drVdffVUtWrS48iQAANRTx+nr7R4Bl9HgIyinTp1Sjx49lJ+ff9Fthg4dqmPHjvlv77zzTq31Y8eO1aeffqqioiKtW7dOW7du1aOPPtrw6QEAQJPU4CMow4YN07Bhwy65jdvtVlJS0gXX7du3T4WFhfr444/Vp08fSdLixYt155136sUXX1RycnJDRwIAAE1MUK5B2bJlixISEtSpUydNnDhRJ06c8K8rKSlRq1at/OVEkjIyMhQeHq4dO3YEYxwAANDINPgIyuUMHTpUo0aNUmpqqg4ePKgnnnhCw4YNU0lJiSIiIlReXq6EhITaQ0RGKj4+XuXl5Rfcp8fjkcfj8d+vrq6WJHm9Xnm93kBHCJnzszfmDA3ltMxOyys5L7PT8kpNI7M7wqr/tuFWrT9/qD7fg4Y8linOZw30z7gh+wuzLOuKv3NhYWF1LpL9sS+++ELXX3+93n//fQ0ePFjPPfecVqxYobKyslrbJSQkaO7cuZo4cWKdfcyZM0dz586ts7ygoEAxMTFXOj4AAAih06dP6/7771dVVZViY2MvuW3Aj6D82HXXXac2bdrowIEDGjx4sJKSknT8+PFa23z33Xc6efLkRa9bmTFjhvLy8vz3q6ur1b59e2VmZl42oMm8Xq+Kioo0ZMgQuVwuu8cJCadldlpeyXmZnZZXahqZu83ZUO9t3eGWnu7j08xd4fL4wmqt2zsnK6CPZYrzmQP9Mz5/BqQ+gl5Qvv76a504cUJt27aVJKWnp6uyslKlpaXq3bu3JOmDDz6Qz+dT//79L7gPt9stt9tdZ7nL5Wq0fzl+qKnkaAinZXZaXsl5mZ2WV2rcmT3nwi6/0Y+/xhdW5+vqk/9KHssUgf4ZN2RfDS4oNTU1OnDggP/+oUOHtGfPHsXHxys+Pl5z585Vdna2kpKSdPDgQT3++OO64YYblJX1fcvs3Lmzhg4dqkceeURLly6V1+tVbm6uxowZwyt4AACApCt4Fc+uXbvUq1cv9erVS5KUl5enXr16adasWYqIiNAnn3yiu+66SzfddJMmTJig3r1768MPP6x1BOTtt99WWlqaBg8erDvvvFODBg3SsmXLApcKAAA0ag0+gnLbbbfpUtfVbthw+XNt8fHxKigoaOhDAwAAh+CzeAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACME2n3AAAANFYdp6+3e4QmiyMoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAc3uoeANCk8PbzTQNHUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4DS4oW7du1YgRI5ScnKywsDCtXr261nrLsjRr1iy1bdtW0dHRysjI0P79+2ttc/LkSY0dO1axsbFq1aqVJkyYoJqamqsKAgAAmo4GF5RTp06pR48eys/Pv+D6hQsX6rXXXtPSpUu1Y8cONW/eXFlZWTpz5ox/m7Fjx+rTTz9VUVGR1q1bp61bt+rRRx+98hQAAKBJiWzoFwwbNkzDhg274DrLsvTKK6/oqaee0t133y1Jeuutt5SYmKjVq1drzJgx2rdvnwoLC/Xxxx+rT58+kqTFixfrzjvv1Isvvqjk5OSriAMAAJqCBheUSzl06JDKy8uVkZHhXxYXF6f+/furpKREY8aMUUlJiVq1auUvJ5KUkZGh8PBw7dixQ/fcc0+d/Xo8Hnk8Hv/96upqSZLX65XX6w1khJA6P3tjztBQTsvstLyS8zI7La9kfmZ3hBXY/YVbtf50gvNZA/0zbsj+AlpQysvLJUmJiYm1licmJvrXlZeXKyEhofYQkZGKj4/3b/Nj8+fP19y5c+ss37hxo2JiYgIxuq2KiorsHiHknJbZaXkl52V2Wl7J3MwL+wVnv0/38QVnxwYL9M/49OnT9d42oAUlWGbMmKG8vDz//erqarVv316ZmZmKjY21cbKr4/V6VVRUpCFDhsjlctk9Tkg4LbPT8krOy+y0vJL5mbvN2RDQ/bnDLT3dx6eZu8Ll8YUFdN+mOp850D/j82dA6iOgBSUpKUmSVFFRobZt2/qXV1RUqGfPnv5tjh8/XuvrvvvuO508edL/9T/mdrvldrvrLHe5XEb+5WioppKjIZyW2Wl5JedldlpeydzMnnPBKREeX1jQ9m2qQP+MG7KvgL4PSmpqqpKSkrRp0yb/surqau3YsUPp6emSpPT0dFVWVqq0tNS/zQcffCCfz6f+/fsHchwAANBINfgISk1NjQ4cOOC/f+jQIe3Zs0fx8fFKSUnR5MmT9cwzz+jGG29UamqqZs6cqeTkZI0cOVKS1LlzZw0dOlSPPPKIli5dKq/Xq9zcXI0ZM4ZX8AAAAElXUFB27dql22+/3X///LUh48eP15tvvqnHH39cp06d0qOPPqrKykoNGjRIhYWFioqK8n/N22+/rdzcXA0ePFjh4eHKzs7Wa6+9FoA4AACgKWhwQbnttttkWRd/qVVYWJjmzZunefPmXXSb+Ph4FRQUNPShAQCAQ/BZPAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDiRdg8AAEB9dZy+3u4RECIcQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIf3QQEAGIH3OMEPcQQFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxeBUPACDoeIUOGoojKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeLZAEAV4ULYBEMHEEBAADGoaAAAADjUFAAAIBxKCgAAMA4XCQLALioC10A646wtLCf1G3OBnnOhdkwFZyAIygAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgnIAXlDlz5igsLKzWLS0tzb/+zJkzysnJUevWrdWiRQtlZ2eroqIi0GMAAIBGLChHULp27apjx475b9u2bfOvmzJlitauXauVK1equLhYR48e1ahRo4IxBgAAaKQig7LTyEglJSXVWV5VVaU33nhDBQUFuuOOOyRJy5cvV+fOnbV9+3YNGDAgGOMAAIBGJigFZf/+/UpOTlZUVJTS09M1f/58paSkqLS0VF6vVxkZGf5t09LSlJKSopKSkosWFI/HI4/H479fXV0tSfJ6vfJ6vcGIEBLnZ2/MGRrKaZmdlldyXuamntcdYdVdFm7V+rOpc1pe6X9ZA/173ZD9hVmWFdDv+Hvvvaeamhp16tRJx44d09y5c3XkyBHt3btXa9eu1UMPPVSrbEhSv379dPvtt2vBggUX3OecOXM0d+7cOssLCgoUExMTyPEBAECQnD59Wvfff7+qqqoUGxt7yW0DXlB+rLKyUh06dNCiRYsUHR19RQXlQkdQ2rdvr//85z+XDWgyr9eroqIiDRkyRC6Xy+5xQsJpmZ2WV3Je5qaet9ucDXWWucMtPd3Hp5m7wuXxhdkwVWg5La/0v8yB/r2urq5WmzZt6lVQgnKK54datWqlm266SQcOHNCQIUN09uxZVVZWqlWrVv5tKioqLnjNynlut1tut7vOcpfL1SSeEJpKjoZwWman5ZWcl7mp5vWcu/g/yB5f2CXXNzVOyysF/ve6IfsK+vug1NTU6ODBg2rbtq169+4tl8ulTZs2+deXlZXp8OHDSk9PD/YoAACgkQj4EZTf/e53GjFihDp06KCjR49q9uzZioiI0H333ae4uDhNmDBBeXl5io+PV2xsrCZNmqT09HRewQMAAPwCXlC+/vpr3XfffTpx4oSuvfZaDRo0SNu3b9e1114rSXr55ZcVHh6u7OxseTweZWVl6fXXXw/0GADgaB2nr7d7BOCqBLyg/PnPf77k+qioKOXn5ys/Pz/QDw0AAJoIPosHAAAYh4ICAACMQ0EBAADGCfr7oAAAAosLYOEEHEEBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA40TaPQAAOEXH6evtHgFoNDiCAgAAjENBAQAAxuEUDwBcxqVOzbgjLC3sF8JhAIfgCAoAADAOR1AAIAC6zdkgz7kwu8cAmgyOoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeXGQNolOrzuTb/en54CCYBEAwcQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIf3QQEMFaj3+eD9QgA0RhxBAQAAxuEICgBHq88RJgChR0GBI3Ca4+rV9x9yd4Slhf2kbnM2yHMurM56vs8A6oNTPAAAwDgUFAAAYBwKCgAAMA7XoKAWrtW4ej/8Hl7segy+h6HBBbBA40VBARrAtH/wTJunPijBAOqDUzwAAMA4HEFpIvhfKQCgKaGgoMEoQ1evMZ6aCSW+PwA4xQMAAIzDEZQrxFGEpof/tQOAOSgoAChnAIzDKR4AAGAcjqAY4GIfqtaYXex/5D9847KyZ/8vqI8FAGi8OIICAACMwxEU2IYLjQEAF0NBuQBOGQAAYC9O8QAAAONwBCWILnck5vwFo7g4jmYBgDNxBAUAABiHggIAAIzDKR4H4XQJAKCx4AgKAAAwjq0FJT8/Xx07dlRUVJT69++vnTt32jkOAAAwhG0F5S9/+Yvy8vI0e/Zs7d69Wz169FBWVpaOHz9u10gAAMAQthWURYsW6ZFHHtFDDz2kLl26aOnSpYqJidEf//hHu0YCAACGsOUi2bNnz6q0tFQzZszwLwsPD1dGRoZKSkrqbO/xeOTxePz3q6qqJEknT56U1+sN+HyR350K+D4v+Dg+S6dP+xTpDdc5X9P6sMCLcVpmp+WVnJfZaXkl52V2Wl7pf5lPnDghl8sVsP1+8803kiTLsi6/sWWDI0eOWJKsjz76qNbyqVOnWv369auz/ezZsy1J3Lhx48aNG7cmcPvqq68u2xUaxcuMZ8yYoby8PP99n8+nkydPqnXr1goLa7xttrq6Wu3bt9dXX32l2NhYu8cJCadldlpeyXmZnZZXcl5mp+WVgpfZsix98803Sk5Ovuy2thSUNm3aKCIiQhUVFbWWV1RUKCkpqc72brdbbre71rJWrVoFc8SQio2Ndcwv/XlOy+y0vJLzMjstr+S8zE7LKwUnc1xcXL22s+Ui2WbNmql3797atGmTf5nP59OmTZuUnp5ux0gAAMAgtp3iycvL0/jx49WnTx/169dPr7zyik6dOqWHHnrIrpEAAIAhbCsoo0eP1r///W/NmjVL5eXl6tmzpwoLC5WYmGjXSCHndrs1e/bsOqevmjKnZXZaXsl5mZ2WV3JeZqfllczIHGZZ9XmtDwAAQOjwWTwAAMA4FBQAAGAcCgoAADAOBQUAABiHgmKTI0eO6IEHHlDr1q0VHR2t7t27a9euXXaPFTTnzp3TzJkzlZqaqujoaF1//fV6+umn6/d5DI3A1q1bNWLECCUnJyssLEyrV6+utd6yLM2aNUtt27ZVdHS0MjIytH//fnuGDZBLZfZ6vZo2bZq6d++u5s2bKzk5Wb/85S919OhR+wa+Spf7Gf/Qr371K4WFhemVV14J2XzBUJ/M+/bt01133aW4uDg1b95cffv21eHDh0M/bABcLm9NTY1yc3PVrl07RUdH+z/otrGaP3+++vbtq5YtWyohIUEjR45UWVlZrW3OnDmjnJwctW7dWi1atFB2dnadN1kNFgqKDf773/9q4MCBcrlceu+99/TZZ5/ppZde0jXXXGP3aEGzYMECLVmyRL///e+1b98+LViwQAsXLtTixYvtHi0gTp06pR49eig/P/+C6xcuXKjXXntNS5cu1Y4dO9S8eXNlZWXpzJkzIZ40cC6V+fTp09q9e7dmzpyp3bt3629/+5vKysp011132TBpYFzuZ3zeqlWrtH379nq9lbfpLpf54MGDGjRokNLS0rRlyxZ98sknmjlzpqKiokI8aWBcLm9eXp4KCwv1pz/9Sfv27dPkyZOVm5urNWvWhHjSwCguLlZOTo62b9+uoqIieb1eZWZm6tSp/31g7pQpU7R27VqtXLlSxcXFOnr0qEaNGhWaAQPx4X9omGnTplmDBg2ye4yQGj58uPXwww/XWjZq1Chr7NixNk0UPJKsVatW+e/7fD4rKSnJeuGFF/zLKisrLbfbbb3zzjs2TBh4P858ITt37rQkWV9++WVohgqii+X9+uuvrZ/85CfW3r17rQ4dOlgvv/xyyGcLlgtlHj16tPXAAw/YM1CQXShv165drXnz5tVa9rOf/cx68sknQzhZ8Bw/ftySZBUXF1uW9f3zlMvlslauXOnfZt++fZYkq6SkJOjzcATFBmvWrFGfPn30i1/8QgkJCerVq5f+8Ic/2D1WUN18883atGmTPv/8c0nSP//5T23btk3Dhg2zebLgO3TokMrLy5WRkeFfFhcXp/79+6ukpMTGyUKrqqpKYWFhTepztH7I5/Np3Lhxmjp1qrp27Wr3OEHn8/m0fv163XTTTcrKylJCQoL69+9/yVNfjd3NN9+sNWvW6MiRI7IsS5s3b9bnn3+uzMxMu0cLiKqqKklSfHy8JKm0tFRer7fWc1daWppSUlJC8txFQbHBF198oSVLlujGG2/Uhg0bNHHiRD322GNasWKF3aMFzfTp0zVmzBilpaXJ5XKpV69emjx5ssaOHWv3aEFXXl4uSXXeJTkxMdG/rqk7c+aMpk2bpvvuu6/JftjaggULFBkZqccee8zuUULi+PHjqqmp0fPPP6+hQ4dq48aNuueeezRq1CgVFxfbPV5QLF68WF26dFG7du3UrFkzDR06VPn5+brlllvsHu2q+Xw+TZ48WQMHDlS3bt0kff/c1axZszr/qQjVc5dtb3XvZD6fT3369NFzzz0nSerVq5f27t2rpUuXavz48TZPFxx//etf9fbbb6ugoEBdu3bVnj17NHnyZCUnJzfZzPie1+vVvffeK8uytGTJErvHCYrS0lK9+uqr2r17t8LCwuweJyR8Pp8k6e6779aUKVMkST179tRHH32kpUuX6tZbb7VzvKBYvHixtm/frjVr1qhDhw7aunWrcnJylJycXOsoQ2OUk5OjvXv3atu2bXaP4scRFBu0bdtWXbp0qbWsc+fOjfbK9/qYOnWq/yhK9+7dNW7cOE2ZMkXz58+3e7SgS0pKkqQ6V75XVFT41zVV58vJl19+qaKioiZ79OTDDz/U8ePHlZKSosjISEVGRurLL7/Ub3/7W3Xs2NHu8YKiTZs2ioyMdMxz2bfffqsnnnhCixYt0ogRI/TTn/5Uubm5Gj16tF588UW7x7squbm5WrdunTZv3qx27dr5lyclJens2bOqrKystX2onrsoKDYYOHBgnZdyff755+rQoYNNEwXf6dOnFR5e+9ctIiLC/7+wpiw1NVVJSUnatGmTf1l1dbV27Nih9PR0GycLrvPlZP/+/Xr//ffVunVru0cKmnHjxumTTz7Rnj17/Lfk5GRNnTpVGzZssHu8oGjWrJn69u3rmOcyr9crr9fbpJ7HLMtSbm6uVq1apQ8++ECpqam11vfu3Vsul6vWc1dZWZkOHz4ckucuTvHYYMqUKbr55pv13HPP6d5779XOnTu1bNkyLVu2zO7RgmbEiBF69tlnlZKSoq5du+of//iHFi1apIcfftju0QKipqZGBw4c8N8/dOiQ9uzZo/j4eKWkpGjy5Ml65plndOONNyo1NVUzZ85UcnKyRo4cad/QV+lSmdu2bauf//zn2r17t9atW6dz5875z1nHx8erWbNmdo19xS73M/5xAXO5XEpKSlKnTp1CPWrAXC7z1KlTNXr0aN1yyy26/fbbVVhYqLVr12rLli32DX0VLpf31ltv1dSpUxUdHa0OHTqouLhYb731lhYtWmTj1FcuJydHBQUFevfdd9WyZUv/39G4uDhFR0crLi5OEyZMUF5enuLj4xUbG6tJkyYpPT1dAwYMCP6AQX+dEC5o7dq1Vrdu3Sy3222lpaVZy5Yts3ukoKqurrZ+85vfWCkpKVZUVJR13XXXWU8++aTl8XjsHi0gNm/ebEmqcxs/frxlWd+/1HjmzJlWYmKi5Xa7rcGDB1tlZWX2Dn2VLpX50KFDF1wnydq8ebPdo1+Ry/2Mf6wpvMy4PpnfeOMN64YbbrCioqKsHj16WKtXr7Zv4Kt0ubzHjh2zHnzwQSs5OdmKioqyOnXqZL300kuWz+ezd/ArdLG/o8uXL/dv8+2331q//vWvrWuuucaKiYmx7rnnHuvYsWMhmS/s/w8JAABgDK5BAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4/w8bT+KFEqNf2AAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# plot duration hist and duration retained\n", "l = []\n", "for _, m in out:\n", " for mm in m:\n", " l.append((mm[\"end_s\"] - mm[\"start_s\"]))\n", "print(round(np.sum(l)/60/60, 1), \"hours retained\")\n", "pd.Series(l).hist(bins=50);\n", "# 12.9 hours retained" ] }, { "cell_type": "code", "execution_count": 19, "id": "5f106b58", "metadata": { "ExecuteTime": { "end_time": "2023-10-16T20:18:29.884331Z", "start_time": "2023-10-16T20:18:25.742149Z" }, "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "b7323f28-ca3b-4ada-a810-8a7456843581\n", "070-shake-rocketship-lyrics\n", "----------\n", "[Verse 1]\n", "Yeah, I heard the message you relayed, oh woah-woah, woah\n", "Say I always go away, oh woah-woah, woah\n", "Ask how long I'll be this way, oh woah-woah, woah\n", "You don't know if you can wait, oh woah-woah, woah, hm\n", "Baby think it's straight, oh woah-woah, woah, yeah\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "How long I'll be this way? Mm\n", "\n", "[Refrain]\n", "Forever (Say, hm), forever (Yeah)\n", "Forever (Say, hm), forever\n", "Forever (Forever)\n", "Forever (Forever)\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "[Chorus]\n", "It feel like I'm on a rocket ship, rocket ship\n", "It feel like I'm on a rocket ship, rocket ship\n", "And ain't nobody stoppin' this rocket ship\n", "Ain't nobody stoppin' this rocket ship (How long?)\n", "\n", "[Verse 2]\n", "Yeah, I leave for a month, couple weeks ain't enough\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "I can't be here for fun, yeah, you see what that does\n", "I gave you all in one, triple three sum it up\n", "I'm in need for that rush, like the tree need the sun\n", "And there's something 'bout it\n", "Can't have fun without it (No)\n", "Start to form a habit (Woah)\n", "Then become an addict (Uh-uh)\n", "Movin' slow, then increasin' speed\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "Take it off, now we sittin' free\n", "\n", "[Chorus]\n", "It feel like I'm on a rocket ship, rocket ship\n", "It feel like I'm on a rocket ship, rocket ship\n", "And ain't nobody stoppin' this rocket ship\n", "And ain't nobody stoppin' this rocket ship (How long?)\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "\n", "[Refrain]\n", "Forever (Forever)\n", "Forever (Forever)\n", "\n", "\n", "[Outro]\n", "Can't find the one, I guess I'm better off\n", "Now know it's genuine, I just can't tell at all\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "I, I, I really wanna have you\n", "I'll be your backing\n", "Right now the moon feels lighter than the sun\n", "Rocket ship, rocket ship\n", "It feel like I'm on a rocket ship, rocket ship\n", "And ain't nobody stoppin' this rocket ship\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "Right now the moon feels lighter than the sun\n", "Forever (Forever)\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "Forever (Forever)\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n" ] } ], "source": [ "# # listen to some\n", "k, segments = random.choice(out)\n", "meta = metas_map[k]\n", "audio = Audio.from_s3(meta[\"audio_filepath\"])\n", "print(f\"{meta['id']}\")\n", "print(meta[\"genius_slug\"])\n", "print(\"-\"*10)\n", "for s in segments:\n", " print(s[\"text\"])\n", " audio.get_segment(s[\"start_s\"], s[\"end_s\"]).play()\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 21, "id": "f7462849", "metadata": { "ExecuteTime": { "end_time": "2023-10-16T20:29:11.710762Z", "start_time": "2023-10-16T20:29:11.708424Z" } }, "outputs": [], "source": [ "import warnings\n", "warnings.filterwarnings(\"ignore\")" ] }, { "cell_type": "code", "execution_count": null, "id": "6f809ee6", "metadata": { "ExecuteTime": { "start_time": "2023-10-16T20:29:27.305Z" }, "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 0/44 [00:00= 5 and len(v) <= 15 and max_n_tokens <= 100:\n", " sample_data.append((k, v))\n", " n += 1\n", " if n == n_max_sample:\n", " break\n", "# print(int(len(unique_ids)*n_tot_rows/n_max_sample), \"items\")\n", "print(round(tot_duration_s*n_tot_rows/n_max_sample/60/60,1), \"hours\")\n", "print(round(tot_duration_silence_s*n_tot_rows/n_max_sample/60/60,1), \"hours silence\")\n", "## last round\n", "# 748206 items\n", "# 34517.4 hours\n", "# 7617.4 hours silence\n", "## this round (approx)\n", "# 687017 items\n", "# 32635.1 hours\n", "# 0.0 hours silence\n", "## this round (approx)\n", "# 685557 items\n", "# 35829.3 hours\n", "# 0.0 hours silence" ] }, { "cell_type": "code", "execution_count": 32, "id": "e60f82fc", "metadata": { "ExecuteTime": { "end_time": "2023-10-19T19:50:20.092386Z", "start_time": "2023-10-19T19:50:19.928842Z" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAui0lEQVR4nO3df3RTdZ7/8Vdb0kCBFMFp0y6ldmAGWig/hBEyOhyU0opdRkd2ZxAERlEWTnEW6iJ2voAFRKQOIirCuP7APUtHcY64CkgbQECkgFQ6CLgdZcCq0HaPSCNUQmjz/YNpxgi0BNskn/J8nJNzvPe+k7zvmxBf3HuTRHi9Xq8AAAAMEhnqBgAAAAJFgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGKdNqBtoKfX19Tp27Jg6duyoiIiIULcDAAAug9fr1TfffKPExERFRl76OEurDTDHjh1TUlJSqNsAAABX4PPPP1fXrl0vuT2gALNixQqtWLFCR48elST17t1bc+fO1ciRIyVJw4YN07Zt2/zu82//9m9auXKlb7miokJTp07Vu+++qw4dOmjixIlatGiR2rT5Rytbt25Vbm6uDh48qKSkJM2ePVu//e1vA2lVHTt2lHR+ADabLaD7tjYej0fFxcXKzMyUxWIJdTutGrMODuYcHMw5OJizP5fLpaSkJN//xy8loADTtWtXPf744/rJT34ir9erV155Rbfffrv27dun3r17S5Luv/9+zZ8/33efmJgY33/X1dUpOztbdrtdO3fu1PHjxzVhwgRZLBY99thjkqQjR44oOztbU6ZM0erVq7V582bdd999SkhIUFZW1mX32nDayGazEWA8HsXExMhms/GXo4Ux6+BgzsHBnIODOV9cU5d/BBRgRo0a5be8cOFCrVixQrt27fIFmJiYGNnt9ovev7i4WIcOHdKmTZsUHx+v/v37a8GCBZo1a5by8/MVHR2tlStXKiUlRUuWLJEkpaamaseOHVq6dGlAAQYAALReV/wppLq6Or366qs6ffq0HA6Hb/3q1at17bXXqk+fPsrLy1Ntba1vW0lJidLT0xUfH+9bl5WVJZfLpYMHD/pqMjIy/J4rKytLJSUlV9oqAABoZQK+iPejjz6Sw+HQmTNn1KFDB61du1ZpaWmSpLFjxyo5OVmJiYnav3+/Zs2apfLycr3xxhuSpMrKSr/wIsm3XFlZ2WiNy+XSt99+q3bt2l20L7fbLbfb7Vt2uVySzh+a83g8ge5mq9Kw/1f7HIKBWQcHcw4O5hwczNnf5c4h4ADTs2dPlZWVqaamRn/+8581ceJEbdu2TWlpaZo8ebKvLj09XQkJCRo+fLgOHz6s7t27B/pUAVm0aJHmzZt3wfri4mK/63CuZk6nM9QtXDWYdXAw5+BgzsHBnM/77pmbxgQcYKKjo9WjRw9J0sCBA/XBBx9o2bJl+uMf/3hB7eDBgyVJn376qbp37y673a49e/b41VRVVUmS77oZu93uW/fdGpvNdsmjL5KUl5en3Nxc33LDVcyZmZlcxOvxyOl0asSIEVwg1sKYdXAw5+BgzsHBnP01nEFpyg/+Hpj6+nq/UzffVVZWJklKSEiQJDkcDi1cuFDV1dWKi4uTdD5x2mw232koh8OhDRs2+D2O0+n0u87mYqxWq6xW6wXrLRYLL4i/YxbBw6yDgzkHB3MODuZ83uXOIKAAk5eXp5EjR6pbt2765ptvVFhYqK1bt6qoqEiHDx9WYWGhbrvtNnXp0kX79+/XjBkzNHToUPXt21eSlJmZqbS0NI0fP14FBQWqrKzU7NmzlZOT4wsfU6ZM0bPPPquHHnpI9957r7Zs2aI1a9Zo/fr1AY4AAAC0VgEFmOrqak2YMEHHjx9XbGys+vbtq6KiIo0YMUKff/65Nm3apKeeekqnT59WUlKSRo8erdmzZ/vuHxUVpXXr1mnq1KlyOBxq3769Jk6c6Pe9MSkpKVq/fr1mzJihZcuWqWvXrnrhhRf4CDUAAPAJKMC8+OKLl9yWlJR0wbfwXkxycvIFp4i+b9iwYdq3b18grQEAgKsIv0YNAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4P/iL7AAgVPrkF8ldF3HJ7Ucfzw5iNwCCiSMwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgnDahbgAArhbXPby+yZqjj2cHoRPAfByBAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACME1CAWbFihfr27SubzSabzSaHw6F33nnHt/3MmTPKyclRly5d1KFDB40ePVpVVVV+j1FRUaHs7GzFxMQoLi5OM2fO1Llz5/xqtm7dquuvv15Wq1U9evTQqlWrrnwPAQBAqxNQgOnatasef/xxlZaWau/evbrlllt0++236+DBg5KkGTNm6O2339brr7+ubdu26dixY7rzzjt996+rq1N2drbOnj2rnTt36pVXXtGqVas0d+5cX82RI0eUnZ2tm2++WWVlZZo+fbruu+8+FRUVNdMuAwAA0wX0UwKjRo3yW164cKFWrFihXbt2qWvXrnrxxRdVWFioW265RZL08ssvKzU1Vbt27dKQIUNUXFysQ4cOadOmTYqPj1f//v21YMECzZo1S/n5+YqOjtbKlSuVkpKiJUuWSJJSU1O1Y8cOLV26VFlZWc202wAAwGRXfA1MXV2dXn31VZ0+fVoOh0OlpaXyeDzKyMjw1fTq1UvdunVTSUmJJKmkpETp6emKj4/31WRlZcnlcvmO4pSUlPg9RkNNw2MAAAAE/GOOH330kRwOh86cOaMOHTpo7dq1SktLU1lZmaKjo9WpUye/+vj4eFVWVkqSKisr/cJLw/aGbY3VuFwuffvtt2rXrt1F+3K73XK73b5ll8slSfJ4PPJ4PIHuZqvSsP9X+xyCgVkHR8N8rZHey6oLF9aoxvuVwqtnXs/BwZz9Xe4cAg4wPXv2VFlZmWpqavTnP/9ZEydO1LZt2wJusLktWrRI8+bNu2B9cXGxYmJiQtBR+HE6naFu4arBrINjwaD6Rrdv2LAhSJ1cnoIbmq4Jt54lXs/BwpzPq62tvay6gANMdHS0evToIUkaOHCgPvjgAy1btky/+c1vdPbsWZ08edLvKExVVZXsdrskyW63a8+ePX6P1/Appe/WfP+TS1VVVbLZbJc8+iJJeXl5ys3N9S27XC4lJSUpMzNTNpst0N1sVTwej5xOp0aMGCGLxRLqdlo1Zh0cDXOeszdS7vqIS9YdyA+v6+b65Df9YYRw6pnXc3AwZ38NZ1CaEnCA+b76+nq53W4NHDhQFotFmzdv1ujRoyVJ5eXlqqiokMPhkCQ5HA4tXLhQ1dXViouLk3Q+cdpsNqWlpflqvv8vEKfT6XuMS7FarbJarRest1gsvCD+jlkED7MODnd9hNx1lw4w4fZn0FivDcKtZ4nXc7Aw5/MudwYBBZi8vDyNHDlS3bp10zfffKPCwkJt3bpVRUVFio2N1aRJk5Sbm6vOnTvLZrPpgQcekMPh0JAhQyRJmZmZSktL0/jx41VQUKDKykrNnj1bOTk5vvAxZcoUPfvss3rooYd07733asuWLVqzZo3Wr18f4AgAAEBrFVCAqa6u1oQJE3T8+HHFxsaqb9++Kioq0ogRIyRJS5cuVWRkpEaPHi23262srCw999xzvvtHRUVp3bp1mjp1qhwOh9q3b6+JEydq/vz5vpqUlBStX79eM2bM0LJly9S1a1e98MILfIQaAAD4BBRgXnzxxUa3t23bVsuXL9fy5csvWZOcnNzkRWrDhg3Tvn37AmkNAABcRfgtJAAAYBwCDAAAMA4BBgAAGIcAAwAAjPODvwcGgPmue7jpryk4+nh2EDoBgMvDERgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjBNQgFm0aJF+9rOfqWPHjoqLi9Mdd9yh8vJyv5phw4YpIiLC7zZlyhS/moqKCmVnZysmJkZxcXGaOXOmzp0751ezdetWXX/99bJarerRo4dWrVp1ZXsIAABanYACzLZt25STk6Ndu3bJ6XTK4/EoMzNTp0+f9qu7//77dfz4cd+toKDAt62urk7Z2dk6e/asdu7cqVdeeUWrVq3S3LlzfTVHjhxRdna2br75ZpWVlWn69Om67777VFRU9AN3FwAAtAZtAineuHGj3/KqVasUFxen0tJSDR061Lc+JiZGdrv9oo9RXFysQ4cOadOmTYqPj1f//v21YMECzZo1S/n5+YqOjtbKlSuVkpKiJUuWSJJSU1O1Y8cOLV26VFlZWYHuIwAAaGV+0DUwNTU1kqTOnTv7rV+9erWuvfZa9enTR3l5eaqtrfVtKykpUXp6uuLj433rsrKy5HK5dPDgQV9NRkaG32NmZWWppKTkh7QLAABaiYCOwHxXfX29pk+frhtvvFF9+vTxrR87dqySk5OVmJio/fv3a9asWSovL9cbb7whSaqsrPQLL5J8y5WVlY3WuFwuffvtt2rXrt0F/bjdbrndbt+yy+WSJHk8Hnk8nivdzVahYf+v9jkEg6mztkZ5m6wJp31q6MUa2Xjf4dSzZO6cw6mn1og5+7vcOVxxgMnJydGBAwe0Y8cOv/WTJ0/2/Xd6eroSEhI0fPhwHT58WN27d7/Sp2vSokWLNG/evAvWFxcXKyYmpsWe1yROpzPULVw1TJt1wQ1N12zYsKHlGwnQgkH1jW4Pt55NnbNpr2dTMefzvnvWpjFXFGCmTZumdevWafv27eratWujtYMHD5Ykffrpp+revbvsdrv27NnjV1NVVSVJvutm7Ha7b913a2w220WPvkhSXl6ecnNzfcsul0tJSUnKzMyUzWYLbAdbGY/HI6fTqREjRshisYS6nVbN1Fn3yW/6AvkD+eFz/VnDnOfsjZS7PuKSdeHUs2TunE17PZuGOftrOIPSlIACjNfr1QMPPKC1a9dq69atSklJafI+ZWVlkqSEhARJksPh0MKFC1VdXa24uDhJ51OnzWZTWlqar+b7/wpxOp1yOByXfB6r1Sqr1XrBeovFwgvi75hF8Jg2a3fdpUNAg3DcH3d9RKO9h1vPps7ZtNezqZjzeZc7g4Au4s3JydF///d/q7CwUB07dlRlZaUqKyv17bffSpIOHz6sBQsWqLS0VEePHtVbb72lCRMmaOjQoerbt68kKTMzU2lpaRo/frz+8pe/qKioSLNnz1ZOTo4vgEyZMkV/+9vf9NBDD+l///d/9dxzz2nNmjWaMWNGIO0CAIBWKqAAs2LFCtXU1GjYsGFKSEjw3V577TVJUnR0tDZt2qTMzEz16tVLDz74oEaPHq23337b9xhRUVFat26doqKi5HA4dPfdd2vChAmaP3++ryYlJUXr16+X0+lUv379tGTJEr3wwgt8hBoAAEi6glNIjUlKStK2bduafJzk5OQmL1QbNmyY9u3bF0h7AADgKsFvIQEAAOMQYAAAgHEIMAAAwDhX/EV2AICrQ5/8okY/An708ewgdgOcxxEYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADG4VNIQAvgUxsA0LI4AgMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAME5AAWbRokX62c9+po4dOyouLk533HGHysvL/WrOnDmjnJwcdenSRR06dNDo0aNVVVXlV1NRUaHs7GzFxMQoLi5OM2fO1Llz5/xqtm7dquuvv15Wq1U9evTQqlWrrmwPAQBAqxNQgNm2bZtycnK0a9cuOZ1OeTweZWZm6vTp076aGTNm6O2339brr7+ubdu26dixY7rzzjt92+vq6pSdna2zZ89q586deuWVV7Rq1SrNnTvXV3PkyBFlZ2fr5ptvVllZmaZPn6777rtPRUVFzbDLAADAdG0CKd64caPf8qpVqxQXF6fS0lINHTpUNTU1evHFF1VYWKhbbrlFkvTyyy8rNTVVu3bt0pAhQ1RcXKxDhw5p06ZNio+PV//+/bVgwQLNmjVL+fn5io6O1sqVK5WSkqIlS5ZIklJTU7Vjxw4tXbpUWVlZzbTrAADAVAEFmO+rqamRJHXu3FmSVFpaKo/Ho4yMDF9Nr1691K1bN5WUlGjIkCEqKSlRenq64uPjfTVZWVmaOnWqDh48qAEDBqikpMTvMRpqpk+ffsle3G633G63b9nlckmSPB6PPB7PD9lN4zXs/9U+h2BomLE10ntZdeHCGtV4v1J49cycg8PUOZuG92h/lzuHKw4w9fX1mj59um688Ub16dNHklRZWano6Gh16tTJrzY+Pl6VlZW+mu+Gl4btDdsaq3G5XPr222/Vrl27C/pZtGiR5s2bd8H64uJixcTEXNlOtjJOpzPULVw1Fgyqb3T7hg0bgtTJ5Sm4oemacOtZYs7BYtqcTcV79Hm1tbWXVXfFASYnJ0cHDhzQjh07rvQhmlVeXp5yc3N9yy6XS0lJScrMzJTNZgthZ6Hn8XjkdDo1YsQIWSyWULfTqjXMes7eSLnrIy5ZdyA/vE6F9slv+vqycOqZOQeHqXM2De/R/hrOoDTligLMtGnTtG7dOm3fvl1du3b1rbfb7Tp79qxOnjzpdxSmqqpKdrvdV7Nnzx6/x2v4lNJ3a77/yaWqqirZbLaLHn2RJKvVKqvVesF6i8XCC+LvmEXwuOsj5K679Bt+uP05NNZrg3DrWWLOwWLanE3Fe/R5lzuDgD6F5PV6NW3aNK1du1ZbtmxRSkqK3/aBAwfKYrFo8+bNvnXl5eWqqKiQw+GQJDkcDn300Ueqrq721TidTtlsNqWlpflqvvsYDTUNjwEAAK5uAR2BycnJUWFhof7nf/5HHTt29F2zEhsbq3bt2ik2NlaTJk1Sbm6uOnfuLJvNpgceeEAOh0NDhgyRJGVmZiotLU3jx49XQUGBKisrNXv2bOXk5PiOoEyZMkXPPvusHnroId17773asmWL1qxZo/Xr1zfz7gMAABMFdARmxYoVqqmp0bBhw5SQkOC7vfbaa76apUuX6p//+Z81evRoDR06VHa7XW+88YZve1RUlNatW6eoqCg5HA7dfffdmjBhgubPn++rSUlJ0fr16+V0OtWvXz8tWbJEL7zwAh+hBgAAkgI8AuP1Nv0RwLZt22r58uVavnz5JWuSk5ObvGp92LBh2rdvXyDtAQCAqwS/hQQAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOG1C3QAAAM3puofXN1lz9PHsIHSClkSAQVjjjQgAcDGcQgIAAMYhwAAAAOMEHGC2b9+uUaNGKTExUREREXrzzTf9tv/2t79VRESE3+3WW2/1qzlx4oTGjRsnm82mTp06adKkSTp16pRfzf79+/WLX/xCbdu2VVJSkgoKCgLfOwAA0CoFHGBOnz6tfv36afny5ZesufXWW3X8+HHf7U9/+pPf9nHjxungwYNyOp1at26dtm/frsmTJ/u2u1wuZWZmKjk5WaWlpXriiSeUn5+v559/PtB2AQBAKxTwRbwjR47UyJEjG62xWq2y2+0X3fbxxx9r48aN+uCDDzRo0CBJ0jPPPKPbbrtNf/jDH5SYmKjVq1fr7NmzeumllxQdHa3evXurrKxMTz75pF/QAQAAV6cW+RTS1q1bFRcXp2uuuUa33HKLHn30UXXp0kWSVFJSok6dOvnCiyRlZGQoMjJSu3fv1q9+9SuVlJRo6NChio6O9tVkZWVp8eLF+vrrr3XNNddc8Jxut1tut9u37HK5JEkej0cej6cldtMYDftv4hysUd4ma8Jpvxp6sUY23nc49Swx52BhzsFh6pzDqadQutw5RHi93qb/pC9154gIrV27VnfccYdv3auvvqqYmBilpKTo8OHD+v3vf68OHTqopKREUVFReuyxx/TKK6+ovLzc77Hi4uI0b948TZ06VZmZmUpJSdEf//hH3/ZDhw6pd+/eOnTokFJTUy/oJT8/X/PmzbtgfWFhoWJiYq50FwEAQBDV1tZq7Nixqqmpkc1mu2Rdsx+BGTNmjO+/09PT1bdvX3Xv3l1bt27V8OHDm/vpfPLy8pSbm+tbdrlcSkpKUmZmZqMDuBp4PB45nU6NGDFCFosl1O0EpE9+UZM1B/KzgtDJ5WmY9Zy9kXLXR1yyLpx6lphzsDDn4DB1zia+R7eEhjMoTWnxL7L78Y9/rGuvvVaffvqphg8fLrvdrurqar+ac+fO6cSJE77rZux2u6qqqvxqGpYvdW2N1WqV1Wq9YL3FYuEF8XcmzsJdd+k3zQbhuE/u+ohGew+3nplzcDDn4DB1zia+R7eEy51Bi38PzBdffKGvvvpKCQkJkiSHw6GTJ0+qtLTUV7NlyxbV19dr8ODBvprt27f7nQdzOp3q2bPnRa9/AQAAV5eAA8ypU6dUVlamsrIySdKRI0dUVlamiooKnTp1SjNnztSuXbt09OhRbd68Wbfffrt69OihrKzzh+tSU1N166236v7779eePXv0/vvva9q0aRozZowSExMlSWPHjlV0dLQmTZqkgwcP6rXXXtOyZcv8ThEBAICrV8ABZu/evRowYIAGDBggScrNzdWAAQM0d+5cRUVFaf/+/frlL3+pn/70p5o0aZIGDhyo9957z+/0zurVq9WrVy8NHz5ct912m2666Sa/73iJjY1VcXGxjhw5ooEDB+rBBx/U3Llz+Qg1AACQdAXXwAwbNkyNfXCpqKjpi6c6d+6swsLCRmv69u2r9957L9D2AADAVYDfQgIAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGCTjAbN++XaNGjVJiYqIiIiL05ptv+m33er2aO3euEhIS1K5dO2VkZOiTTz7xqzlx4oTGjRsnm82mTp06adKkSTp16pRfzf79+/WLX/xCbdu2VVJSkgoKCgLfOwAA0CoFHGBOnz6tfv36afny5RfdXlBQoKefflorV67U7t271b59e2VlZenMmTO+mnHjxungwYNyOp1at26dtm/frsmTJ/u2u1wuZWZmKjk5WaWlpXriiSeUn5+v559//gp2EQAAtDZtAr3DyJEjNXLkyItu83q9euqppzR79mzdfvvtkqT/+q//Unx8vN58802NGTNGH3/8sTZu3KgPPvhAgwYNkiQ988wzuu222/SHP/xBiYmJWr16tc6ePauXXnpJ0dHR6t27t8rKyvTkk0/6BR0AAHB1CjjANObIkSOqrKxURkaGb11sbKwGDx6skpISjRkzRiUlJerUqZMvvEhSRkaGIiMjtXv3bv3qV79SSUmJhg4dqujoaF9NVlaWFi9erK+//lrXXHPNBc/tdrvldrt9yy6XS5Lk8Xjk8XiaczeN07D/Js7BGuVtsiac9quhF2tk432HU88Scw4W5hwcps45nHoKpcudQ7MGmMrKSklSfHy83/r4+HjftsrKSsXFxfk30aaNOnfu7FeTkpJywWM0bLtYgFm0aJHmzZt3wfri4mLFxMRc4R61Lk6nM9QtBKzghqZrNmzY0PKNBGjBoPpGt4dbz8w5OJhzcJg6ZxPfo1tCbW3tZdU1a4AJpby8POXm5vqWXS6XkpKSlJmZKZvNFsLOQs/j8cjpdGrEiBGyWCyhbicgffKLmqw5kJ8VhE4uT8Os5+yNlLs+4pJ14dSzxJyDhTkHh6lzNvE9uiU0nEFpSrMGGLvdLkmqqqpSQkKCb31VVZX69+/vq6murva737lz53TixAnf/e12u6qqqvxqGpYbar7ParXKarVesN5isfCC+DsTZ+Guu/SbZoNw3Cd3fUSjvYdbz8w5OJhzcJg6ZxPfo1vC5c6gWb8HJiUlRXa7XZs3b/atc7lc2r17txwOhyTJ4XDo5MmTKi0t9dVs2bJF9fX1Gjx4sK9m+/btfufBnE6nevbsedHTRwAA4OoScIA5deqUysrKVFZWJun8hbtlZWWqqKhQRESEpk+frkcffVRvvfWWPvroI02YMEGJiYm64447JEmpqam69dZbdf/992vPnj16//33NW3aNI0ZM0aJiYmSpLFjxyo6OlqTJk3SwYMH9dprr2nZsmV+p4gAAMDVK+BTSHv37tXNN9/sW24IFRMnTtSqVav00EMP6fTp05o8ebJOnjypm266SRs3blTbtm1991m9erWmTZum4cOHKzIyUqNHj9bTTz/t2x4bG6vi4mLl5ORo4MCBuvbaazV37lw+Qg0AACRdQYAZNmyYvN5Lf0QtIiJC8+fP1/z58y9Z07lzZxUWFjb6PH379tV7770XaHsAABipT35Ro9fvHH08O4jdhD9+CwkAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHFazU8JoGlNXeEucZU7AMAMHIEBAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHGaPcDk5+crIiLC79arVy/f9jNnzignJ0ddunRRhw4dNHr0aFVVVfk9RkVFhbKzsxUTE6O4uDjNnDlT586da+5WAQCAodq0xIP27t1bmzZt+seTtPnH08yYMUPr16/X66+/rtjYWE2bNk133nmn3n//fUlSXV2dsrOzZbfbtXPnTh0/flwTJkyQxWLRY4891hLtAgAAw7RIgGnTpo3sdvsF62tqavTiiy+qsLBQt9xyiyTp5ZdfVmpqqnbt2qUhQ4aouLhYhw4d0qZNmxQfH6/+/ftrwYIFmjVrlvLz8xUdHd0SLQMAAIO0SID55JNPlJiYqLZt28rhcGjRokXq1q2bSktL5fF4lJGR4avt1auXunXrppKSEg0ZMkQlJSVKT09XfHy8ryYrK0tTp07VwYMHNWDAgIs+p9vtltvt9i27XC5JksfjkcfjaYndNEbD/lsjvZddGy6sUWb1fLmzDqeeJeYcLMw5OJjzD9cnv6jJmgP5WS3y3Je7nxFer7fpP+kAvPPOOzp16pR69uyp48ePa968efryyy914MABvf3227rnnnv8goYk3XDDDbr55pu1ePFiTZ48WZ999pmKiv4xvNraWrVv314bNmzQyJEjL/q8+fn5mjdv3gXrCwsLFRMT05y7CAAAWkhtba3Gjh2rmpoa2Wy2S9Y1+xGY7waMvn37avDgwUpOTtaaNWvUrl275n46n7y8POXm5vqWXS6XkpKSlJmZ2egArgYej0dOp1Nz9kbKXR/RaG1LJeorFcp/BVyJy511OPUsMedgYc7BwZx/uFDOsOEMSlNa5BTSd3Xq1Ek//elP9emnn2rEiBE6e/asTp48qU6dOvlqqqqqfNfM2O127dmzx+8xGj6ldLHrahpYrVZZrdYL1lssFlkslmbYE/O56yPkrms8wITbrJrqVwq/nqWmZx1uPTPn4GDOwcGcm6GXEM7wch+3xb8H5tSpUzp8+LASEhI0cOBAWSwWbd682be9vLxcFRUVcjgckiSHw6GPPvpI1dXVvhqn0ymbzaa0tLSWbhcAABig2Y/A/Md//IdGjRql5ORkHTt2TI888oiioqJ01113KTY2VpMmTVJubq46d+4sm82mBx54QA6HQ0OGDJEkZWZmKi0tTePHj1dBQYEqKys1e/Zs5eTkXPQICwAAuPo0e4D54osvdNddd+mrr77Sj370I910003atWuXfvSjH0mSli5dqsjISI0ePVput1tZWVl67rnnfPePiorSunXrNHXqVDkcDrVv314TJ07U/Pnzm7tVAABgqGYPMK+++mqj29u2bavly5dr+fLll6xJTk7Whg0bmrs1AADQSvBbSAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgnGb/KYGrwXUPr2+y5ujj2UHoBACAqxNHYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjBPWAWb58uW67rrr1LZtWw0ePFh79uwJdUsAACAMhG2Aee2115Sbm6tHHnlEH374ofr166esrCxVV1eHujUAABBiYRtgnnzySd1///265557lJaWppUrVyomJkYvvfRSqFsDAAAh1ibUDVzM2bNnVVpaqry8PN+6yMhIZWRkqKSk5KL3cbvdcrvdvuWamhpJ0okTJ+TxeJq1vzbnTjdZ89VXXzXrc/4QHo9HtbW1auOJVF19RKO14dS31HpnHU49S8w5WJhzcDDnHy6UM/zmm28kSV6vt/FCbxj68ssvvZK8O3fu9Fs/c+ZM7w033HDR+zzyyCNeSdy4cePGjRu3VnD7/PPPG80KYXkE5krk5eUpNzfXt1xfX68TJ06oS5cuioho/KhDa+dyuZSUlKTPP/9cNpst1O20asw6OJhzcDDn4GDO/rxer7755hslJiY2WheWAebaa69VVFSUqqqq/NZXVVXJbrdf9D5Wq1VWq9VvXadOnVqqRSPZbDb+cgQJsw4O5hwczDk4mPM/xMbGNlkTlhfxRkdHa+DAgdq8ebNvXX19vTZv3iyHwxHCzgAAQDgIyyMwkpSbm6uJEydq0KBBuuGGG/TUU0/p9OnTuueee0LdGgAACLGwDTC/+c1v9H//93+aO3euKisr1b9/f23cuFHx8fGhbs04VqtVjzzyyAWn2ND8mHVwMOfgYM7BwZyvTITX29TnlAAAAMJLWF4DAwAA0BgCDAAAMA4BBgAAGIcAAwAAjEOAaeW+/PJL3X333erSpYvatWun9PR07d27N9RttSp1dXWaM2eOUlJS1K5dO3Xv3l0LFixo+nc80KTt27dr1KhRSkxMVEREhN58802/7V6vV3PnzlVCQoLatWunjIwMffLJJ6Fp1mCNzdnj8WjWrFlKT09X+/btlZiYqAkTJujYsWOha9hQTb2ev2vKlCmKiIjQU089FbT+TEOAacW+/vpr3XjjjbJYLHrnnXd06NAhLVmyRNdcc02oW2tVFi9erBUrVujZZ5/Vxx9/rMWLF6ugoEDPPPNMqFsz3unTp9WvXz8tX778otsLCgr09NNPa+XKldq9e7fat2+vrKwsnTlzJsidmq2xOdfW1urDDz/UnDlz9OGHH+qNN95QeXm5fvnLX4agU7M19XpusHbtWu3atavJr9K/6jXHjy8iPM2aNct70003hbqNVi87O9t77733+q278847vePGjQtRR62TJO/atWt9y/X19V673e594oknfOtOnjzptVqt3j/96U8h6LB1+P6cL2bPnj1eSd7PPvssOE21Qpea8xdffOH9p3/6J++BAwe8ycnJ3qVLlwa9N1NwBKYVe+uttzRo0CD967/+q+Li4jRgwAD953/+Z6jbanV+/vOfa/PmzfrrX/8qSfrLX/6iHTt2aOTIkSHurHU7cuSIKisrlZGR4VsXGxurwYMHq6SkJISdtX41NTWKiIjg9+aaWX19vcaPH6+ZM2eqd+/eoW4n7IXtN/Hih/vb3/6mFStWKDc3V7///e/1wQcf6He/+52io6M1ceLEULfXajz88MNyuVzq1auXoqKiVFdXp4ULF2rcuHGhbq1Vq6yslKQLvp07Pj7etw3N78yZM5o1a5buuusufniwmS1evFht2rTR7373u1C3YgQCTCtWX1+vQYMG6bHHHpMkDRgwQAcOHNDKlSsJMM1ozZo1Wr16tQoLC9W7d2+VlZVp+vTpSkxMZM5oVTwej37961/L6/VqxYoVoW6nVSktLdWyZcv04YcfKiIiItTtGIFTSK1YQkKC0tLS/NalpqaqoqIiRB21TjNnztTDDz+sMWPGKD09XePHj9eMGTO0aNGiULfWqtntdklSVVWV3/qqqirfNjSfhvDy2Wefyel0cvSlmb333nuqrq5Wt27d1KZNG7Vp00afffaZHnzwQV133XWhbi8sEWBasRtvvFHl5eV+6/76178qOTk5RB21TrW1tYqM9P+rFBUVpfr6+hB1dHVISUmR3W7X5s2bfetcLpd2794th8MRws5an4bw8sknn2jTpk3q0qVLqFtqdcaPH6/9+/errKzMd0tMTNTMmTNVVFQU6vbCEqeQWrEZM2bo5z//uR577DH9+te/1p49e/T888/r+eefD3VrrcqoUaO0cOFCdevWTb1799a+ffv05JNP6t577w11a8Y7deqUPv30U9/ykSNHVFZWps6dO6tbt26aPn26Hn30Uf3kJz9RSkqK5syZo8TERN1xxx2ha9pAjc05ISFB//Iv/6IPP/xQ69atU11dne8ao86dOys6OjpUbRunqdfz94OhxWKR3W5Xz549g92qGUL9MSi0rLffftvbp08fr9Vq9fbq1cv7/PPPh7qlVsflcnn//d//3dutWzdv27ZtvT/+8Y+9/+///T+v2+0OdWvGe/fdd72SLrhNnDjR6/We/yj1nDlzvPHx8V6r1eodPny4t7y8PLRNG6ixOR85cuSi2yR533333VC3bpSmXs/fx8eoGxfh9fJ1oQAAwCxcAwMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcf4/wi3qdF5RKHcAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "pd.Series(len(x[1]) for x in sample_data).hist(bins=50);" ] }, { "cell_type": "code", "execution_count": 33, "id": "8cdd3fd1", "metadata": { "ExecuteTime": { "end_time": "2023-10-19T19:50:25.284171Z", "start_time": "2023-10-19T19:50:25.088706Z" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGdCAYAAADwjmIIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAur0lEQVR4nO3de3BUZZ7/8U8Skg5RmnAxCVkCRFEucjVI6FFZlJCAKVeUsgQpFxWhYJMpIQ5KXIwBZisOjigqSlmOxq0FRaZGXIGBtEFApAGJZBEUSlhYxpUODgjhIkmTnN8fbs6PNpDkxCaXJ+9XVQr6nG8//ZxvnQ4fzqU7zLIsSwAAAIYJb+4JAAAAXA2EHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkdo19wSaU3V1tb7//nt16NBBYWFhzT0dAADQAJZl6cyZM0pMTFR4+JWP17TpkPP9998rKSmpuacBAAAa4W9/+5u6d+9+xfVtOuR06NBB0s9NcrvdIRkzEAioqKhI6enpioyMDMmYpqNnztAv5+iZc/TMOXrmXGN7Vl5erqSkJPvf8Stp0yGn5hSV2+0OaciJiYmR2+1mJ28geuYM/XKOnjlHz5yjZ8792p7Vd6kJFx4DAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGKldc08ALUevuWvrrTnyfGYTzAQAgF+PIzkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCS+1qGNaMhXNgAAYBKO5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjMTn5MCRhnzezpHnM5tgJgAA1I0jOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwkqOQU1BQoFtvvVUdOnRQXFycxo8frwMHDgTVjBo1SmFhYUE/M2bMCKo5evSoMjMzFRMTo7i4OM2ZM0cXL14Mqtm0aZNuueUWuVwu9e7dW4WFhbXms3TpUvXq1UvR0dFKTU3Vzp07nWwOAAAwmKOQs3nzZmVlZWn79u3yer0KBAJKT0/XuXPnguqmTZumY8eO2T+LFi2y11VVVSkzM1OVlZXatm2b3n33XRUWFiovL8+uOXz4sDIzM3XnnXeqtLRUs2bN0uOPP64NGzbYNStXrlROTo6ee+45ffnllxo8eLAyMjJ0/PjxxvYCAAAYxNG3kK9fvz7ocWFhoeLi4lRSUqKRI0fay2NiYpSQkHDZMYqKivT111/rk08+UXx8vIYMGaKFCxfq6aefVn5+vqKiorRs2TIlJyfrxRdflCT169dPW7du1UsvvaSMjAxJ0uLFizVt2jQ9+uijkqRly5Zp7dq1evvttzV37lwnmwUAAAzkKOT80unTpyVJnTt3Dlq+fPly/cd//IcSEhJ0zz336Nlnn1VMTIwkyefzaeDAgYqPj7frMzIyNHPmTO3bt09Dhw6Vz+dTWlpa0JgZGRmaNWuWJKmyslIlJSXKzc2114eHhystLU0+n++K862oqFBFRYX9uLy8XJIUCAQUCAQa0YHaasYJ1Xih4oqwmuy1nG57S+1ZS0W/nKNnztEz5+iZc43tWUPrGx1yqqurNWvWLN12220aMGCAvfyhhx5Sz549lZiYqD179ujpp5/WgQMH9Je//EWS5Pf7gwKOJPux3++vs6a8vFw//fSTfvzxR1VVVV22Zv/+/Vecc0FBgebPn19reVFRkR3CQsXr9YZ0vF9r0fCme61169Y16nktrWctHf1yjp45R8+co2fOOe3Z+fPnG1TX6JCTlZWlvXv3auvWrUHLp0+fbv994MCB6tatm0aPHq1Dhw7phhtuaOzLhURubq5ycnLsx+Xl5UpKSlJ6errcbndIXiMQCMjr9WrMmDGKjIwMyZihMCB/Q/1FIbI3P8NRfUvtWUtFv5yjZ87RM+fomXON7VnNmZj6NCrkZGdna82aNdqyZYu6d+9eZ21qaqok6eDBg7rhhhuUkJBQ6y6osrIySbKv40lISLCXXVrjdrvVvn17RUREKCIi4rI1V7oWSJJcLpdcLlet5ZGRkSHfIa/GmL9GRVVYk71WY7e7pfWspaNfztEz5+iZc/TMOac9a2ito7urLMtSdna2PvzwQ23cuFHJycn1Pqe0tFSS1K1bN0mSx+PRV199FXQXlNfrldvtVv/+/e2a4uLioHG8Xq88Ho8kKSoqSikpKUE11dXVKi4utmsAAEDb5uhITlZWllasWKGPPvpIHTp0sK+h6dixo9q3b69Dhw5pxYoVuvvuu9WlSxft2bNHs2fP1siRIzVo0CBJUnp6uvr376+HH35YixYtkt/v17x585SVlWUfZZkxY4Zee+01PfXUU3rssce0ceNGffDBB1q7dq09l5ycHE2ZMkXDhg3T8OHD9fLLL+vcuXP23VYAAKBtcxRy3njjDUk/f+Dfpd555x098sgjioqK0ieffGIHjqSkJE2YMEHz5s2zayMiIrRmzRrNnDlTHo9H11xzjaZMmaIFCxbYNcnJyVq7dq1mz56tJUuWqHv37nrrrbfs28cl6cEHH9QPP/ygvLw8+f1+DRkyROvXr691MTIAAGibHIUcy6r7NuSkpCRt3ry53nF69uxZ7x04o0aN0u7du+usyc7OVnZ2dr2vBwAA2h6+uwoAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACM1K65J4Bfr9fctc09BQAAWhyO5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGKldc08AbVOvuWvtv7siLC0aLg3I36CKqjB7+ZHnM5tjagAAQ3AkBwAAGImQAwAAjOQo5BQUFOjWW29Vhw4dFBcXp/Hjx+vAgQNBNRcuXFBWVpa6dOmia6+9VhMmTFBZWVlQzdGjR5WZmamYmBjFxcVpzpw5unjxYlDNpk2bdMstt8jlcql3794qLCysNZ+lS5eqV69eio6OVmpqqnbu3OlkcwAAgMEchZzNmzcrKytL27dvl9frVSAQUHp6us6dO2fXzJ49Wx9//LFWrVqlzZs36/vvv9f9999vr6+qqlJmZqYqKyu1bds2vfvuuyosLFReXp5dc/jwYWVmZurOO+9UaWmpZs2apccff1wbNmywa1auXKmcnBw999xz+vLLLzV48GBlZGTo+PHjv6YfAADAEI4uPF6/fn3Q48LCQsXFxamkpEQjR47U6dOn9ac//UkrVqzQXXfdJUl655131K9fP23fvl0jRoxQUVGRvv76a33yySeKj4/XkCFDtHDhQj399NPKz89XVFSUli1bpuTkZL344ouSpH79+mnr1q166aWXlJGRIUlavHixpk2bpkcffVSStGzZMq1du1Zvv/225s6d+6sbAwAAWrdfdXfV6dOnJUmdO3eWJJWUlCgQCCgtLc2u6du3r3r06CGfz6cRI0bI5/Np4MCBio+Pt2syMjI0c+ZM7du3T0OHDpXP5wsao6Zm1qxZkqTKykqVlJQoNzfXXh8eHq60tDT5fL4rzreiokIVFRX24/LycklSIBBQIBBoZBeC1YwTqvEawhVhNdlrNURDtv3SObvCraA/nYzTFjXHPtba0TPn6Jlz9My5xvasofWNDjnV1dWaNWuWbrvtNg0YMECS5Pf7FRUVpdjY2KDa+Ph4+f1+u+bSgFOzvmZdXTXl5eX66aef9OOPP6qqquqyNfv377/inAsKCjR//vxay4uKihQTE9OArW44r9cb0vHqsmh4k71Ug6xbt67emsvNeeGwasfjtGVNuY+Zgp45R8+co2fOOe3Z+fPnG1TX6JCTlZWlvXv3auvWrY0dosnl5uYqJyfHflxeXq6kpCSlp6fL7XaH5DUCgYC8Xq/GjBmjyMjIkIxZnwH5G+ovakJ78zPqrbl0zq5wSwuHVevZXeGqqP7/n5PTkHHaoubYx1o7euYcPXOOnjnX2J7VnImpT6NCTnZ2ttasWaMtW7aoe/fu9vKEhARVVlbq1KlTQUdzysrKlJCQYNf88i6omruvLq355R1ZZWVlcrvdat++vSIiIhQREXHZmpoxLsflcsnlctVaHhkZGfId8mqMeSWXfoBeS9CQ7b7cnCuqw4KW80uibk25j5mCnjlHz5yjZ8457VlDax2FHMuy9Nvf/lYffvihNm3apOTk5KD1KSkpioyMVHFxsSZMmCBJOnDggI4ePSqPxyNJ8ng8+rd/+zcdP35ccXFxkn4+TOV2u9W/f3+75penKrxerz1GVFSUUlJSVFxcrPHjx0v6+fRZcXGxsrOznWwSroJLP80YAIDm4ijkZGVlacWKFfroo4/UoUMH+xqajh07qn379urYsaOmTp2qnJwcde7cWW63W7/97W/l8Xg0YsQISVJ6err69++vhx9+WIsWLZLf79e8efOUlZVlH2WZMWOGXnvtNT311FN67LHHtHHjRn3wwQdau/b//+OZk5OjKVOmaNiwYRo+fLhefvllnTt3zr7bCgAAtG2OQs4bb7whSRo1alTQ8nfeeUePPPKIJOmll15SeHi4JkyYoIqKCmVkZOj111+3ayMiIrRmzRrNnDlTHo9H11xzjaZMmaIFCxbYNcnJyVq7dq1mz56tJUuWqHv37nrrrbfs28cl6cEHH9QPP/ygvLw8+f1+DRkyROvXr691MTIAAGibHJ+uqk90dLSWLl2qpUuXXrGmZ8+e9d45M2rUKO3evbvOmuzsbE5PAQCAy+K7qwAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACM1K65J4C69Zq7trmnAABAq8SRHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABG4lvI0WI15BvYjzyf2QQzAQC0RhzJAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjOQ45GzZskX33HOPEhMTFRYWptWrVwetf+SRRxQWFhb0M3bs2KCakydPavLkyXK73YqNjdXUqVN19uzZoJo9e/bojjvuUHR0tJKSkrRo0aJac1m1apX69u2r6OhoDRw4UOvWrXO6OQAAwFCOQ865c+c0ePBgLV269Io1Y8eO1bFjx+yf9957L2j95MmTtW/fPnm9Xq1Zs0ZbtmzR9OnT7fXl5eVKT09Xz549VVJSohdeeEH5+fl688037Zpt27Zp0qRJmjp1qnbv3q3x48dr/Pjx2rt3r9NNAgAABmrn9Anjxo3TuHHj6qxxuVxKSEi47LpvvvlG69ev1xdffKFhw4ZJkl599VXdfffd+uMf/6jExEQtX75clZWVevvttxUVFaWbb75ZpaWlWrx4sR2GlixZorFjx2rOnDmSpIULF8rr9eq1117TsmXLnG4WAAAwjOOQ0xCbNm1SXFycOnXqpLvuuku///3v1aVLF0mSz+dTbGysHXAkKS0tTeHh4dqxY4fuu+8++Xw+jRw5UlFRUXZNRkaG/vCHP+jHH39Up06d5PP5lJOTE/S6GRkZtU6fXaqiokIVFRX24/LycklSIBBQIBAIxabb44RqPFeEFZJxWjJXuBX0pxOh6nNrEup9rC2gZ87RM+fomXON7VlD60MecsaOHav7779fycnJOnTokJ555hmNGzdOPp9PERER8vv9iouLC55Eu3bq3Lmz/H6/JMnv9ys5OTmoJj4+3l7XqVMn+f1+e9mlNTVjXE5BQYHmz59fa3lRUZFiYmIatb1X4vV6QzLOouEhGaZVWDis2vFz2vJ1WKHax9oSeuYcPXOOnjnntGfnz59vUF3IQ87EiRPtvw8cOFCDBg3SDTfcoE2bNmn06NGhfjlHcnNzg47+lJeXKykpSenp6XK73SF5jUAgIK/XqzFjxigyMvJXjzcgf0MIZtWyucItLRxWrWd3hauiOszRc/fmZ1ylWbVcod7H2gJ65hw9c46eOdfYntWcianPVTlddanrr79eXbt21cGDBzV69GglJCTo+PHjQTUXL17UyZMn7et4EhISVFZWFlRT87i+mitdCyT9fK2Qy+WqtTwyMjLkO2SoxqyocvaPfmtWUR3meHvb8i+Sq7Hfmo6eOUfPnKNnzjntWUNrr/rn5Hz33Xc6ceKEunXrJknyeDw6deqUSkpK7JqNGzequrpaqampds2WLVuCzrl5vV716dNHnTp1smuKi4uDXsvr9crj8VztTQIAAK2A45Bz9uxZlZaWqrS0VJJ0+PBhlZaW6ujRozp79qzmzJmj7du368iRIyouLta9996r3r17KyPj59MK/fr109ixYzVt2jTt3LlTn3/+ubKzszVx4kQlJiZKkh566CFFRUVp6tSp2rdvn1auXKklS5YEnWp64okntH79er344ovav3+/8vPztWvXLmVnZ4egLQAAoLVzHHJ27dqloUOHaujQoZKknJwcDR06VHl5eYqIiNCePXv0T//0T7rppps0depUpaSk6LPPPgs6TbR8+XL17dtXo0eP1t13363bb7896DNwOnbsqKKiIh0+fFgpKSl68sknlZeXF/RZOr/5zW+0YsUKvfnmmxo8eLD+/Oc/a/Xq1RowYMCv6QcAADCE42tyRo0aJcu68q2+GzbUf6Fs586dtWLFijprBg0apM8++6zOmgceeEAPPPBAva8Hc/Wau7bemiPPZzbBTAAALQ3fXQUAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIzn+7iqETkO+dwkAADQOR3IAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARmrX3BMArrZec9fWW3Pk+cwmmAkAoClxJAcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGatfcEwBagl5z19Zbc+T5zCaYCQAgVDiSAwAAjETIAQAARiLkAAAAIxFyAACAkRyHnC1btuiee+5RYmKiwsLCtHr16qD1lmUpLy9P3bp1U/v27ZWWlqZvv/02qObkyZOaPHmy3G63YmNjNXXqVJ09ezaoZs+ePbrjjjsUHR2tpKQkLVq0qNZcVq1apb59+yo6OloDBw7UunXrnG4OAAAwlOOQc+7cOQ0ePFhLly697PpFixbplVde0bJly7Rjxw5dc801ysjI0IULF+yayZMna9++ffJ6vVqzZo22bNmi6dOn2+vLy8uVnp6unj17qqSkRC+88ILy8/P15ptv2jXbtm3TpEmTNHXqVO3evVvjx4/X+PHjtXfvXqebBAAADOT4FvJx48Zp3Lhxl11nWZZefvllzZs3T/fee68k6d///d8VHx+v1atXa+LEifrmm2+0fv16ffHFFxo2bJgk6dVXX9Xdd9+tP/7xj0pMTNTy5ctVWVmpt99+W1FRUbr55ptVWlqqxYsX22FoyZIlGjt2rObMmSNJWrhwobxer1577TUtW7asUc0AAADmCOnn5Bw+fFh+v19paWn2so4dOyo1NVU+n08TJ06Uz+dTbGysHXAkKS0tTeHh4dqxY4fuu+8++Xw+jRw5UlFRUXZNRkaG/vCHP+jHH39Up06d5PP5lJOTE/T6GRkZtU6fXaqiokIVFRX24/LycklSIBBQIBD4tZtvj3Xpn3VxRVghec3WzhVuBf3ZUoVqH/m1nOxj+Bk9c46eOUfPnGtszxpaH9KQ4/f7JUnx8fFBy+Pj4+11fr9fcXFxwZNo106dO3cOqklOTq41Rs26Tp06ye/31/k6l1NQUKD58+fXWl5UVKSYmJiGbGKDeb3eemsWDQ/pS7Z6C4dVN/cU6tTSrvlqyD6GYPTMOXrmHD1zzmnPzp8/36C6NvWJx7m5uUFHf8rLy5WUlKT09HS53e6QvEYgEJDX69WYMWMUGRlZZ+2A/A0hec3WzhVuaeGwaj27K1wV1WHNPZ0r2puf0dxTkORsH8PP6Jlz9Mw5euZcY3tWcyamPiENOQkJCZKksrIydevWzV5eVlamIUOG2DXHjx8Pet7Fixd18uRJ+/kJCQkqKysLqql5XF9NzfrLcblccrlctZZHRkaGfIdsyJgVVS33H/TmUFEd1qJ70tJ+aV2N/dZ09Mw5euYcPXPOac8aWhvSkJOcnKyEhAQVFxfboaa8vFw7duzQzJkzJUkej0enTp1SSUmJUlJSJEkbN25UdXW1UlNT7Zp//dd/VSAQsDfE6/WqT58+6tSpk11TXFysWbNm2a/v9Xrl8XhCuUmAje+3AoDWxfEt5GfPnlVpaalKS0sl/XyxcWlpqY4ePaqwsDDNmjVLv//97/Wf//mf+uqrr/TP//zPSkxM1Pjx4yVJ/fr109ixYzVt2jTt3LlTn3/+ubKzszVx4kQlJiZKkh566CFFRUVp6tSp2rdvn1auXKklS5YEnWp64okntH79er344ovav3+/8vPztWvXLmVnZ//6rgAAgFbP8ZGcXbt26c4777Qf1wSPKVOmqLCwUE899ZTOnTun6dOn69SpU7r99tu1fv16RUdH289Zvny5srOzNXr0aIWHh2vChAl65ZVX7PUdO3ZUUVGRsrKylJKSoq5duyovLy/os3R+85vfaMWKFZo3b56eeeYZ3XjjjVq9erUGDBjQqEYAAACzOA45o0aNkmVd+VbfsLAwLViwQAsWLLhiTefOnbVixYo6X2fQoEH67LPP6qx54IEH9MADD9Q9YQAA0Cbx3VUAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEgh/YJOAPXjiz4BoGlwJAcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCQ+Jwdoger6LB1XhKVFw5twMgDQSnEkBwAAGImQAwAAjETIAQAARuKaHCCEGvK9VACApsGRHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYie+uAlqpAfkbVFEVVmfNkeczm2g2ANDycCQHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJH4FnLAYL3mrq23hm8qB2AqjuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADBSyENOfn6+wsLCgn769u1rr79w4YKysrLUpUsXXXvttZowYYLKysqCxjh69KgyMzMVExOjuLg4zZkzRxcvXgyq2bRpk2655Ra5XC717t1bhYWFod4UAADQil2VIzk333yzjh07Zv9s3brVXjd79mx9/PHHWrVqlTZv3qzvv/9e999/v72+qqpKmZmZqqys1LZt2/Tuu++qsLBQeXl5ds3hw4eVmZmpO++8U6WlpZo1a5Yef/xxbdiw4WpsDgAAaIWuyufktGvXTgkJCbWWnz59Wn/605+0YsUK3XXXXZKkd955R/369dP27ds1YsQIFRUV6euvv9Ynn3yi+Ph4DRkyRAsXLtTTTz+t/Px8RUVFadmyZUpOTtaLL74oSerXr5+2bt2ql156SRkZGVdjkwAAQCtzVULOt99+q8TEREVHR8vj8aigoEA9evRQSUmJAoGA0tLS7Nq+ffuqR48e8vl8GjFihHw+nwYOHKj4+Hi7JiMjQzNnztS+ffs0dOhQ+Xy+oDFqambNmlXnvCoqKlRRUWE/Li8vlyQFAgEFAoEQbLnscRoynivCCslrtnaucCvoT9Qt1P0K1b7fkjl5X+Jn9Mw5euZcY3vW0PqQh5zU1FQVFhaqT58+OnbsmObPn6877rhDe/fuld/vV1RUlGJjY4OeEx8fL7/fL0ny+/1BAadmfc26umrKy8v1008/qX379pedW0FBgebPn19reVFRkWJiYhq1vVfi9XrrrVk0PKQv2eotHFbd3FNoVULVr3Xr1oVknNagIe9LBKNnztEz55z27Pz58w2qC3nIGTdunP33QYMGKTU1VT179tQHH3xwxfDRVHJzc5WTk2M/Li8vV1JSktLT0+V2u0PyGoFAQF6vV2PGjFFkZGSdtQPyuYZI+vmIxMJh1Xp2V7gqqsOaezotXqj7tTff/FO8Tt6X+Bk9c46eOdfYntWcianPVf/uqtjYWN100006ePCgxowZo8rKSp06dSroaE5ZWZl9DU9CQoJ27twZNEbN3VeX1vzyjqyysjK53e46g5TL5ZLL5aq1PDIyMuQ7ZEPGrKjiH/RLVVSH0RMHQtWvtvTL+Gq8101Hz5yjZ8457VlDa6/65+ScPXtWhw4dUrdu3ZSSkqLIyEgVFxfb6w8cOKCjR4/K4/FIkjwej7766isdP37crvF6vXK73erfv79dc+kYNTU1YwAAAIQ85Pzud7/T5s2bdeTIEW3btk333XefIiIiNGnSJHXs2FFTp05VTk6OPv30U5WUlOjRRx+Vx+PRiBEjJEnp6enq37+/Hn74Yf3Xf/2XNmzYoHnz5ikrK8s+CjNjxgz993//t5566int379fr7/+uj744APNnj071JsDAABaqZCfrvruu+80adIknThxQtddd51uv/12bd++Xdddd50k6aWXXlJ4eLgmTJigiooKZWRk6PXXX7efHxERoTVr1mjmzJnyeDy65pprNGXKFC1YsMCuSU5O1tq1azV79mwtWbJE3bt311tvvdWibh8fkL+BUy8AADSjkIec999/v8710dHRWrp0qZYuXXrFmp49e9Z7x8eoUaO0e/fuRs0RAACYj++uAgAARiLkAAAAI131W8gBtGy95q5tUN2R5zOv8kwAILQ4kgMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwUrvmngCA1qHX3LX11hx5PrMJZgIADcORHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABG4lvIAYQM31QOoCXhSA4AADASIQcAABiJkAMAAIxEyAEAAEbiwmMATYqLkwE0FY7kAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiVvIAbQ43GYOIBQ4kgMAAIxEyAEAAEYi5AAAACNxTQ6AVonrdgDUhyM5AADASIQcAABgJE5XATBWzSktV4SlRcOlAfkbVFEVFlTDKS3AXK3+SM7SpUvVq1cvRUdHKzU1VTt37mzuKQEAgBagVR/JWblypXJycrRs2TKlpqbq5ZdfVkZGhg4cOKC4uLjmnh6AVoALmAFzteqQs3jxYk2bNk2PPvqoJGnZsmVau3at3n77bc2dO7eZZwfAFAQhoHVqtSGnsrJSJSUlys3NtZeFh4crLS1NPp/vss+pqKhQRUWF/fj06dOSpJMnTyoQCIRkXoFAQOfPn1e7QLiqqsPqfwLUrtrS+fPV9KyB6JdzTdGz3r/7ICTj7MgdHZJxfq2a32UnTpxQZGRkc0+nVaBnzjW2Z2fOnJEkWZZVZ12rDTl///vfVVVVpfj4+KDl8fHx2r9//2WfU1BQoPnz59danpycfFXmiIZ7qLkn0MrQL+daS8+6vtjcMwBajzNnzqhjx45XXN9qQ05j5ObmKicnx35cXV2tkydPqkuXLgoLC83/7srLy5WUlKS//e1vcrvdIRnTdPTMGfrlHD1zjp45R8+ca2zPLMvSmTNnlJiYWGddqw05Xbt2VUREhMrKyoKWl5WVKSEh4bLPcblccrlcQctiY2Ovyvzcbjc7uUP0zBn65Rw9c46eOUfPnGtMz+o6glOj1d5CHhUVpZSUFBUXF9vLqqurVVxcLI/H04wzAwAALUGrPZIjSTk5OZoyZYqGDRum4cOH6+WXX9a5c+fsu60AAEDb1apDzoMPPqgffvhBeXl58vv9GjJkiNavX1/rYuSm5HK59Nxzz9U6LYYro2fO0C/n6Jlz9Mw5eubc1e5ZmFXf/VcAAACtUKu9JgcAAKAuhBwAAGAkQg4AADASIQcAABiJkBNCS5cuVa9evRQdHa3U1FTt3LmzuafUYuTn5yssLCzop2/fvvb6CxcuKCsrS126dNG1116rCRMm1PqgR9Nt2bJF99xzjxITExUWFqbVq1cHrbcsS3l5eerWrZvat2+vtLQ0ffvtt0E1J0+e1OTJk+V2uxUbG6upU6fq7NmzTbgVTau+nj3yyCO19ruxY8cG1bSlnhUUFOjWW29Vhw4dFBcXp/Hjx+vAgQNBNQ15Lx49elSZmZmKiYlRXFyc5syZo4sXLzblpjSZhvRs1KhRtfazGTNmBNW0pZ698cYbGjRokP0Bfx6PR3/961/t9U25jxFyQmTlypXKycnRc889py+//FKDBw9WRkaGjh8/3txTazFuvvlmHTt2zP7ZunWrvW727Nn6+OOPtWrVKm3evFnff/+97r///macbdM7d+6cBg8erKVLl152/aJFi/TKK69o2bJl2rFjh6655hplZGTowoULds3kyZO1b98+eb1erVmzRlu2bNH06dObahOaXH09k6SxY8cG7Xfvvfde0Pq21LPNmzcrKytL27dvl9frVSAQUHp6us6dO2fX1PderKqqUmZmpiorK7Vt2za9++67KiwsVF5eXnNs0lXXkJ5J0rRp04L2s0WLFtnr2lrPunfvrueff14lJSXatWuX7rrrLt17773at2+fpCbexyyExPDhw62srCz7cVVVlZWYmGgVFBQ046xajueee84aPHjwZdedOnXKioyMtFatWmUv++abbyxJls/na6IZtiySrA8//NB+XF1dbSUkJFgvvPCCvezUqVOWy+Wy3nvvPcuyLOvrr7+2JFlffPGFXfPXv/7VCgsLs/73f/+3yebeXH7ZM8uyrClTplj33nvvFZ/T1nt2/PhxS5K1efNmy7Ia9l5ct26dFR4ebvn9frvmjTfesNxut1VRUdG0G9AMftkzy7Ksf/zHf7SeeOKJKz6nrffMsiyrU6dO1ltvvdXk+xhHckKgsrJSJSUlSktLs5eFh4crLS1NPp+vGWfWsnz77bdKTEzU9ddfr8mTJ+vo0aOSpJKSEgUCgaD+9e3bVz169KB//+fw4cPy+/1BPerYsaNSU1PtHvl8PsXGxmrYsGF2TVpamsLDw7Vjx44mn3NLsWnTJsXFxalPnz6aOXOmTpw4Ya9r6z07ffq0JKlz586SGvZe9Pl8GjhwYNCHrmZkZKi8vNz+n7rJftmzGsuXL1fXrl01YMAA5ebm6vz58/a6ttyzqqoqvf/++zp37pw8Hk+T72Ot+hOPW4q///3vqqqqqvVJy/Hx8dq/f38zzaplSU1NVWFhofr06aNjx45p/vz5uuOOO7R37175/X5FRUXV+rLU+Ph4+f3+5plwC1PTh8vtYzXr/H6/4uLigta3a9dOnTt3brN9HDt2rO6//34lJyfr0KFDeuaZZzRu3Dj5fD5FRES06Z5VV1dr1qxZuu222zRgwABJatB70e/3X3Y/rFlnssv1TJIeeugh9ezZU4mJidqzZ4+efvppHThwQH/5y18ktc2effXVV/J4PLpw4YKuvfZaffjhh+rfv79KS0ubdB8j5KBJjBs3zv77oEGDlJqaqp49e+qDDz5Q+/btm3FmMNnEiRPtvw8cOFCDBg3SDTfcoE2bNmn06NHNOLPml5WVpb179wZdG4e6Xalnl17DNXDgQHXr1k2jR4/WoUOHdMMNNzT1NFuEPn36qLS0VKdPn9af//xnTZkyRZs3b27yeXC6KgS6du2qiIiIWleHl5WVKSEhoZlm1bLFxsbqpptu0sGDB5WQkKDKykqdOnUqqIb+/X81fahrH0tISKh1ofvFixd18uRJ+vh/rr/+enXt2lUHDx6U1HZ7lp2drTVr1ujTTz9V9+7d7eUNeS8mJCRcdj+sWWeqK/XsclJTUyUpaD9raz2LiopS7969lZKSooKCAg0ePFhLlixp8n2MkBMCUVFRSklJUXFxsb2surpaxcXF8ng8zTizluvs2bM6dOiQunXrppSUFEVGRgb178CBAzp69Cj9+z/JyclKSEgI6lF5ebl27Nhh98jj8ejUqVMqKSmxazZu3Kjq6mr7l25b99133+nEiRPq1q2bpLbXM8uylJ2drQ8//FAbN25UcnJy0PqGvBc9Ho+++uqroHDo9XrldrvVv3//ptmQJlRfzy6ntLRUkoL2s7bUs8uprq5WRUVF0+9jobhqGpb1/vvvWy6XyyosLLS+/vpra/r06VZsbGzQ1eFt2ZNPPmlt2rTJOnz4sPX5559baWlpVteuXa3jx49blmVZM2bMsHr06GFt3LjR2rVrl+XxeCyPx9PMs25aZ86csXbv3m3t3r3bkmQtXrzY2r17t/U///M/lmVZ1vPPP2/FxsZaH330kbVnzx7r3nvvtZKTk62ffvrJHmPs2LHW0KFDrR07dlhbt261brzxRmvSpEnNtUlXXV09O3PmjPW73/3O8vl81uHDh61PPvnEuuWWW6wbb7zRunDhgj1GW+rZzJkzrY4dO1qbNm2yjh07Zv+cP3/erqnvvXjx4kVrwIABVnp6ulVaWmqtX7/euu6666zc3Nzm2KSrrr6eHTx40FqwYIG1a9cu6/Dhw9ZHH31kXX/99dbIkSPtMdpaz+bOnWtt3rzZOnz4sLVnzx5r7ty5VlhYmFVUVGRZVtPuY4ScEHr11VetHj16WFFRUdbw4cOt7du3N/eUWowHH3zQ6tatmxUVFWX9wz/8g/Xggw9aBw8etNf/9NNP1r/8y79YnTp1smJiYqz77rvPOnbsWDPOuOl9+umnlqRaP1OmTLEs6+fbyJ999lkrPj7ecrlc1ujRo60DBw4EjXHixAlr0qRJ1rXXXmu53W7r0Ucftc6cOdMMW9M06urZ+fPnrfT0dOu6666zIiMjrZ49e1rTpk2r9R+PttSzy/VKkvXOO+/YNQ15Lx45csQaN26c1b59e6tr167Wk08+aQUCgSbemqZRX8+OHj1qjRw50urcubPlcrms3r17W3PmzLFOnz4dNE5b6tljjz1m9ezZ04qKirKuu+46a/To0XbAsaym3cfCLMuynB37AQAAaPm4JgcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAI/0/SuSF5q0e+PsAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# max is prob around 1000\n", "pd.Series(n_tokens_list).hist(bins=50);" ] }, { "cell_type": "code", "execution_count": 34, "id": "f1611437", "metadata": { "ExecuteTime": { "end_time": "2023-10-19T19:50:31.488306Z", "start_time": "2023-10-19T19:50:31.486008Z" } }, "outputs": [ { "data": { "text/plain": [ "{'id': '577bba83-2cfa-46d8-b127-933c7469602b',\n", " 'lyrics': \"[Pre-Chorus]\\nDon't be the boy that spits on girls\\nEverybody hates that guy\\nThat'll be the reason why\\nThat'll be the reason why\\nKill off the character, what'd she say?\\nWhat the fuck would she know anyway?\\nWhy would I even buy a new dress, just to impress\\n\\n[Verse 1]\\nWell I hope one day it all comes out\\nBelieve in the power of word of mouth\\nI know everybody in this town\\nGood luck keeping that one down\\n[Chorus]\\nAnd her mother likes a drink\\nAnd my heart is always thinking of her\\nAnd in case I knew the queen\\nI could pull a few strings\\nIf I wanted someone dead\\nIf I wanted somone's head\\n\\n[Pre-Chorus]\\nDon't be the boy that spits on girls\\nEverybody hates that guy\\nThat'll be the reason why\\nThat'll be, that will be the reason why\\nKill off the character, what'd she say?\\nWhat the fuck would she know anyway?\\nWhy would I even buy a new dress, just to impress\\n\\n[Chorus]\\nAnd my mother likes a drink\\nAnd my father's always thinking of her\\nAnd in case I knew the queen\\nI could pull a few strings\\nIf I wanted someone dead\\nIf I wanted someone's head\\n\\n[Bridge]\\nCould not be bothered\\nI gave all your madness away\\nBillions of people see hell on this earth all day\\nYou know I can love you back\\nYou know I can love you back\\n\\n\\n[Outro]\\nIf you like your life, stay off the drugs\\nIf you like the drugs, share them with celebrities\\nIf you like your life, stay off the drugs\\nIf you like the drugs, share them with celebrities\\nIf you like your life, stay off the drugs\\nIf you like the drugs, share them with celebrities\\nIf you like your life, stay off the drugs\\nIf you like the drugs, share them with celebrities\\nIf you like your life, if you like your life\",\n", " 'duration_s': 212,\n", " 'lang': 'en',\n", " 'audio_filepath': 's3://suno-data/datasets/harvest/genius_hq/audio/hIP1Ip2HgKE.webm',\n", " 'original_id': 'hIP1Ip2HgKE',\n", " 'genius_slug': 'Amy-shark-spits-on-girls-lyrics'}" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "metas_map[k]" ] }, { "cell_type": "code", "execution_count": 35, "id": "71969894", "metadata": { "ExecuteTime": { "end_time": "2023-10-19T19:50:36.178115Z", "start_time": "2023-10-19T19:50:31.940356Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "577bba83-2cfa-46d8-b127-933c7469602b\n", "Amy-shark-spits-on-girls-lyrics\n", "----------\n", "[Pre-Chorus]\n", "Don't be the boy that spits on girls\n", "Everybody hates that guy\n", "That'll be the reason why\n", "That'll be the reason why\n", "Kill off the character, what'd she say?\n", "What the fuck would she know anyway?\n", "Why would I even buy a new dress, just to impress 6.3 24.74 77\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "\n", "[Verse 1]\n", "Well I hope one day it all comes out\n", "Believe in the power of word of mouth\n", "I know everybody in this town\n", "Good luck keeping that one down\n", "[Chorus]\n", "And her mother likes a drink 28.72 47.75 52\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "And my heart is always thinking of her\n", "And in case I knew the queen\n", "I could pull a few strings\n", "If I wanted someone dead\n", "If I wanted somone's head 47.75 62.96 38\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "\n", "[Pre-Chorus]\n", "Don't be the boy that spits on girls\n", "Everybody hates that guy\n", "That'll be the reason why\n", "That'll be, that will be the reason why\n", "Kill off the character, what'd she say?\n", "What the fuck would she know anyway?\n", "Why would I even buy a new dress, just to impress 82.57 101.26 81\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "\n", "[Chorus]\n", "And my mother likes a drink\n", "And my father's always thinking of her\n", "And in case I knew the queen\n", "I could pull a few strings\n", "If I wanted someone dead\n", "If I wanted someone's head 105.15 124.31 50\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "\n", "[Bridge]\n", "Could not be bothered\n", "I gave all your madness away\n", "Billions of people see hell on this earth all day\n", "You know I can love you back 143.93 163.09 38\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "You know I can love you back\n", "\n", "\n", "[Outro]\n", "If you like your life, stay off the drugs\n", "If you like the drugs, share them with celebrities\n", "If you like your life, stay off the drugs 163.09 179.96 46\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "If you like the drugs, share them with celebrities\n", "If you like your life, stay off the drugs\n", "If you like the drugs, share them with celebrities\n", "If you like your life, stay off the drugs\n", "If you like the drugs, share them with celebrities 179.96 199.23 57\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "\n", "If you like your life, if you like your life 199.23 204.34 11\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n" ] } ], "source": [ "meta = metas_map[k]\n", "audio = Audio.from_s3(meta[\"audio_filepath\"])\n", "print(f\"{meta['id']}\")\n", "print(meta[\"genius_slug\"])\n", "print(\"-\"*10)\n", "for s in v:\n", " print(s[\"text\"], s[\"start_s\"], s[\"end_s\"], len(tokenize(s[\"text\"], max_tokens=512*8)))\n", " audio.get_segment(s[\"start_s\"], s[\"end_s\"]).play()\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 49, "id": "75bdde26", "metadata": { "ExecuteTime": { "end_time": "2023-10-19T20:16:51.221411Z", "start_time": "2023-10-19T20:16:48.153377Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "0it [00:03, ?it/s]\n" ] } ], "source": [ "for i, (k, v) in tqdm.tqdm(enumerate(sample_data)):\n", " if i % 10 != 0:\n", " data_split = \"train\"\n", " else:\n", " if i % 20 == 0:\n", " data_split = \"dev\"\n", " else:\n", " data_split = \"test\"\n", " meta = metas_map[k]\n", " audio = Audio.from_s3(meta[\"audio_filepath\"])\n", " data_path = os.path.join(f\"/home/tony/Data/Lyrics/{data_split}\", f\"{i}\", \"0\")\n", " file_name = f\"{i}-0\"\n", " os.makedirs(data_path, exist_ok=True)\n", " text_lines = []\n", " for j, s in enumerate(v):\n", " # print(s[\"text\"], s[\"start_s\"], s[\"end_s\"], len(tokenize(s[\"text\"], max_tokens=512*8)))\n", " segment = audio.get_segment(s[\"start_s\"], s[\"end_s\"])\n", " audio_name = f\"00{str(j) if j > 10 else '0' + str(j)}\"\n", " segment.to_wav(os.path.join(data_path, f\"{file_name}-{audio_name}.wav\"))\n", " cleaned_text = \" \".join(re.findall(r\"([\\w\\']+)\", s['text'])).upper()\n", " text_lines.append(f\"{audio_name} {cleaned_text} \\n\")\n", " with open(os.path.join(data_path, f\"{file_name}.trans.txt\"), \"w\") as fp:\n", " fp.writelines(text_lines)\n", " break" ] }, { "cell_type": "code", "execution_count": 51, "id": "53bdbeb4", "metadata": { "ExecuteTime": { "end_time": "2023-10-19T20:19:15.094224Z", "start_time": "2023-10-19T20:19:01.241947Z" } }, "outputs": [], "source": [ "with open(\"hoot/metas_map.json\", \"w\") as fp:\n", " json.dump(metas_map, fp)" ] }, { "cell_type": "code", "execution_count": 52, "id": "bc0643f0", "metadata": { "ExecuteTime": { "end_time": "2023-10-19T20:19:19.244321Z", "start_time": "2023-10-19T20:19:18.001682Z" } }, "outputs": [], "source": [ "with open(\"hoot/sample_data.json\", \"w\") as fp:\n", " json.dump(sample_data, fp)" ] }, { "cell_type": "code", "execution_count": null, "id": "e3b77911", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }