{ "cells": [ { "cell_type": "code", "execution_count": 14, "id": "bf70f34f", "metadata": {}, "outputs": [], "source": [ "import os\n", "import re\n", "import json\n", "import tqdm\n", "import uuid\n", "import funcy\n", "import numpy as np\n", "from suno_utils.audio import Audio, Token, Tokens\n", "from suno_utils.audio.conversion import get_audio_properties, play_audio\n", "from suno_utils.utils.numbers import safe_round\n", "from suno_utils.utils.text import normalize_whitespace\n", "from suno_utils.web.harvest import get_file_ext, get_filename\n", "\n", "BASE_DIR = \"/mnt/data-ssd-1/data/private/customer/speechly/2022-09-23_gymclass_40h\"\n", "RAW_DATA_DIR = os.path.join(BASE_DIR, \"raw_data\")\n", "RAW_AUDIO_DIR = os.path.join(RAW_DATA_DIR, \"audio\")\n", "\n", "AUDIO_DIR = os.path.join(BASE_DIR, \"audio\")\n", "TO_REV_DIR = os.path.join(BASE_DIR, \"to_rev\")\n", "TO_REV_AUDIO_DIR = os.path.join(TO_REV_DIR, \"audio\")\n", "FROM_REV_DIR = os.path.join(BASE_DIR, \"from_rev\")\n", "\n", "SEGMENT_DATE_DIR = \"2022_09_23\"\n", "SEGMENTS_DIR = os.path.join(BASE_DIR, \"pipeline\", SEGMENT_DATE_DIR, \"segments\")\n", "ARTIFACTS_DIR = os.path.join(BASE_DIR, \"pipeline\", SEGMENT_DATE_DIR, \"artifacts\")" ] }, { "cell_type": "code", "execution_count": 9, "id": "c769f21a", "metadata": {}, "outputs": [], "source": [ "os.makedirs(AUDIO_DIR, exist_ok=True)\n", "os.makedirs(TO_REV_DIR, exist_ok=True)\n", "os.makedirs(TO_REV_AUDIO_DIR, exist_ok=True)\n", "os.makedirs(FROM_REV_DIR, exist_ok=True)\n", "os.makedirs(SEGMENTS_DIR, exist_ok=True)\n", "os.makedirs(ARTIFACTS_DIR, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": 10, "id": "c3cd79ed", "metadata": {}, "outputs": [], "source": [ "GLOBAL_HOTWORDS = [\n", " 'beta lobby',\n", " 'beta physics',\n", " 'boom discord ball',\n", " 'bozo',\n", " 'carpet muncher',\n", " 'coke butter cheese',\n", " 'comp rules',\n", " 'dadass',\n", " 'deadass',\n", " 'dribble',\n", " 'dunk competition',\n", " 'dunk dink dunk',\n", " 'dunk on dunk',\n", " 'faggy',\n", " 'fcuk',\n", " 'freaking weirdo',\n", " 'fuckulus',\n", " 'gorilla tag mid',\n", " 'green bean',\n", " 'high glitch',\n", " 'invisible wall',\n", " 'red skin',\n", " 'rim',\n", " 'watermelon munchers',\n", " 'you sped',\n", "]" ] }, { "cell_type": "markdown", "id": "3bf11bab", "metadata": {}, "source": [ "## Prepare rev data" ] }, { "cell_type": "code", "execution_count": 17, "id": "965eb6c8", "metadata": {}, "outputs": [], "source": [ "SAMPLE_RATE = 16_000" ] }, { "cell_type": "code", "execution_count": 13, "id": "9ff97b1c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 11105/11105 [00:03<00:00, 3406.20it/s]\n" ] } ], "source": [ "# some basic checks (since uid is in filename)\n", "uids = []\n", "for fn in os.listdir(RAW_AUDIO_DIR):\n", " if get_file_ext(fn) != \"wav\":\n", " print(\"found unknown file:\", fn)\n", " continue\n", " uid = get_filename(fn)\n", " assert(len(uid) == 36)\n", " uids.append(uid)\n", "assert(len(uids) == len(set(uids)))\n", "\n", "# save in dir with known sample rate as wavs\n", "filepaths_info = []\n", "for fn in tqdm.tqdm(os.listdir(RAW_AUDIO_DIR)):\n", " uid = get_filename(fn)\n", " from_fp = os.path.join(RAW_AUDIO_DIR, fn)\n", " to_fp = os.path.join(RAW_AUDIO_DIR, fn)\n", " audio = Audio.from_file(os.path.join(RAW_AUDIO_DIR, fn), sample_rate=SAMPLE_RATE, byte_width=2)\n", " to_filepath = os.path.join(AUDIO_DIR, uid + \".wav\")\n", " audio.to_wav(to_filepath)\n", " filepaths_info.append((uid, from_fp, to_filepath))" ] }, { "cell_type": "code", "execution_count": 14, "id": "deb0bebc", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(BASE_DIR, \"raw_data_manifest.json\"), \"w\") as f:\n", " json.dump(filepaths_info, f)" ] }, { "cell_type": "code", "execution_count": 20, "id": "4da818da", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "45.1 mins of audio (10.5% overhead)\n", "45.2 mins of audio (12.6% overhead)\n", "45.3 mins of audio (11.5% overhead)\n", "45.0 mins of audio (10.0% overhead)\n", "45.1 mins of audio (9.6% overhead)\n", "45.2 mins of audio (10.4% overhead)\n", "45.2 mins of audio (10.3% overhead)\n", "45.1 mins of audio (9.1% overhead)\n", "45.0 mins of audio (9.7% overhead)\n", "45.0 mins of audio (10.8% overhead)\n", "45.1 mins of audio (11.0% overhead)\n", "45.1 mins of audio (12.0% overhead)\n", "45.1 mins of audio (11.8% overhead)\n", "45.4 mins of audio (10.1% overhead)\n", "45.1 mins of audio (10.8% overhead)\n", "45.0 mins of audio (11.6% overhead)\n", "45.1 mins of audio (10.3% overhead)\n", "46.6 mins of audio (10.0% overhead)\n", "45.1 mins of audio (9.8% overhead)\n", "45.1 mins of audio (9.6% overhead)\n", "45.3 mins of audio (9.2% overhead)\n", "45.1 mins of audio (9.4% overhead)\n", "45.2 mins of audio (10.0% overhead)\n", "45.1 mins of audio (9.5% overhead)\n", "45.1 mins of audio (11.7% overhead)\n", "45.6 mins of audio (10.0% overhead)\n", "47.5 mins of audio (10.3% overhead)\n", "45.1 mins of audio (11.4% overhead)\n", "49.3 mins of audio (8.8% overhead)\n", "45.3 mins of audio (10.3% overhead)\n", "45.1 mins of audio (9.0% overhead)\n", "45.1 mins of audio (10.2% overhead)\n", "45.0 mins of audio (11.1% overhead)\n", "47.1 mins of audio (11.5% overhead)\n", "45.1 mins of audio (10.9% overhead)\n", "45.2 mins of audio (10.9% overhead)\n", "45.1 mins of audio (11.8% overhead)\n", "45.0 mins of audio (11.0% overhead)\n", "45.3 mins of audio (10.2% overhead)\n", "46.5 mins of audio (9.7% overhead)\n", "45.5 mins of audio (10.2% overhead)\n", "45.1 mins of audio (9.5% overhead)\n", "45.1 mins of audio (9.6% overhead)\n", "45.3 mins of audio (10.8% overhead)\n", "45.1 mins of audio (8.7% overhead)\n", "45.3 mins of audio (9.0% overhead)\n", "47.2 mins of audio (9.7% overhead)\n", "46.5 mins of audio (10.1% overhead)\n", "45.0 mins of audio (10.0% overhead)\n", "45.0 mins of audio (10.5% overhead)\n", "45.4 mins of audio (10.7% overhead)\n", "45.8 mins of audio (10.4% overhead)\n", "46.1 mins of audio (10.8% overhead)\n", "45.0 mins of audio (11.3% overhead)\n", "45.0 mins of audio (7.7% overhead)\n", "46.6 mins of audio (11.2% overhead)\n", "45.0 mins of audio (8.9% overhead)\n", "45.1 mins of audio (12.4% overhead)\n", "40.6 mins of audio (9.9% overhead)\n" ] } ], "source": [ "MIN_REV_FILE_DURATION_S = 45 * 60\n", "INDICATOR_AUDIO_DIR = \"/mnt/data-ssd-1/data/custom/segment_indicators\"\n", "SILENCE_DURATION_S = 0.25\n", "\n", "audio_rev_metas = []\n", "tmp_segments_audio = []\n", "tmp_segments_meta = []\n", "tmp_n = 0\n", "for n_step, (uid, _, filepath) in enumerate(filepaths_info):\n", " # add indicator\n", " indicator_str = str(tmp_n).zfill(3)\n", " tmp_indicator_audios = []\n", " for c in indicator_str:\n", " tmp_indicator_audios.append(\n", " Audio.from_file(\n", " os.path.join(INDICATOR_AUDIO_DIR, f\"{c}_fast.wav\"), \n", " sample_rate=SAMPLE_RATE, \n", " byte_width=2,\n", " )\n", " )\n", " indicator_audio = Audio.concatenate(tmp_indicator_audios)\n", " tmp_segments_audio.append(indicator_audio)\n", " tmp_segments_meta.append({\n", " \"segment_number\": tmp_n,\n", " \"type\": \"indicator\",\n", " \"duration_s\": indicator_audio.duration_s,\n", " \"indicator_str\": indicator_str,\n", " })\n", " # add silence\n", " if SILENCE_DURATION_S > 0:\n", " silence_audio = Audio.from_array(\n", " np.zeros(int(SAMPLE_RATE * SILENCE_DURATION_S), dtype=np.int16), SAMPLE_RATE\n", " )\n", " tmp_segments_audio.append(silence_audio)\n", " tmp_segments_meta.append({\n", " \"segment_number\": tmp_n,\n", " \"type\": \"silence\",\n", " \"duration_s\": SILENCE_DURATION_S,\n", " })\n", " # add audio\n", " audio = Audio.from_file(filepath, sample_rate=SAMPLE_RATE, byte_width=2)\n", "# audio = remove_music(audio, wet=1.0, model_name=\"mdx_extra_q\")\n", " tmp_segments_audio.append(audio)\n", " tmp_segments_meta.append({\n", " \"segment_number\": tmp_n,\n", " \"type\": \"speech\",\n", " \"duration_s\": audio.duration_s,\n", " \"orginial_audio_offset_s\": 0,\n", " \"orginial_audio_uid\": uid,\n", " })\n", " # add silence\n", " if SILENCE_DURATION_S > 0:\n", " silence_audio = Audio.from_array(\n", " np.zeros(int(SAMPLE_RATE * SILENCE_DURATION_S), dtype=np.int16), SAMPLE_RATE\n", " )\n", " tmp_segments_audio.append(silence_audio)\n", " tmp_segments_meta.append({\n", " \"segment_number\": tmp_n,\n", " \"type\": \"silence\",\n", " \"duration_s\": SILENCE_DURATION_S,\n", " })\n", " # prep for next step and decide if we make new file\n", " tmp_n += 1\n", " running_duration_s = np.sum([e[\"duration_s\"] for e in tmp_segments_meta])\n", " if (\n", " (running_duration_s >= MIN_REV_FILE_DURATION_S) or \n", " (n_step == len(filepaths_info) - 1 and running_duration_s > 0)\n", " ):\n", " audio_rev = Audio.concatenate(tmp_segments_audio)\n", " running_speech_duration_s = np.sum([e[\"duration_s\"] for e in tmp_segments_meta if e[\"type\"] == \"speech\"])\n", " print(\"{} mins of audio ({}% overhead)\".format(\n", " round(running_duration_s / 60, 1),\n", " round((1 - running_speech_duration_s / running_duration_s) * 100, 1)\n", " )) \n", " uid = str(uuid.uuid4())\n", " rev_audio_filepath = os.path.join(TO_REV_AUDIO_DIR, f\"{uid}.mp3\")\n", " audio_rev.to_mp3(rev_audio_filepath)\n", " audio_rev_metas.append({\n", " \"uid\": uid,\n", " \"audio_filepath\": rev_audio_filepath,\n", " \"segments_meta\": tmp_segments_meta,\n", " \"duration_s\": audio_rev.duration_s,\n", " })\n", " tmp_segments_audio = []\n", " tmp_segments_meta = []\n", " tmp_n = 0" ] }, { "cell_type": "code", "execution_count": 22, "id": "4d16d48e", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\"), \"w\") as f:\n", " json.dump(audio_rev_metas, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "a453909d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "5fc18307", "metadata": {}, "source": [ "## (optional) investigate" ] }, { "cell_type": "code", "execution_count": 25, "id": "dc409c9d", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", " audio_rev_metas = json.load(f)" ] }, { "cell_type": "code", "execution_count": 24, "id": "ea85a5f2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "11105 transcripts\n" ] }, { "data": { "text/plain": [ "[('09e56ba8-49c6-4301-802a-13a9d25d56aa', 'ohh fuck bro had a bounce'),\n", " ('081b33a7-128f-46f3-a40b-e83114bc8752',\n", " \"bro i snatch blocked him bro what the fuck no god this game is so delayed with that stuff it's not hate socially\")]" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "with open(os.path.join(RAW_DATA_DIR, \"transcripts.tsv\")) as f:\n", " transcripts = [e.split(\"\\t\") for e in f.read().strip().split(\"\\n\")]\n", "assert(all([len(e) == 2 for e in transcripts]))\n", "transcripts = [(uid, text.lower()) for uid, text in transcripts]\n", "print(len(transcripts), \"transcripts\")\n", "transcripts[:2]" ] }, { "cell_type": "code", "execution_count": 27, "id": "3604b3cb", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "Audio.from_file(audio_rev_metas[0][\"audio_filepath\"]).play()" ] }, { "cell_type": "code", "execution_count": 28, "id": "40f5c4a2", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "Audio.from_file(audio_rev_metas[-1][\"audio_filepath\"]).play()" ] }, { "cell_type": "code", "execution_count": 63, "id": "b10f8455", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['because my arms are not that long and yours']" ] }, "execution_count": 63, "metadata": {}, "output_type": "execute_result" } ], "source": [ "[text for uid, text in transcripts if uid == \"83550ed7-3300-493c-b0f7-5fff71655e96\"]" ] }, { "cell_type": "code", "execution_count": 62, "id": "bd121247", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[\"it's gotta say i'm eleven i'm black bullied in school i've called all the time\"]" ] }, "execution_count": 62, "metadata": {}, "output_type": "execute_result" } ], "source": [ "[text for uid, text in transcripts if uid == \"08ee313e-7906-409a-91bc-4d1fa666e7cd\"]" ] }, { "cell_type": "code", "execution_count": null, "id": "58a2ca6f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "41e4dbe5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "426ba2c3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "dfc212cd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "88a9e9cc", "metadata": {}, "source": [ "## Send stuff to rev" ] }, { "cell_type": "code", "execution_count": 75, "id": "05fbe672", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.rev import get_auth_string, upload_file, make_order, make_cc_order\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 83, "id": "2db71de1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "44.6 hours total\n" ] } ], "source": [ "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", " audio_rev_metas = json.load(f)\n", "\n", "ORDER_REF_STR = \"sl_gc_40h\"\n", "\n", "tot_duration_s = np.sum([m[\"duration_s\"] for m in audio_rev_metas])\n", "print(round(tot_duration_s / 60 / 60, 1), \"hours total\")" ] }, { "cell_type": "code", "execution_count": 78, "id": "8c66f7f9", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 59/59 [04:13<00:00, 4.30s/it]\n" ] } ], "source": [ "order_items = []\n", "for m in tqdm.tqdm(audio_rev_metas):\n", " media_loc = upload_file(AUTH_STR, m[\"audio_filepath\"], file_ref_str=m[\"uid\"])\n", " order_items.append(\n", " {\n", " \"media_loc\": media_loc,\n", " \"hotwords\": GLOBAL_HOTWORDS,\n", " \"duration_s\": int(np.ceil(m[\"duration_s\"])),\n", " \"uuid\": m[\"uid\"],\n", " }\n", " )" ] }, { "cell_type": "code", "execution_count": 79, "id": "a0ee5f4f", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, \"01_order_items.json\"), \"w\") as f:\n", " json.dump(order_items, f)" ] }, { "cell_type": "code", "execution_count": 80, "id": "8318e46f", "metadata": {}, "outputs": [], "source": [ "# order_number = make_order(AUTH_STR, order_items, order_ref_str=ORDER_REF_STR)\n", "order_number = make_cc_order(AUTH_STR, order_items, order_ref_str=ORDER_REF_STR)" ] }, { "cell_type": "code", "execution_count": 81, "id": "27549d51", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, \"02_order_numer.txt\"), \"w\") as f:\n", " f.write(order_number)" ] }, { "cell_type": "code", "execution_count": null, "id": "0096715b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "576b2bef", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b33dc8e0", "metadata": {}, "source": [ "## Get rev stuff" ] }, { "cell_type": "code", "execution_count": 18, "id": "e8b59ece", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.rev import get_auth_string, get_finished_order, get_transcript\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 19, "id": "9e4e40af", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, \"02_order_numer.txt\")) as f:\n", " order_number = f.read()" ] }, { "cell_type": "code", "execution_count": 30, "id": "d2a371b5", "metadata": {}, "outputs": [], "source": [ "# grab while still in progress\n", "from suno_utils.web.rev import _get_order_details, _parse_order_response\n", "order_details = _get_order_details(AUTH_STR, order_number)\n", "_, transcript_metas = _parse_order_response(order_details)" ] }, { "cell_type": "code", "execution_count": 21, "id": "9275ae0d", "metadata": {}, "outputs": [], "source": [ "# # grab all of it\n", "# transcript_metas = get_finished_order(AUTH_STR, order_number, blocking=False)" ] }, { "cell_type": "code", "execution_count": 31, "id": "df28d887", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 59/59 [00:56<00:00, 1.05it/s]\n" ] } ], "source": [ "# 59 total\n", "for transcript_meta in tqdm.tqdm(transcript_metas):\n", " transcript_filepath = os.path.join(FROM_REV_DIR, transcript_meta[\"uuid\"] + \".srt\")\n", " if os.path.exists(transcript_filepath):\n", " continue\n", " transcript_subrip = get_transcript(AUTH_STR, transcript_meta)\n", " with open(transcript_filepath, \"w\") as f:\n", " f.write(transcript_subrip)" ] }, { "cell_type": "code", "execution_count": 82, "id": "ed8a84c6", "metadata": {}, "outputs": [], "source": [ "# !rm /mnt/data-ssd-1/data/private/customer/speechly/2022-08-30_gymclass_1h/from_rev/acc78c33-25fa-4d16-9e02-6c4a74f5b373.srt" ] }, { "cell_type": "code", "execution_count": null, "id": "627dfcfc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f2b631bc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b15ce9de", "metadata": {}, "source": [ "## Parsing rev subrips" ] }, { "cell_type": "code", "execution_count": 526, "id": "2b80ff9a", "metadata": {}, "outputs": [], "source": [ "from nltk.corpus import words as nltk_words\n", "from suno_utils.utils.numbers import safe_round\n", "from suno_utils.audio.data_model import Token, Tokens, TEXT, TAG\n", "from suno_utils.customers.sanas.pipeline import get_segments\n", "from suno_utils.utils.text import normalize_whitespace\n", "\n", "COMMON_EN_WORDS = set(nltk_words.words())\n", "\n", "def _parse_srt(transcript_srt):\n", " blocks = []\n", " for block in transcript_srt.strip().split(\"\\n\\n\"):\n", " lines = block.strip().split(\"\\n\")\n", " segment_nr = int(lines[0])\n", " start_str, end_str = lines[1].split(\" --> \")\n", " h, m, s = start_str.split(\":\")\n", " start_s = int(h) * 60**2 + int(m) * 60 + float(s.replace(\",\", \".\"))\n", " h, m, s = end_str.split(\":\")\n", " end_s = int(h) * 60**2 + int(m) * 60 + float(s.replace(\",\", \".\"))\n", " text_lines = lines[2:]\n", " blocks.append({\n", " \"segment_nr\": segment_nr, \n", " \"offset_s\": safe_round(start_s),\n", " \"duration_s\": safe_round(end_s - start_s),\n", " \"text_lines\": text_lines, \n", " })\n", " if blocks[0][\"segment_nr\"] != 1:\n", " raise ValueError(\"First block doesn't start with 1.\")\n", " if any(np.diff([e[\"segment_nr\"] for e in blocks]) != 1):\n", " raise ValueError(\"Parsed block numbers are not consecutive.\")\n", " return blocks\n", "\n", "def _tag_repl(m):\n", " s = m.group()\n", " if s is not None:\n", " s = s.lower()\n", " s = re.sub(r\"[\\(\\)]\", \" \", s.lower())\n", " s = normalize_whitespace(s)\n", " s = s.replace(\" \", \"_\")\n", " return f\" [{s}] \"\n", "\n", "def _partial_word_repl(m):\n", " s = m.group()\n", " word = s.strip(\".- \")\n", " probably_fragment = word not in COMMON_EN_WORDS\n", " if probably_fragment:\n", " return f\"{word}* -- \"\n", " else:\n", " return f\"{word} -- \"\n", "\n", "from suno_utils.utils.text import make_unique_list\n", "\n", "def _desetify_meta_repl(m):\n", " s = m.group()\n", " tags = re.findall(r\"\\[.*?\\]\", s)\n", " tags = make_unique_list(tags)\n", " return \" {} \".format(\" \".join(tags))\n", " \n", "ALLOWED_TAGS = set([\n", " \"laughter\", \"foreign_language\", \"inaudible\", \"music\", \"redacted\", \"crosstalk\"\n", "])\n", "\n", "def _standardize_tags(text):\n", " # collapse meta tags and filter\n", " for m in list(re.finditer(r\"\\[.*?\\]\", text))[::-1]:\n", " tag_text = text[m.start() + 1:m.end() - 1]\n", " # collapse notation\n", " if \"foreign\" in tag_text:\n", " tag_text = \"foreign_language\"\n", " if \"laugh\" in tag_text:\n", " tag_text = \"laughter\"\n", " if \"indistinct\" in tag_text or \"mumbl\" in tag_text:\n", " tag_text = \"inaudible\"\n", " if \"music\" in tag_text or \"\\\"\" in tag_text or \"plays\" in tag_text or \"sing\" in tag_text:\n", " tag_text = \"music\"\n", " if \"chatter\" in tag_text:\n", " tag_text = \"crosstalk\"\n", " # filter\n", " if tag_text in ALLOWED_TAGS:\n", " cleaned_tag_text = f\"[{tag_text}]\"\n", " else:\n", " cleaned_tag_text = \"\"\n", " text = text[:m.start()] + cleaned_tag_text + text[m.end():]\n", " return text\n", " \n", "def parse_rev_captions(transcript_srt, lang=\"en\"):\n", " if lang != \"en\":\n", " raise NotImplementedError(\"only en is supported for now.\")\n", " if len(re.findall(r\"\\(beep\\)\", transcript_srt)) >= 3:\n", " print(\"careful, lots of (beep) detected\")\n", " cur_speaker_id = 0\n", " token_list = []\n", " for block in _parse_srt(transcript_srt):\n", " lines = block[\"text_lines\"]\n", " if len(lines) == 0:\n", " continue\n", " # handle music lyrics\n", " annotated_lines = []\n", " cur_lyrics = False\n", " for line in lines:\n", " is_start = re.match(r\"^\\s*♪.*\", line)\n", " is_end = re.match(r\".*♪\\s*$\", line)\n", " if is_start:\n", " line = line.strip()[1:]\n", " if is_end:\n", " line = line.strip()[:-1]\n", " if is_start and is_end:\n", " annotated_lines.append((line, True))\n", " cur_lyrics = False\n", " elif is_start:\n", " annotated_lines.append((line, True))\n", " cur_lyrics = True\n", " elif is_end:\n", " annotated_lines.append((line, True))\n", " cur_lyrics = False\n", " else:\n", " annotated_lines.append((line, False))\n", " if re.match(r\".*♪\\s*$\", line):\n", " cur_lyrics = False\n", " # merge if no lyrics and if no new speaker\n", " if all([not is_lyrics for _, is_lyrics in annotated_lines]):\n", " merged_lines = []\n", " tmp_lines = []\n", " for line, _ in annotated_lines:\n", " if re.search(r\"^\\s*\\-\", line):\n", " if len(tmp_lines) > 0:\n", " merged_lines.append((\" \".join([s.strip() for s in tmp_lines]), False))\n", " tmp_lines = []\n", " tmp_lines.append(line)\n", " if len(tmp_lines) > 0:\n", " merged_lines.append((\" \".join([s.strip() for s in tmp_lines]), False))\n", " tmp_lines = []\n", " else:\n", " merged_lines = annotated_lines\n", " # add actual lines\n", " for raw_line, is_lyrics in annotated_lines:\n", " # handle speaker change\n", " line = raw_line.strip()\n", " if line[:1] == \"-\":\n", " cur_speaker_id += 1\n", " line = line[1:].strip()\n", " # handle meta tags\n", " if re.search(r\"\\([^\\)]*\\(\", line): # treat nested by brute forcing\n", " line = re.sub(r\"\\((.*)\\)\", _tag_repl, line)\n", " line = re.sub(r\"\\((.*?)\\)\", _tag_repl, line)\n", " # handle redacted\n", " line = re.sub(r\"[^\\s]*\\*+\", \" [redacted] \", line)\n", " # handle hesitations and word fragments\n", " line = re.sub(r\"\\s\\-[\\s\\-]*\", \" -- \", line) # handle single dash\n", " line = re.sub(r\"[A-Za-z]+(\\.{2,}|\\-(\\s|$))\", _partial_word_repl, line)\n", " line = re.sub(r\"\\.{2,}\", \" \", line) # we only keep interruptions as --\n", " # collapse meta tags and discard if necessary\n", " line = _standardize_tags(line)\n", " # discard partial tags (usually nested)\n", " line = re.sub(r\"^[^\\[]{,20}\\]\", \"\", line)\n", " # clean standalone punctuation (from behind tags)\n", "# line = re.sub(r\"(\\[.*?\\])([\\.\\,\\!\\?\\s]+)\", \" \\\\2 \\\\1 \", line) # swap with meta\n", " line = re.sub(r\"\\][\\.\\,\\!\\?\\-\\s]+\", \"] \", line) # remove if two consecutive metaas\n", " line = re.sub(r\"[\\.\\,\\!\\?\\-\\s]*\\s([\\.\\,\\!\\?])\", \"\\\\1\", line)\n", " line = re.sub(r\"(^|\\s+)[\\.\\,\\?\\!\\-\\s]+\", \" \", line)\n", " # add tokens\n", " line = normalize_whitespace(line)\n", " if len(line) == 0:\n", " continue\n", " tokens = []\n", " for s in line.split():\n", " # identify word fragment\n", " if re.match(r\"^[^\\*]+\\*$\", s):\n", " is_fragment = True\n", " s = s[:-1]\n", " else:\n", " is_fragment = False\n", " if s[:1] == \"[\" and s[-1:] == \"]\":\n", " is_text = False\n", " elif s == \"--\":\n", " is_text = False\n", " else:\n", " is_text = True\n", " if is_text:\n", " token = Token(\n", " s, \n", " speaker_id=cur_speaker_id, \n", " type=TEXT, \n", " metadata={\"is_lyrics\": is_lyrics},\n", " )\n", " else:\n", " token = Token(s, type=TAG)\n", " if token.type == TAG and len(tokens) > 0 and token == tokens[-1]:\n", " # skip consecutive meta tokens\n", " continue\n", " tokens.append(token)\n", " if is_fragment:\n", " tokens.append(Token(\"[word_fragment_boundary]\", type=TAG))\n", " token_list.extend(tokens)\n", " # do things with global context\n", " # de-setify metas\n", " cleaned_tokens = []\n", " tmp_tokens = []\n", " for token in token_list:\n", " if token.type != TAG:\n", " if len(tmp_tokens) > 0:\n", " cleaned_tokens.extend(tmp_tokens)\n", " tmp_tokens = []\n", " cleaned_tokens.append(token)\n", " else:\n", " if token.value not in set([t.value for t in tmp_tokens]):\n", " tmp_tokens.append(token)\n", " if len(tmp_tokens) > 0:\n", " cleaned_tokens.extend(tmp_tokens)\n", " tmp_tokens = []\n", " transcript_tokens = Tokens(cleaned_tokens)\n", " return transcript_tokens" ] }, { "cell_type": "code", "execution_count": 527, "id": "00e04fcf", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", " audio_rev_metas = json.load(f)" ] }, { "cell_type": "code", "execution_count": 544, "id": "936b45dd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "217/217 segments retained in 59511101-f9fe-4fd8-aff4-69927a0ad3e4.\n" ] } ], "source": [ "# from suno_utils.web.rev import parse_transcript\n", "\n", "segment_transcripts = []\n", "for m in audio_rev_metas:#[3:4]:\n", " if m[\"uid\"] != \"59511101-f9fe-4fd8-aff4-69927a0ad3e4\":\n", " continue\n", " srt_filepath = os.path.join(FROM_REV_DIR, m[\"uid\"] + \".srt\")\n", " if not os.path.exists(srt_filepath):\n", " print(\"not found, skipping...\")\n", " continue\n", " with open(srt_filepath) as f:\n", " transcript_srt = f.read()\n", " rev_transcript = parse_rev_captions(transcript_srt)\n", " speech_segment_info = [mm for mm in m[\"segments_meta\"] if mm[\"type\"] == \"speech\"]\n", " n_segments = len(speech_segment_info)\n", " assert(len(speech_segment_info) == m[\"segments_meta\"][-1][\"segment_number\"] + 1)\n", " token_segments = get_segments(rev_transcript, n_segments, global_indicator_speaker=False, n_indicator_digits=3)\n", " segment_transcripts_chunk = []\n", " missing_ids = set(range(n_segments))\n", " for segment_idx, tokens in token_segments:\n", " segment_info = speech_segment_info[segment_idx]\n", " segment_transcripts_chunk.append({\n", " \"uid\": segment_info[\"orginial_audio_uid\"],\n", " \"duration_s\": segment_info[\"duration_s\"],\n", " \"transcript\": tokens.anonymize_speakers(),\n", " })\n", " missing_ids -= set([segment_idx])\n", " missing_ids = sorted([str(n) for n in missing_ids])\n", " missing_str = \"\"\n", " if len(missing_ids) > 0:\n", " missing_str = \" (\" + \",\".join(missing_ids[:5])\n", " if len(missing_ids) > 5:\n", " missing_str += \"...\"\n", " missing_str += \")\"\n", " print(\"{}/{} segments retained in {}{}.\".format(\n", " len(segment_transcripts_chunk), n_segments, m[\"uid\"], missing_str\n", " ))\n", " segment_transcripts.extend(segment_transcripts_chunk)\n", " break" ] }, { "cell_type": "code", "execution_count": 530, "id": "2cbe6494", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "45a812db-16ef-494f-acde-6d151d012a70\n", "Not even close, bro. Shut the fuck up.\n", "----------\n", "c18d98ba-0602-40cf-9b43-a1e97d2d6515\n", "Turning that bitch ass shit off, bro\n", "----------\n", "603dac77-5fad-403c-80a4-486d97279033\n", "What the fuck [laughter] Who says that, bro? Who says that? Oh my God. Get roasted toasted. Yeah. Get roasted, get toasted.\n", "----------\n", "c5b8420f-e457-4cda-bc31-f9b9199b1df7\n", "Are you fucking serious?\n", "----------\n", "83e11a4f-f5c5-4f66-95e8-1da3e2d86130\n", "What the fuck?\n", "----------\n" ] } ], "source": [ "for m in segment_transcripts[:5]:\n", " print(m[\"uid\"])\n", " print(m[\"transcript\"].text)\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 546, "id": "136464c0", "metadata": {}, "outputs": [], "source": [ "# print(transcript_srt)" ] }, { "cell_type": "code", "execution_count": 543, "id": "50313c41", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "transcript id: 59511101-f9fe-4fd8-aff4-69927a0ad3e4\n" ] } ], "source": [ "segment_uid = \"3cb0cc5b-c5be-46a6-966b-2684c5fe4868\"\n", "for m in segments_manifest:\n", " if m[\"id\"] == segment_uid:\n", " print(\"transcript id:\", m[\"source_transcript_id\"])\n", " break" ] }, { "cell_type": "code", "execution_count": 533, "id": "90178e26", "metadata": {}, "outputs": [], "source": [ "# save as json\n", "import copy\n", "l = []\n", "for m in segment_transcripts:\n", " e = copy.deepcopy(m)\n", " e[\"transcript\"] = e[\"transcript\"].as_dict()\n", " l.append(e)\n", "with open(os.path.join(BASE_DIR, \"parsed_transcripts.json\"), \"w\") as f:\n", " json.dump(l, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "c2636a15", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c34c0db7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9937474e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "2624f95e", "metadata": {}, "source": [ "## prepare pipeline" ] }, { "cell_type": "code", "execution_count": 534, "id": "9433c48a", "metadata": {}, "outputs": [], "source": [ "# map segment id to rev submission id\n", "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", " audio_rev_metas = json.load(f)\n", "segment_to_rev_id = {}\n", "for m in audio_rev_metas:\n", " rev_id = m[\"uid\"]\n", " for mm in m[\"segments_meta\"]:\n", " if mm[\"type\"] != \"speech\":\n", " continue\n", " segment_to_rev_id[mm[\"orginial_audio_uid\"]] = rev_id" ] }, { "cell_type": "code", "execution_count": 535, "id": "a29859c9", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(BASE_DIR, \"parsed_transcripts.json\")) as f:\n", " segment_transcripts = json.load(f)" ] }, { "cell_type": "code", "execution_count": 536, "id": "020e71bb", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 11105/11105 [00:00<00:00, 159494.80it/s]\n" ] } ], "source": [ "manifest = []\n", "for m in tqdm.tqdm(segment_transcripts):\n", " segment_filepath = os.path.join(AUDIO_DIR, m[\"uid\"] + \".wav\") \n", " new_segment_filepath = os.path.join(SEGMENTS_DIR, \"{}.wav\".format(m[\"uid\"]))\n", " if not os.path.exists(new_segment_filepath):\n", " audio = Audio.from_file(segment_filepath, sample_rate=16_000, byte_width=2)\n", " audio.to_wav(new_segment_filepath)\n", " manifest.append({\n", " \"id\": m[\"uid\"],\n", " \"uri\": new_segment_filepath,\n", " \"duration_s\": m[\"duration_s\"],\n", " \"transcript\": {\"tokens\": m[\"transcript\"]},\n", " \"ignore_segment\": False,\n", " \"ignore_reason\": \"\",\n", " \"source_audio_offset_s\": 0,\n", " \"source_audio_uri\": segment_filepath,\n", " \"source_audio_id\": m[\"uid\"],\n", " \"source_transcript_id\": segment_to_rev_id[m[\"uid\"]],\n", " \"source_transcript_uri\": os.path.join(FROM_REV_DIR, segment_to_rev_id[m[\"uid\"]] + \".srt\"),\n", " })" ] }, { "cell_type": "code", "execution_count": 537, "id": "cee8873a", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"01_segment_meta.json\"), \"w\") as f:\n", " json.dump(manifest, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "9aff6c62", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6cf2b71d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "651e980e", "metadata": {}, "source": [ "## QA" ] }, { "cell_type": "code", "execution_count": 538, "id": "a8dae9e1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "40.0 hours\n" ] } ], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"01_segment_meta.json\")) as f:\n", " segments_manifest = json.load(f)\n", "print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": 539, "id": "6a5f0e8c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "8861a8fc-7bc0-4a21-b26d-94c9488296d1\n", "Shut up! I am Stephen Curry. Watch this, bro. I'm Steph'er Curry, bro. Watch this. Fuck! I'm a Steph'er Curry. Wait, hold up, hold up. I gotta change my shoot power.\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "63863713-cc4e-4431-9301-4f7a44c15b30\n", "That's some for big [redacted] gger, [redacted] gger.\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "da230152-7c00-4b54-9f98-569167e0437a\n", "Bro, I'm fucking retarded. I changed my throwing settings to the max on accident tryna see what happens if I put my hand settings to the max and I actually was like, \"Oh my god, this is good hand settings.\" And I'm, I'm still, I still think I'm mentally autistic.\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "e74e29d5-e537-4c44-96e2-96ea3f2ce12e\n", "How about, how about I fuck your mom?\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1f4eca12-077c-4a84-8010-cc0b85d76636\n", "I'm gonna stop this bit.\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n" ] } ], "source": [ "import random\n", "# random ones\n", "# l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n", "l = segments_manifest[:]\n", "random.shuffle(l)\n", "for e in l[:5]:\n", " print(e[\"id\"])\n", " print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n", " Audio.from_file(e[\"uri\"]).play()\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 489, "id": "081a8b29", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "45bfa253-011b-4027-9162-5d14f0403d58\n" ] } ], "source": [ "# get rev id for segment\n", "segment_uid = \"846622bb-b4ab-439c-9316-3683831bafbe\"\n", "for m in segments_manifest:\n", " if m[\"id\"] == segment_uid:\n", " print(m[\"source_transcript_id\"])\n", " break" ] }, { "cell_type": "code", "execution_count": null, "id": "a574e8b2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "36eb7b98", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "64fe98cc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "7b8ddb47", "metadata": {}, "source": [ "## run pipeline" ] }, { "cell_type": "code", "execution_count": 11, "id": "1e319351", "metadata": {}, "outputs": [], "source": [ "# # reset pipeline\n", "# !mv /mnt/data-ssd-1/data/private/customer/speechly/2022-09-23_gymclass_40h/pipeline/2022_09_23/artifacts/01_segment_meta.json /tmp/01_segment_meta.json\n", "# !rm -rf /mnt/data-ssd-1/data/private/customer/speechly/2022-09-23_gymclass_40h/pipeline/2022_09_23/artifacts/*\n", "# !mv /tmp/01_segment_meta.json /mnt/data-ssd-1/data/private/customer/speechly/2022-09-23_gymclass_40h/pipeline/2022_09_23/artifacts/" ] }, { "cell_type": "code", "execution_count": 153, "id": "7fcf3139", "metadata": {}, "outputs": [], "source": [ "# # 17mins for 40h\n", "# CUDA_VISIBLE_DEVICES=0 python /home/georg/notebooks/customers/speechly/gymclass_40h/run_modified_pipeline.py" ] }, { "cell_type": "code", "execution_count": null, "id": "ec2da368", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fb1aec05", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "cfa1946d", "metadata": {}, "source": [ "## look at results" ] }, { "cell_type": "code", "execution_count": 12, "id": "30275498", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "40.0 hours\n" ] } ], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"02_segment_meta_post_norm_and_asr.json\")) as f:\n", " segments_manifest = json.load(f)\n", "print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": 15, "id": "5848265d", "metadata": { "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "5f31e854-186a-4eed-8769-67c7bce8fba7\n", "What happened to my ball? Bro, he keeps stealing my ball. I can't dunk. I can't do nothing, he keeps stealing my dunk.\n", "what happened to my ball bro he keeps stealing my ball i can't dunk i can't do nothing he keeps stealing my dunk\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "d848ff19-a66d-448b-85b5-4f3682599dc4\n", "Bro, fuck.\n", "bro fuck\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "3f02dc10-3cb7-4620-85ec-d34176aefe07\n", "Let's get nigga.\n", "let's get nigga\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "e47a13ac-ef9e-4bff-9b13-e815f328a80f\n", "Gimme that shit, nigga. Gimme that, teammate, right here.\n", "gimme that shit nigga gimme that teammate right here\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "9376ced2-9f27-4ef1-8b59-fe795f591bac\n", "What the Fucked it up, bro. [laughter] Wait, where's it going, knowing with your body, bruh?\n", "what the fucked it up bro [laughter] wait where's it going knowing with your body bruh\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n" ] } ], "source": [ "import random\n", "# random ones\n", "# l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n", "l = segments_manifest[:]\n", "random.shuffle(l)\n", "for e in l[:5]:\n", " print(e[\"id\"])\n", " print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n", " print(Tokens.from_dict(e[\"transcript_normalized\"][\"tokens\"]).text)\n", " Audio.from_file(e[\"uri\"]).play()\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 16, "id": "34165d5a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "full: Sitting on a squashed banana. What the frick. [inaudible] Nigga [inaudible] Cristiano Reinaldo, suey.\n", "norm: sitting on a squashed banana what the frick [inaudible] nigga [inaudible] cristiano reinaldo suey\n", "asr : sitting on a squash banalon a hermani bamba what a frak ding abuga digga ching a wing dinga t jiga winga diggo w kima nima iga inga ing a inga mingle inga jingleing ingaing christiana nato sui\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "uid = \"6643b5a8-06de-45cb-a99f-351c54d49133\"\n", "for e in segments_manifest:\n", " if e[\"id\"] == uid:\n", " print(\"full:\", Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n", " print(\"norm:\", Tokens.from_dict(e[\"transcript_normalized\"][\"tokens\"]).text)\n", " print(\"asr :\", e[\"transcript_asr\"][\"transcript\"])\n", " Audio.from_file(e[\"uri\"]).play()" ] }, { "cell_type": "code", "execution_count": 28, "id": "afd18433", "metadata": {}, "outputs": [], "source": [ "# l = []\n", "# for e in segments_manifest:\n", "# l.extend(re.findall(r\"[a-zA-Z]+\\-[a-zA-Z]+\", Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text))" ] }, { "cell_type": "code", "execution_count": 494, "id": "51e32035", "metadata": {}, "outputs": [], "source": [ "# # segment_uid = \"30815903-8a47-4d14-8557-41c352526d5d\"\n", "# # segment_uid = \"974c5846-5ae1-4521-9ca2-83175d563c55\"\n", "# # segment_uid = \"45a812db-16ef-494f-acde-6d151d012a70\"\n", "# segment_uid = \"383220cb-e8f1-4b6d-b39b-2d126eb50163\" \n", "\n", "# for n, m in enumerate(audio_rev_metas):\n", "# rev_id = m[\"uid\"]\n", "# for nn, mm in enumerate(m[\"segments_meta\"]):\n", "# if mm[\"type\"] != \"speech\":\n", "# continue\n", "# if mm[\"orginial_audio_uid\"] == segment_uid:\n", "# print(n, nn)\n", "# print(audio_rev_metas[n][\"uid\"])\n", "# print(audio_rev_metas[n][\"segments_meta\"][nn])\n", "# # print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)" ] }, { "cell_type": "code", "execution_count": null, "id": "539144e1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "dc0d9e56", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "938c2793", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "6b0192cb", "metadata": {}, "source": [ "## format output for speechly styleguide" ] }, { "cell_type": "code", "execution_count": 17, "id": "336f6909", "metadata": {}, "outputs": [], "source": [ "# !wget https://www.cs.cmu.edu/~biglou/resources/bad-words.txt" ] }, { "cell_type": "code", "execution_count": 18, "id": "798221f7", "metadata": {}, "outputs": [], "source": [ "# TODO: do we need to keep dashes during norm? what about M-mhm uh-huh, ...\n", "# TODO:\n", "# https://www.cs.cmu.edu/~biglou/resources/bad-words.txt\n", "# https://github.com/zacanger/profane-words" ] }, { "cell_type": "code", "execution_count": 19, "id": "7c3f2643", "metadata": {}, "outputs": [], "source": [ "with open(\"bad-words.txt\") as f:\n", " profane_words = set([s_clean for s in f.read().strip().split(\"\\n\") if len((s_clean := s.strip())) > 0])\n", "profane_words_ptn = \"|\".join(re.escape(s.lower()) for s in sorted(profane_words, key=len, reverse=True))\n", "PROFANE_WORDS_RE = re.compile(r\"\\b(?:\" + profane_words_ptn + r\")\\b\")\n", "\n", "def _contains_profanity(text):\n", " return bool(PROFANE_WORDS_RE.search(text.lower()))" ] }, { "cell_type": "code", "execution_count": 20, "id": "290b5f06", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.harvest import get_filename\n", "\n", "with open(os.path.join(BASE_DIR, \"raw_data_manifest.json\")) as f:\n", " filepaths_info = json.load(f)\n", "id_to_original_id_map = {e[0]: get_filename(e[2], keep_extension=True) for e in filepaths_info}\n", "inv_id_to_original_id_map = {v: k for k, v in id_to_original_id_map.items()}\n", "\n", "with open(os.path.join(ARTIFACTS_DIR, \"02_segment_meta_post_norm_and_asr.json\")) as f:\n", " segments_manifest = json.load(f)" ] }, { "cell_type": "code", "execution_count": 21, "id": "ecd8a4d5", "metadata": {}, "outputs": [], "source": [ "def _convert_full_transcript(tokens):\n", " transcript_l = []\n", " for turn in tokens.speaker_turns:\n", " turn_tokens = Tokens.from_dict(turn[\"tokens\"])\n", " active_lyrics = False\n", " text_l = []\n", " for t in turn_tokens:\n", " is_lyrics = False\n", " if t.metadata is not None:\n", " is_lyrics = t.metadata[\"is_lyrics\"]\n", " if is_lyrics and not active_lyrics:\n", " text_l.append(\"\")\n", " if not is_lyrics and active_lyrics:\n", " text_l.append(\"\")\n", " text_l.append(t.value)\n", " active_lyrics = is_lyrics\n", " if active_lyrics:\n", " text_l.append(\"\")\n", " text = \" \".join(text_l)\n", " text = re.sub(r\"\\\\s+\", \"\", text)\n", " text = re.sub(r\"\\s+\\<\\/lyrics\\>\", \"\", text)\n", " transcript_l.append(text)\n", " transcript = \" [speaker_change] \".join(transcript_l)\n", " transcript = re.sub(r\"\\s+\\[word\\_fragment\\_boundary\\]\", \"*\", transcript)\n", " # fix some basic parse mistackes\n", " transcript = re.sub(r\"^\\s*\\[music\\] \\[speaker\\_change\\]\\s*\", \"[music] \", transcript)\n", " transcript = re.sub(r\"\\b([Ff])cuk\\b\", \"\\\\1uck\", transcript)\n", " transcript = normalize_whitespace(transcript)\n", " return transcript\n", "\n", "def _convert_norm_transcript(tokens):\n", " transcript = tokens.text\n", " transcript = transcript.replace(\" [word_fragment_boundary]\", \"*\")\n", " transcript = transcript.replace(\"[foreign_language]\", \"\")\n", " transcript = transcript.replace(\"[inaudible]\", \"\")\n", " transcript = transcript.replace(\"[crosstalk]\", \"\")\n", " transcript = transcript.replace(\"[redacted]\", \"\")\n", " transcript = transcript.replace(\"--\", \" \")\n", " transcript = re.sub(r\"\\[.*?\\]\", \" \", transcript)\n", " teanscript = re.sub(r\"\\bfcuk\\b\", \"fuck\", transcript)\n", " transcript = normalize_whitespace(transcript)\n", " return transcript" ] }, { "cell_type": "code", "execution_count": 25, "id": "b2465b35", "metadata": {}, "outputs": [], "source": [ "# jsonl \n", "\n", "# {\n", "# \"audio\": \"0a4ac9ac-42a4-4e77-91ad-862b0509d913.wav.\", // the audio file name\n", "# \"transcript\": \".....\", // The transcript of the audio\n", "# \"tags\": [ ] // The \"OFF\" and/or \"AC\" tags if relevant to the utterance\n", "# }\n", "# Audio corrupted utterance (AC): for only severely corrupted audio files\n", "# Offensive utterance (OFF): profanities, bullying language, hate speech included in the audio file\n", "output_metadata = []\n", "for n, m in enumerate(segments_manifest):\n", " norm_tokens = Tokens.from_dict(m[\"transcript_normalized\"][\"tokens\"])\n", " tokens = Tokens.from_dict(m[\"transcript\"][\"tokens\"])\n", " norm_transcript = _convert_norm_transcript(norm_tokens)\n", " transcript = _convert_full_transcript(tokens)\n", " tags = []\n", " if _contains_profanity(norm_transcript):\n", " tags.append(\"OFF\")\n", " original_filename = id_to_original_id_map[m[\"id\"]]\n", " \n", "# if \"foreign\" in transcript:\n", "# print(norm_transcript)\n", "# print(\"-\")\n", "# print(transcript)\n", "# print(\"-\"*10)\n", "# if len(norm_transcript) == 0:\n", "# print(norm_transcript)\n", "# print(\"-\")\n", "# print(transcript)\n", "# print(\"-\"*10)\n", " \n", " # some manual overrides\n", " # TODO: use CER here in future to punt and maybe reannotate\n", "# if original_filename == \"c7bfc752-be04-4c91-b018-bf809ad19323.mp3\":\n", "# transcript = transcript.replace(\"Banja not to this barely\", \"[foreign_language] Chef Bally\")\n", "# norm_transcript = norm_transcript.replace(\"we ' re\", \"we're\")\n", "# if original_filename == \"cc6284ac-dc8b-420c-9bd0-8470d2e2d88d.mp3\":\n", "# transcript = \"Damn.\"\n", "# norm_transcript = \"damn\"\n", "# tags = [\"OFF\"]\n", "# if original_filename == \"b451b2ff-0dd2-4a8b-a317-34ea67f172bd.mp3\":\n", "# transcript = transcript.replace(\"I can't\", \"I can't, I can't\")\n", "# norm_transcript = norm_transcript.replace(\"i can't\", \"i can't i can't\")\n", "# if original_filename == \"4600e16e-fd7d-4c69-9e1d-49b1a3c2b088.mp3\":\n", "# transcript = transcript.replace(\"y'all\", \"yo\")\n", "# norm_transcript = norm_transcript.replace(\"y'all\", \"yo\") \n", "# if original_filename == \"3cd8a90f-8475-4d7c-84f3-07e9b7aa0227.mp3\":\n", "# transcript = \"What?\"\n", "# norm_transcript = \"what\"\n", "# if original_filename == \"558a3f70-8f55-4b7e-9b14-6d6e2a436218.mp3\":\n", "# transcript = \"\"\n", "# norm_transcript = \"[inaudible]\"\n", " if original_filename == \"35063d27-f049-41d1-a157-417ea07f068c.mp3\":\n", " transcript = transcript.replace(\"one zero\", \"ten\")\n", "\n", " output_metadata.append({\n", " \"audio\": original_filename,\n", " \"transcript\": normalize_whitespace(norm_transcript),\n", " \"transcript_full\": normalize_whitespace(transcript),\n", " \"tags\": tags,\n", " })" ] }, { "cell_type": "code", "execution_count": 26, "id": "00cecd18", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"delivery_meta.jsonl\"), \"w\") as f:\n", " for e in output_metadata:\n", " f.write(json.dumps(e))\n", " f.write(\"\\n\")" ] }, { "cell_type": "code", "execution_count": null, "id": "4fd2b759", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fc59e298", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "635386fe", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "6b0b614f", "metadata": {}, "source": [ "## Final QA" ] }, { "cell_type": "code", "execution_count": 27, "id": "a61aa689", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"delivery_meta.jsonl\")) as f:\n", " output_metadata = [json.loads(e) for e in f.read().strip().split(\"\\n\")]" ] }, { "cell_type": "code", "execution_count": 28, "id": "8a5d9f26", "metadata": {}, "outputs": [], "source": [ "# for e in output_metadata:\n", "# if e[\"audio\"] == \"1034ef38-79aa-48de-a908-0dd6a65c1a36.mp3\":\n", "# print(e[\"audio\"], \"- profanity:\", \"OFF\" in e[\"tags\"])\n", "# Audio.from_file(os.path.join(BASE_DIR, \"audio\", e[\"audio\"])).play()\n", "# print(e[\"transcript\"])\n", "# print(\"-\")\n", "# print(e[\"transcript_full\"])\n", "# break" ] }, { "cell_type": "code", "execution_count": 29, "id": "201ac267", "metadata": {}, "outputs": [], "source": [ "# # figure out rev\n", "# with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", "# audio_rev_metas = json.load(f)\n", "# uid = inv_id_to_original_id_map[\"harrisnt_-7072462733219728682.m4v\"]\n", "# for rev_m in audio_rev_metas:\n", "# for seg_m in rev_m[\"segments_meta\"]:\n", "# if seg_m.get(\"orginial_audio_uid\") == uid:\n", "# print(seg_m)" ] }, { "cell_type": "code", "execution_count": 30, "id": "9ccfd86b", "metadata": {}, "outputs": [], "source": [ "# import random\n", "# # random ones\n", "# # l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n", "# l = [e for e in output_metadata if len(e[\"transcript\"]) == 0]\n", "# random.shuffle(l)\n", "# for e in l[:5]:\n", "# print(e[\"audio\"], \"- profanity:\", \"OFF\" in e[\"tags\"])\n", "# Audio.from_file(os.path.join(BASE_DIR, \"audio\", e[\"audio\"])).play()\n", "# print(e[\"transcript\"])\n", "# print(\"-\")\n", "# print(e[\"transcript_full\"])\n", "# print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 31, "id": "a3f6a7dd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "d311ce6a-f289-4bbd-a8de-3fa801b14512.wav - profanity: True\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "marie hey marie get back swimmer what you got bro get your bitch ass back before i punch your\n", "-\n", "Marie, hey Marie. Get back, swimmer, what you got? Bro, get your bitch-ass back before I punch your.\n", "----------\n", "eac61a88-922a-4175-8ee3-af61f2f2a794.wav - profanity: True\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "oh my god move ball wait hold on oh my\n", "-\n", "Oh my God, move ball! Wait, hold on. Oh my\n", "----------\n", "a83ccfc8-d6fc-4978-8384-321cbbb57656.wav - profanity: True\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "nigga\n", "-\n", "Nigga.\n", "----------\n", "27f3c61b-bbd6-4257-a9ef-e48893253e15.wav - profanity: True\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "so early in the morning for me yelling like hey hey with a butt on my i'm spinning on these pros like a mother wait wait wait k flock k flock can i add you\n", "-\n", "So early in the morning for me yelling like Hey, hey, with a butt on my I'm spinning on these pros like a mother Wait, wait, wait, K-Flock, K-Flock, can I add you?\n", "----------\n", "f93fcbdf-bf91-44d0-8659-32064de5e020.wav - profanity: False\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "where is the mine's a one fourteen mine's a one fourteen my angle thirty four\n", "-\n", "Where is the [speaker_change] Mine's a 1, 14. [speaker_change] Mine's a 1, 14. My angle? 34.\n", "----------\n" ] } ], "source": [ "import random\n", "# random ones\n", "# l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n", "l = output_metadata[:]\n", "random.shuffle(l)\n", "for e in l[:5]:\n", " print(e[\"audio\"], \"- profanity:\", \"OFF\" in e[\"tags\"])\n", " Audio.from_file(os.path.join(BASE_DIR, \"audio\", e[\"audio\"])).play()\n", " print(e[\"transcript\"])\n", " print(\"-\")\n", " print(e[\"transcript_full\"])\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": null, "id": "76c34c77", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a33601a9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "9f84953b", "metadata": {}, "source": [ "## compare with their output" ] }, { "cell_type": "code", "execution_count": 35, "id": "08368ccc", "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 33, "id": "18112b59", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "11105 transcripts\n" ] }, { "data": { "text/plain": [ "[('09e56ba8-49c6-4301-802a-13a9d25d56aa', 'ohh fuck bro had a bounce'),\n", " ('081b33a7-128f-46f3-a40b-e83114bc8752',\n", " \"bro i snatch blocked him bro what the fuck no god this game is so delayed with that stuff it's not hate socially\")]" ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "with open(os.path.join(RAW_DATA_DIR, \"transcripts.tsv\")) as f:\n", " sl_transcripts = [e.split(\"\\t\") for e in f.read().strip().split(\"\\n\")]\n", "assert(all([len(e) == 2 for e in sl_transcripts]))\n", "sl_transcripts = [(uid, text.lower()) for uid, text in sl_transcripts]\n", "print(len(sl_transcripts), \"transcripts\")\n", "sl_transcripts[:2]" ] }, { "cell_type": "code", "execution_count": 36, "id": "e038f68f", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-09-29 17:35:38 nemo_logging:349] /tmp/ipykernel_3407955/2405280702.py:1: FutureWarning: In a future version of pandas all arguments of DataFrame.drop except for the argument 'labels' will be keyword-only\n", " test_df = pd.DataFrame(output_metadata).drop([\"tags\"], 1)\n", " \n" ] } ], "source": [ "test_df = pd.DataFrame(output_metadata).drop([\"tags\"], 1)\n", "test_df[\"uid\"] = test_df[\"audio\"].str.split(\".\").str[0]\n", "test_df[\"sl_transcript\"] = test_df[\"uid\"].map({k: v for k, v in sl_transcripts})" ] }, { "cell_type": "code", "execution_count": 37, "id": "b1f33880", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.metrics import get_wer\n", "test_df[\"wer\"] = [get_wer(a, b) for a, b in zip(test_df[\"transcript\"], test_df[\"sl_transcript\"])]\n", "test_df = test_df.sort_values(\"wer\", ascending=False).reset_index(drop=True)" ] }, { "cell_type": "code", "execution_count": 38, "id": "a66aa1b0", "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
audiotranscripttranscript_fulluidsl_transcriptwer
9ddf85902-0e28-4e22-9959-dbd2fe335efc.wavdingo <u> mute mute hey my friend made a fanma...Dingo! [inaudible] Mute, mute. Hey my friend m...ddf85902-0e28-4e22-9959-dbd2fe335efcbingo new new new new very good opening pingu ...1.0
278630b1cd-b4ce-47d6-b283-9346d3737e71.wavdam what the hell <u> bitch you're a concernin...Dam, what the hell? [crosstalk] [speaker_chang...8630b1cd-b4ce-47d6-b283-9346d3737e711.0
29d0b09d5e-71d8-4c76-848e-831ac750da10.wavbitch shut the fuck up hold up shut the fuck u...[music] Bitch. Shut the fuck up. Hold up. Shut...d0b09d5e-71d8-4c76-848e-831ac750da101.0
30977c3b1d-0304-4f91-b96b-cfb0497c5ead.wavi'm the best player couldn't get rid of him if...<lyrics>I'm the best player</lyrics> [speaker_...977c3b1d-0304-4f91-b96b-cfb0497c5eadfucking you represent it's for you you can't c...1.0
33aaa597f0-4946-444f-a7a3-eec6bf4b26f7.wav<u> eat dirt eat dirt eat dirt eat dirt me tim...[crosstalk] [speaker_change] Eat dirt, eat dir...aaa597f0-4946-444f-a7a3-eec6bf4b26f7better retard he died retard1.0
\n", "
" ], "text/plain": [ " audio \\\n", "9 ddf85902-0e28-4e22-9959-dbd2fe335efc.wav \n", "27 8630b1cd-b4ce-47d6-b283-9346d3737e71.wav \n", "29 d0b09d5e-71d8-4c76-848e-831ac750da10.wav \n", "30 977c3b1d-0304-4f91-b96b-cfb0497c5ead.wav \n", "33 aaa597f0-4946-444f-a7a3-eec6bf4b26f7.wav \n", "\n", " transcript \\\n", "9 dingo mute mute hey my friend made a fanma... \n", "27 dam what the hell bitch you're a concernin... \n", "29 bitch shut the fuck up hold up shut the fuck u... \n", "30 i'm the best player couldn't get rid of him if... \n", "33 eat dirt eat dirt eat dirt eat dirt me tim... \n", "\n", " transcript_full \\\n", "9 Dingo! [inaudible] Mute, mute. Hey my friend m... \n", "27 Dam, what the hell? [crosstalk] [speaker_chang... \n", "29 [music] Bitch. Shut the fuck up. Hold up. Shut... \n", "30 I'm the best player [speaker_... \n", "33 [crosstalk] [speaker_change] Eat dirt, eat dir... \n", "\n", " uid \\\n", "9 ddf85902-0e28-4e22-9959-dbd2fe335efc \n", "27 8630b1cd-b4ce-47d6-b283-9346d3737e71 \n", "29 d0b09d5e-71d8-4c76-848e-831ac750da10 \n", "30 977c3b1d-0304-4f91-b96b-cfb0497c5ead \n", "33 aaa597f0-4946-444f-a7a3-eec6bf4b26f7 \n", "\n", " sl_transcript wer \n", "9 bingo new new new new very good opening pingu ... 1.0 \n", "27 1.0 \n", "29 1.0 \n", "30 fucking you represent it's for you you can't c... 1.0 \n", "33 better retard he died retard 1.0 " ] }, "execution_count": 38, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_df[test_df[\"transcript\"].str.len() >= 50].head()" ] }, { "cell_type": "code", "execution_count": 39, "id": "dc008f15", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
audiotranscripttranscript_fulluidsl_transcriptwer
1798a79cc-96df-409b-82ee-143bf66940c9.wav<u> bro my shoot upon oh shut up like <u>[inaudible] [speaker_change] Bro my shoot upon...798a79cc-96df-409b-82ee-143bf66940c9then leave are you fucking dumb are you really...1.0
575df8c55-72cf-4c2e-a0f3-54d1cd714787.wavgimme dat oh fuck oh my gosh <u>Gimme dat. Oh fuck, oh my gosh. [inaudible]75df8c55-72cf-4c2e-a0f3-54d1cd714787i'm gonna be that ohf for my boy my mom is1.0
68f260fcb-f33a-4d99-aac0-c2d79acbd644.wav<u> watch y'all when i'm fucked[crosstalk] [speaker_change] Watch y'all when ...8f260fcb-f33a-4d99-aac0-c2d79acbd6441.0
9ddf85902-0e28-4e22-9959-dbd2fe335efc.wavdingo <u> mute mute hey my friend made a fanma...Dingo! [inaudible] Mute, mute. Hey my friend m...ddf85902-0e28-4e22-9959-dbd2fe335efcbingo new new new new very good opening pingu ...1.0
113d9c372e-526f-4803-8bee-3362469c0b7f.wav<u>[inaudible]3d9c372e-526f-4803-8bee-3362469c0b7fwell fucking1.0
\n", "
" ], "text/plain": [ " audio \\\n", "1 798a79cc-96df-409b-82ee-143bf66940c9.wav \n", "5 75df8c55-72cf-4c2e-a0f3-54d1cd714787.wav \n", "6 8f260fcb-f33a-4d99-aac0-c2d79acbd644.wav \n", "9 ddf85902-0e28-4e22-9959-dbd2fe335efc.wav \n", "11 3d9c372e-526f-4803-8bee-3362469c0b7f.wav \n", "\n", " transcript \\\n", "1 bro my shoot upon oh shut up like \n", "5 gimme dat oh fuck oh my gosh \n", "6 watch y'all when i'm fucked \n", "9 dingo mute mute hey my friend made a fanma... \n", "11 \n", "\n", " transcript_full \\\n", "1 [inaudible] [speaker_change] Bro my shoot upon... \n", "5 Gimme dat. Oh fuck, oh my gosh. [inaudible] \n", "6 [crosstalk] [speaker_change] Watch y'all when ... \n", "9 Dingo! [inaudible] Mute, mute. Hey my friend m... \n", "11 [inaudible] \n", "\n", " uid \\\n", "1 798a79cc-96df-409b-82ee-143bf66940c9 \n", "5 75df8c55-72cf-4c2e-a0f3-54d1cd714787 \n", "6 8f260fcb-f33a-4d99-aac0-c2d79acbd644 \n", "9 ddf85902-0e28-4e22-9959-dbd2fe335efc \n", "11 3d9c372e-526f-4803-8bee-3362469c0b7f \n", "\n", " sl_transcript wer \n", "1 then leave are you fucking dumb are you really... 1.0 \n", "5 i'm gonna be that ohf for my boy my mom is 1.0 \n", "6 1.0 \n", "9 bingo new new new new very good opening pingu ... 1.0 \n", "11 well fucking 1.0 " ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_df[test_df[\"transcript_full\"].str.contains(r\"\\[\")].head()" ] }, { "cell_type": "code", "execution_count": 40, "id": "faca5ede", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "4b235196-f1e3-4184-932a-18ed4315eb07\n", "bye nigga bye nigga\n", "buying buy\n" ] } ], "source": [ "n = 23\n", "row = test_df.iloc[n]\n", "Audio.from_file(os.path.join(AUDIO_DIR, row[\"audio\"])).play()\n", "print(row[\"uid\"])\n", "print(row[\"transcript\"])\n", "print(row[\"sl_transcript\"])" ] }, { "cell_type": "code", "execution_count": 415, "id": "22240080", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "15 642\n", "2f60e11b-7fbb-498a-86e4-efaeb67a0f17\n", "{'segment_number': 160, 'type': 'speech', 'duration_s': 34.24, 'orginial_audio_offset_s': 0, 'orginial_audio_uid': '6643b5a8-06de-45cb-a99f-351c54d49133'}\n" ] } ], "source": [ "# segment_uid = \"30815903-8a47-4d14-8557-41c352526d5d\"\n", "# segment_uid = \"974c5846-5ae1-4521-9ca2-83175d563c55\"\n", "# segment_uid = \"45a812db-16ef-494f-acde-6d151d012a70\"\n", "segment_uid = \"6643b5a8-06de-45cb-a99f-351c54d49133\" \n", "\n", "for n, m in enumerate(audio_rev_metas):\n", " rev_id = m[\"uid\"]\n", " for nn, mm in enumerate(m[\"segments_meta\"]):\n", " if mm[\"type\"] != \"speech\":\n", " continue\n", " if mm[\"orginial_audio_uid\"] == segment_uid:\n", " print(n, nn)\n", " print(audio_rev_metas[n][\"uid\"])\n", " print(audio_rev_metas[n][\"segments_meta\"][nn])\n", "# print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)" ] }, { "cell_type": "code", "execution_count": null, "id": "fc2a4bce", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7fc6b380", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bc513d9e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d51f20f8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "36ae3093", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": null, "id": "d3d86de5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a6f077fb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d394bb5b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ff7c5d47", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }