{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "5e5e22f5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 2, "id": "c083f71f", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ['CUDA_VISIBLE_DEVICES'] = '0'" ] }, { "cell_type": "code", "execution_count": 48, "id": "9e0547b7", "metadata": {}, "outputs": [], "source": [ "import re\n", "import shutil\n", "\n", "from suno_utils.audio.conversion import Audio, get_audio_properties\n", "from suno_utils.utils.parser import filter_lines, get_pdf_lines, parse_transcript_segments, segments_to_tokens\n", "# from suno_utils.utils.slicer import get_annotated_slices\n", "from suno_utils.utils.text import normalize_whitespace\n", "from suno_utils.tasks.asr import transcribe\n", "from suno_utils.utils.metrics import get_wer\n", "\n", "# from suno_utils.utils.notebook import _" ] }, { "cell_type": "code", "execution_count": 4, "id": "d1ef2bc3", "metadata": {}, "outputs": [], "source": [ "def _format_segments(segments):\n", " \"\"\"Corpus specific basic formatting of text.\"\"\"\n", " clean_segments = []\n", " prev_speaker = None\n", " for segment_type, speaker, text in segments:\n", " text = text.replace(\"[music]\", \" \").strip()\n", " text = re.sub(r\"[0-9]+\\:[0-9]+\\:[0-9]+\\*\", \" \", text).strip()\n", " if len(text) == 0:\n", " continue\n", " if segment_type == \"speech\":\n", " # simplify hesitations\n", " clean_text = text.replace(\"--\", \" -- \").replace(\"...\", \" -- \")\n", " clean_text = re.sub(r\"[^\\s\\-]\\-(?:\\s|$)\", \" -- \", clean_text).strip()\n", " # turn parens sidenotes into commas\n", " # TODO: maybe we should turn this into -- ?\n", " clean_text = re.sub(r\"\\s*\\(\\s*(.+?)\\s*\\)\", \", \\\\1\", clean_text)\n", " clean_text = normalize_whitespace(clean_text)\n", " clean_segments.append((segment_type, speaker, clean_text))\n", " elif segment_type == \"metatag\":\n", " text = text.strip(\"[]\")\n", " if text in [\"laugs\", \"laughter\", \"laughing\"]:\n", " clean_segments.append((segment_type, None, \"[laughter]\"))\n", " elif text == \"hesitation\":\n", " clean_segments.append((\"speech\", None, \"--\"))\n", "\n", " else:\n", " raise NotImplementedError(\"unknown segment type\")\n", " prev_speaker = speaker\n", " return clean_segments\n", "\n", "def merge_segments(segments):\n", " new_segments = []\n", " cur_texts = []\n", " cur_speaker_id = None\n", " for _, speaker_id, text in segments:\n", " if speaker_id is None or speaker_id == cur_speaker_id:\n", " cur_texts.append(text)\n", " else:\n", " if len(cur_texts) > 0:\n", " new_segments.append({\n", " \"text\": \" \".join(cur_texts),\n", " \"speaker_id\": cur_speaker_id,\n", " })\n", " cur_texts = [text]\n", " cur_speaker_id = speaker_id\n", " if len(cur_texts) > 0:\n", " new_segments.append({\n", " \"text\": \" \".join(cur_texts),\n", " \"speaker_id\": cur_speaker_id,\n", " })\n", " return new_segments" ] }, { "cell_type": "code", "execution_count": 90, "id": "b49c9116", "metadata": {}, "outputs": [], "source": [ "BASE_DIR = \"/mnt/data-ssd-1/data/podcasts/other/brain-science/\"\n", "shutil.rmtree(os.path.join(BASE_DIR, \"transcripts_parsed\"))\n", "os.makedirs(os.path.join(BASE_DIR, \"transcripts_parsed\"), exist_ok=True)" ] }, { "cell_type": "code", "execution_count": 92, "id": "7c854da6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "153 pairs found\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 153/153 [56:08<00:00, 22.02s/it]\n" ] } ], "source": [ "import tqdm\n", "import json\n", "import pandas as pd\n", "\n", "audio_fns = os.listdir(os.path.join(BASE_DIR, \"audios\"))\n", "transcript_fns = os.listdir(os.path.join(BASE_DIR, \"transcripts\"))\n", "# find unique pairs\n", "transcript_numbers = [\n", " ee for e in transcript_fns if re.match(r\"^[0-9]+$\", (ee := e.split(\"-\")[0]))\n", "]\n", "transcript_numbers_2 = [\n", " ee for e in transcript_fns if re.match(r\"^[0-9]+$\", (ee := e.split(\"-\")[0])) and e.endswith(\".pdf\")\n", "]\n", "unique_transcript_numbers = (\n", " set(pd.Series(transcript_numbers).value_counts().loc[lambda x: x == 1].index) & \n", " set(transcript_numbers_2)\n", ")\n", "audio_numbers = [ee for e in audio_fns if re.match(r\"^[0-9]+$\", (ee := e.split(\"-\")[0]))]\n", "audio_numbers_2 = [ee for e in audio_fns if re.match(r\"^[0-9]+$\", (ee := e.split(\"-\")[0])) and e.endswith(\".mp3\")]\n", "unique_audio_numbers = (\n", " set(pd.Series(audio_numbers).value_counts().loc[lambda x: x == 1].index) & \n", " set(audio_numbers_2)\n", ")\n", "unique_numbers = unique_transcript_numbers & unique_audio_numbers\n", "audio_fps = [os.path.join(BASE_DIR, \"audios\", fn) for fn in audio_fns if fn.split(\"-\")[0] in unique_numbers]\n", "transcript_fps = [\n", " os.path.join(BASE_DIR, \"transcripts\", fn) for fn in transcript_fns if fn.split(\"-\")[0] in unique_numbers\n", "]\n", "audio_fps = sorted(audio_fps, key=lambda x: int(x.split(\"/\")[-1].split(\"-\")[0]))\n", "transcript_fps = sorted(transcript_fps, key=lambda x: int(x.split(\"/\")[-1].split(\"-\")[0]))\n", "assert(len(audio_fps) == len(transcript_fps))\n", "print(len(audio_fps), \"pairs found\")\n", "\n", "metadata = []\n", "for audio_fp, transcript_fp in tqdm.tqdm(zip(audio_fps, transcript_fps), total=len(audio_fps)):\n", "\n", " uid = transcript_fp.split(\"/\")[-1].split(\"-\")[0]\n", " \n", " output_filepath = os.path.join(BASE_DIR, \"transcripts_parsed\", uid + \".json\")\n", "# if os.path.exists(output_filepath):\n", "# continue\n", " \n", " try:\n", " raw_lines = get_pdf_lines(transcript_fp)\n", " speaker_ptn = r\"^[^0-9]+\\:\"\n", " segments = parse_transcript_segments(raw_lines, speaker_ptn=speaker_ptn)\n", " # filter out obviously incorrect stuff\n", " segments = [s for s in segments if not (s[0] == \"speech\" and s[1] is None and len(s[2]) < 20)]\n", " formatted_segments = _format_segments(segments)\n", " merged_segments = merge_segments(formatted_segments)\n", " except:\n", " continue\n", "\n", " if len(merged_segments) == 0:\n", " continue\n", " \n", " with open(output_filepath, \"w\") as f:\n", " json.dump(merged_segments, f)\n", " \n", " transcript_text = transcribe(audio_fp)\n", " fulltext = normalize_whitespace(\n", " re.sub(r\"[^\\s0-9a-z\\']\", \" \", \" \".join([e[\"text\"] for e in merged_segments]).lower())\n", " )\n", " wer = get_wer(fulltext, transcript_text)\n", "\n", " campbell_speaker_frac = (\n", " np.sum([\n", " len(e[\"text\"]) \n", " for e in merged_segments \n", " if e[\"speaker_id\"] is not None \n", " and \"campbell\" in e[\"speaker_id\"].lower()\n", " ]) /\n", " np.sum([len(e[\"text\"]) for e in merged_segments])\n", " )\n", " \n", " hl = [e[\"speaker_id\"] for e in merged_segments]\n", " potential_single_speaker = len(hl) == len(set(hl))\n", " \n", " metadata.append({\n", " \"id\": uid,\n", " \"audio_fp\": audio_fp,\n", " \"transcript_fp\": transcript_fp,\n", " \"parsed_transcript_fp\": output_filepath,\n", " \"wer\": round(wer, 3),\n", " \"duration_s\": round(get_audio_properties(audio_fp)[\"duration_s\"], 1),\n", " \"campbell_speaker_frac\": campbell_speaker_frac,\n", " \"potential_single_speaker\": potential_single_speaker,\n", " })" ] }, { "cell_type": "code", "execution_count": 93, "id": "a7b4e583", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(BASE_DIR, \"parse_metadata.json\"), \"w\") as f:\n", " json.dump(metadata, f)" ] }, { "cell_type": "code", "execution_count": 98, "id": "11518b02", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[55, 56, 69, 32, 71, 53]" ] }, "execution_count": 98, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# select 10 epsiodes with good wer and reasonable speaker fraction\n", "[int(round(m[\"duration_s\"] / 60)) for m in metadata if m[\"wer\"] <= 0.2 and m[\"campbell_speaker_frac\"] >= 0.3]" ] }, { "cell_type": "code", "execution_count": null, "id": "b31a2d38", "metadata": {}, "outputs": [], "source": [ "speaker_2_pairs = [\n", " (\"audios/191-BS-Review.mp3\", \"transcripts_parsed/191-brainscience-review-2.json\"),\n", " (\"audios/5-BSP-consciousness.mp3\", \"transcripts_parsed/5-brainscience-Consciousness.json\"),\n", " (\"audios/151-BS-emotion.mp3\", \"transcripts_parsed/151-brainscience-emotion.json\"),\n", " (\"audios/125-BSP-Review.mp3\", \"transcripts_parsed/125-brainscience-Review.json\"),\n", " (\"audios/114-BSP-review.mp3\", \"transcripts_parsed/114-brainscience-review8.json\"),\n", "]" ] }, { "cell_type": "code", "execution_count": 100, "id": "d438dd02", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[('audios/73-BSP-Shapiro-extra.mp3', 'transcripts_parsed/73.json'),\n", " ('audios/74-bsp-Sporns.mp3', 'transcripts_parsed/74.json'),\n", " ('audios/91-BSP-Panksepp.mp3', 'transcripts_parsed/91.json'),\n", " ('audios/113-bsp-DalaiLama.mp3', 'transcripts_parsed/113.json'),\n", " ('audios/132-BS-Uttal.mp3', 'transcripts_parsed/132.json'),\n", " ('audios/189-BS-Damasio.mp3', 'transcripts_parsed/189.json')]" ] }, "execution_count": 100, "metadata": {}, "output_type": "execute_result" } ], "source": [ "[\n", " (\n", " m[\"audio_fp\"].replace(\"/mnt/data-ssd-1/data/podcasts/other/brain-science/\", \"\"), \n", " m[\"parsed_transcript_fp\"].replace(\"/mnt/data-ssd-1/data/podcasts/other/brain-science/\", \"\"), \n", " )\n", " for m in metadata if m[\"wer\"] <= 0.2 and m[\"campbell_speaker_frac\"] >= 0.3\n", "]" ] }, { "cell_type": "code", "execution_count": 89, "id": "9783f02e", "metadata": {}, "outputs": [], "source": [ "# !ls /mnt/data-ssd-1/data/podcasts/other/brain-science/transcripts_parsed" ] }, { "cell_type": "code", "execution_count": null, "id": "8ef8d4c0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "4c953781", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1bac2fc4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f9d97338", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "54fde026", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b77e7cdb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9765b5ad", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "da87a161", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": null, "id": "47e4a9df", "metadata": {}, "outputs": [], "source": [ "## Formatting study\n", "# parentheses are side-notes\n", "# trailing off vs hesitation: two minutes...\n", "# \"am gonna sorta try\" -> am going to try\n", "# same for \"sort of\" but not always\n", "# \"i wanna\" - > I want to\n", "# \"okay, you\" - > You\n", "# \"right, and i\" - > And I\n", "# \"the error -- that is the brain\" - > the error -- the brain\n", "# \"like\" removed" ] }, { "cell_type": "code", "execution_count": null, "id": "8aea330f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fd472e72", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "335feb0e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f87b0701", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5969b613", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "afcb7e75", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }