{ "cells": [ { "cell_type": "code", "execution_count": 81, "id": "b304dfff", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-09-08 18:59:24 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/IPython/core/magics/pylab.py:159: UserWarning: pylab import has clobbered these variables: ['f', 'random', 'e']\n", " `%matplotlib` prevents importing * from pylab and numpy\n", " warn(\"pylab import has clobbered these variables: %s\" % clobbered +\n", " \n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 82, "id": "f955e852", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "code", "execution_count": 83, "id": "e9716000", "metadata": {}, "outputs": [], "source": [ "import time\n", "import os\n", "import tqdm\n", "import shutil\n", "import random\n", "import funcy\n", "import json\n", "import uuid\n", "import numpy as np\n", "import multiprocessing\n", "import pandas as pd\n", "from suno_utils.audio import Audio\n", "from suno_utils.audio import Tokens\n", "from suno_utils.audio.conversion import play_audio\n", "\n", "BASE_DIR = \"/home/georg/notebooks/customers/aiera\"\n", "PIPELINE_DIR = \"/home/georg/notebooks/customers/aiera/pipeline\"" ] }, { "cell_type": "markdown", "id": "bb1a375b", "metadata": {}, "source": [ "## get transcript" ] }, { "cell_type": "code", "execution_count": 13, "id": "2b52b360", "metadata": {}, "outputs": [], "source": [ "import json\n", "import os" ] }, { "cell_type": "code", "execution_count": 3, "id": "e8b59ece", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.rev import get_auth_string, get_finished_order, get_transcript\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 4, "id": "9e4e40af", "metadata": {}, "outputs": [], "source": [ "order_number = \"TC0433873319\"\n", "\n", "transcript_metas = get_finished_order(AUTH_STR, order_number)\n", "assert(len(transcript_metas) == 1)" ] }, { "cell_type": "code", "execution_count": 15, "id": "df28d887", "metadata": {}, "outputs": [], "source": [ "transcript_meta = transcript_metas[0]\n", "transcript_filepath = os.path.join(BASE_DIR, \"sample_audio.json\")\n", "transcript_json = get_transcript(AUTH_STR, transcript_meta)\n", "with open(transcript_filepath, \"w\") as f:\n", " json.dump(transcript_json, f)" ] }, { "cell_type": "code", "execution_count": 74, "id": "9d846bb6", "metadata": {}, "outputs": [], "source": [ "manifest_filepath = os.path.join(BASE_DIR, \"annotation_manifest.json\")\n", "manifest = [\n", " {\n", " \"audio_filepath\": os.path.join(BASE_DIR, \"sample_audio.mp3\"),\n", " \"transcript_filepath\": os.path.join(BASE_DIR, \"sample_audio.json\"),\n", " },\n", "]\n", "with open(manifest_filepath, \"w\") as f:\n", " json.dump(manifest, f)" ] }, { "cell_type": "markdown", "id": "21bd9d66", "metadata": {}, "source": [ "## Run pipeline" ] }, { "cell_type": "code", "execution_count": null, "id": "a9b5d0bb", "metadata": {}, "outputs": [], "source": [ "CUDA_VISIBLE_DEVICES=0 python -m suno_utils.scripts.rev_podcast.run \\\n", " --output_dir=\"/home/georg/notebooks/customers/aiera/pipeline\" \\\n", " --use_gpu \\\n", " --conda-env=\"mfa\" \\\n", " --conda_sh_filepath=\"/home/georg/anaconda3/etc/profile.d/conda.sh\" \\\n", " --mfa_dictionary_name=\"english_mfa\" \\\n", " --mfa_acoustic_model_name=\"english_mfa\" \\\n", " --num_cores=20 \\\n", " --manifest_file=\"/home/georg/notebooks/customers/aiera/annotation_manifest.json\" \\\n", " --dataset_name=\"aiera\" \\\n", " --sample_rate=16000 \\\n", " --max_cer_thres=0.6 \\\n", " --skip_verify_boundaries \\\n", " --min_silence_gap_s=0.02 \\\n", " --max_segment_duration_s=20.0 \\\n", " --n_align_success_pad=2 \\\n", " --s3_sample_data_prefix=\"s3://suno-static-public/datasets/tmp/\"" ] }, { "cell_type": "code", "execution_count": 29, "id": "c52a8de1", "metadata": {}, "outputs": [], "source": [ "with open(\"/home/georg/notebooks/customers/aiera/pipeline/alignment-gentle/sample_audio.json\") as f:\n", " a = json.load(f)" ] }, { "cell_type": "code", "execution_count": 36, "id": "3261f1ba", "metadata": {}, "outputs": [], "source": [ "for e in a[\"words\"]:\n", " e.pop(\"phones\", None)" ] }, { "cell_type": "code", "execution_count": 37, "id": "6184c1d4", "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 38, "id": "50a9f2b5", "metadata": {}, "outputs": [], "source": [ "df = pd.DataFrame(a[\"words\"])" ] }, { "cell_type": "code", "execution_count": 40, "id": "2a88852b", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "not-found-in-audio 3389\n", "success 3015\n", "not-found-in-transcript 167\n", "Name: case, dtype: int64" ] }, "execution_count": 40, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"case\"].value_counts()" ] }, { "cell_type": "markdown", "id": "93a4632d", "metadata": {}, "source": [ "## (optional) Try ctc align" ] }, { "cell_type": "code", "execution_count": 44, "id": "485ec244", "metadata": {}, "outputs": [], "source": [ "from suno_utils.audio import Audio\n", "from suno_utils.tasks.ctc_align import align_transcript\n", "from suno_utils.web.rev import parse_transcript\n", "\n", "kenlm_bin_dir = \"/home/georg/tools/kenlm/build/bin\"\n", "audio_filepath = os.path.join(BASE_DIR, \"sample_audio.mp3\")\n", "transcript_filepath = os.path.join(BASE_DIR, \"sample_audio.json\")\n", "audio = Audio.from_file(audio_filepath)\n", "with open(transcript_filepath) as f:\n", " raw_transcript = json.load(f)\n", "transcript_tokens = parse_transcript(raw_transcript)\n", "aligned_tokens = align_transcript(audio_filepath, transcript_tokens, kenlm_bin_dir=kenlm_bin_dir)" ] }, { "cell_type": "code", "execution_count": 63, "id": "fe3d85e0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "144 segments\n", "78.2 % coverage\n" ] } ], "source": [ "from suno_utils.utils.slicer_v2 import get_segments\n", "\n", "segments = get_segments(\n", " aligned_tokens,\n", " audio.duration_s,\n", " min_segment_duration_s=2,\n", " max_segment_duration_s=20,\n", " min_silence_gap_s=0.2,\n", " min_confidence=0.4,\n", " ends_are_safe=True,\n", ")\n", "print(len(segments), \"segments\")\n", "cov_s = 0\n", "for (start_s, end_s), _ in segments:\n", " cov_s += end_s - start_s\n", "print(round(cov_s / audio.duration_s * 100, 1), \"% coverage\")" ] }, { "cell_type": "code", "execution_count": 66, "id": "8bc426e4", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "you know, the thing that really is remarkable about, uh, the product is it's -- the -- the depth that it goes and the breadth. The depth within an indication, we're down to six-month-olds in atopic dermatitis, which, you know, think about a biologic being used\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "a ready -- already focused target than -- than DOI and free cash flow, so to -- You know, [inaudible] to say, but is that right way of -- of [inaudible] You know, [inaudible] things -- things about\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "um, good afternoon, everyone. It's Steven [inaudible] here from the [inaudible] I'm delighted to be hosting Bill Sibold, but I don't think he needs an introduction, but\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "as bad as HR3 would've been. Um, the various components of it, you know, inflation penalties for pricing, uh, for us not a -- a real concern. Uh, we have implemented, uh, an industry-leading, uh, pricing policy\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "is [inaudible] not monopolizing, so if you have any questions\n", "----------\n" ] } ], "source": [ "l = list(zip(range(len(segments)), segments[:]))\n", "random.shuffle(l)\n", "for n, ((start_s, end_s), tokens) in l[:5]:\n", " audio.get_slice(start_s, end_s).play()\n", " print(n)\n", " print(tokens.text)\n", " print(\"-\"*10)" ] }, { "cell_type": "markdown", "id": "4a1bbe14", "metadata": {}, "source": [ "## Run pipeline with new slicer" ] }, { "cell_type": "code", "execution_count": 93, "id": "d8432aa9", "metadata": {}, "outputs": [], "source": [ "import shutil\n", "shutil.rmtree(PIPELINE_DIR, ignore_errors=True)\n", "os.makedirs(PIPELINE_DIR)" ] }, { "cell_type": "code", "execution_count": null, "id": "37c86a06", "metadata": {}, "outputs": [], "source": [ "CUDA_VISIBLE_DEVICES=0 python -m suno_utils.scripts.rev_podcast.run \\\n", " --output_dir=\"/home/georg/notebooks/customers/aiera/pipeline\" \\\n", " --use_gpu \\\n", " --use_slicer_v2 \\\n", " --conda-env=\"mfa\" \\\n", " --conda_sh_filepath=\"/home/georg/anaconda3/etc/profile.d/conda.sh\" \\\n", " --kenlm_dir=\"/home/georg/tools/kenlm/build/bin\" \\\n", " --mfa_dictionary_name=\"english_mfa\" \\\n", " --mfa_acoustic_model_name=\"english_mfa\" \\\n", " --num_cores=20 \\\n", " --manifest_file=\"/home/georg/notebooks/customers/aiera/annotation_manifest.json\" \\\n", " --dataset_name=\"aiera\" \\\n", " --sample_rate=16000 \\\n", " --max_cer_thres=0.6 \\\n", " --skip_verify_boundaries \\\n", " --min_silence_gap_s=0.04 \\\n", " --max_segment_duration_s=10.0 \\\n", " --n_align_success_pad=2 \\\n", " --s3_sample_data_prefix=\"s3://suno-static-public/datasets/tmp/\"" ] }, { "cell_type": "markdown", "id": "b721d110", "metadata": {}, "source": [ "## QA" ] }, { "cell_type": "code", "execution_count": 94, "id": "9b4260c1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.6 hours\n" ] } ], "source": [ "with open(os.path.join(PIPELINE_DIR, \"artifacts/03_segment_meta_post_timestamp.json\")) as f:\n", " segments_manifest = json.load(f)\n", "print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": null, "id": "3ee9828b", "metadata": {}, "outputs": [], "source": [ "62, 52, 136, 201, 49, 156" ] }, { "cell_type": "code", "execution_count": null, "id": "7a5a3fca", "metadata": {}, "outputs": [], "source": [ "# TODO: (norm)\n", "# 52\n", "# subject, uh, to that in a short-term anyways. Uh, and then finally, you know, the Medicare Part D redesign, um, you know, first of all,\n", "# subject uh to that in a short term anyways uh and then finally you know the medicare part five hundred redesign um you know first of all" ] }, { "cell_type": "code", "execution_count": 97, "id": "127e8b77", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "107\n", "our, uh, to our DOI, um, throughout the year. And, uh, as we had to the top-line, obviously, um, we are getting closer to the end of the full\n", "our uh to our doi um throughout the year and uh as we had to the top line obviously um we are getting closer to the end of the full\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "201\n", "on, uh, HEMLIBRA, uh, those that desire a high efficacy are going to consider Efa as well. So, look, we think it's -- it's pretty transformational, don't forget\n", "on uh hemlibra uh those that desire a high efficacy are going to consider efa as well so look we think it's -- it's pretty transformational don't forget\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "49\n", "pricing, uh, for us not a -- a real concern. Uh, we have implemented, uh, an industry-leading, uh, pricing policy which\n", "pricing uh for us not a -- a real concern uh we have implemented uh an industry leading uh pricing policy which\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "81\n", "during the vaccine phase, so I just push everyone back to take a look and see what we -- we said there. So, um, we'll have, uh, results in the next three quarters about that to\n", "during the vaccine phase so i just push everyone back to take a look and see what we -- we said there so um we'll have uh results in the next three quarters about that to\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "156\n", "that FDA had and we hope by the end of the year to have that result so that we can resume those again in the US as well. So, look, we\n", "that f d a had and we hope by the end of the year to have that result so that we can resume those again in the us as well so look we\n", "----------\n" ] } ], "source": [ "l = list(zip(range(len(segments_manifest)), segments_manifest[:]))\n", "random.shuffle(l)\n", "for n, m in l[:5]:\n", " play_audio(m[\"uri\"])\n", " print(n)\n", " print(Tokens.from_dict(m[\"transcript\"][\"tokens\"]).text)\n", " print(Tokens.from_dict(m[\"transcript_normalized\"][\"tokens\"]).text)\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 85, "id": "f1bdebcf", "metadata": {}, "outputs": [], "source": [ "# sort by cer\n", "from suno_utils.utils.metrics import get_cer\n", "\n", "l = segments_manifest[:]\n", "cer_list = []\n", "for m in l:\n", " m[\"cer\"] = get_cer(m[\"transcript_normalized\"][\"text\"], m[\"transcript_asr\"][\"transcript\"])\n", "l = sorted(l, key=lambda x: x[\"cer\"], reverse=True)" ] }, { "cell_type": "code", "execution_count": 86, "id": "bdc9bc66", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1.0\n", "829cae3f-f7e0-4459-aa90-4affdf300314\n", "characterize as the good [inaudible] and the\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "bd0c39f5-aa2c-4340-8a9e-dae230d88ece\n", "years [inaudible] as well, [inaudible] in\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "79debbcd-453e-4f2a-b841-d808538cbe5e\n", "the [inaudible] was\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "ddb90f45-82a3-4fc2-aef2-ba084f5096c2\n", "an update [inaudible] Well, look,\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "500a7cfa-0437-47b4-983b-f5d58f5bedcf\n", "much [inaudible] at\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "78cf8ab6-6b1f-4945-b9da-25d2841afef9\n", "spending [inaudible] Yeah,\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "999b8281-cf1e-4cbf-a9a0-ac991e3d3450\n", "up [inaudible] the fact it\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "df466104-4d4c-40a9-9421-51397d617dcc\n", "year for [inaudible] but [inaudible] of RNB and [inaudible] next year.\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "9826a33e-efdb-475c-a818-4ef0ada8720a\n", "is [inaudible] issues in Q3 near the, you know, [inaudible] how -- how much focus it was,\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "4826df1a-466f-4bb3-a6cf-f21f3793393f\n", "on -- on, uh, a [inaudible] um, like in terms of [inaudible] product?\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n" ] } ], "source": [ "for e in l[:10]:\n", " print(e[\"cer\"])\n", " print(e[\"id\"])\n", " print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n", " Audio.from_file(e[\"uri\"]).play()\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": null, "id": "55387924", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6a5dce21", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "638fe2d5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "01a0250a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "10196a91", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b449a068", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }