{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "b304dfff", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 2, "id": "f955e852", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"" ] }, { "cell_type": "code", "execution_count": 3, "id": "e9716000", "metadata": {}, "outputs": [], "source": [ "import time\n", "import os\n", "import tqdm\n", "import shutil\n", "import random\n", "import funcy\n", "import json\n", "import numpy as np\n", "import multiprocessing\n", "import pandas as pd\n", "from suno_utils.audio import Audio, Tokens, Token\n", "from suno_utils.audio.conversion import play_audio, get_audio_properties\n", "from suno_utils.utils.slicer import get_vad_gaps, get_intervals_from_gaps\n", "\n", "\n", "DATE_DIR = \"2022_09_08\"\n", "\n", "BASE_DIR = \"/mnt/data-ssd-1/data/private/customer/aiera/2022_09_08-sample_call/\"\n", "RAW_AUDIO_DIR = os.path.join(BASE_DIR, \"raw_audio\")\n", "TO_REV_DIR = os.path.join(BASE_DIR, \"to_rev\", DATE_DIR)\n", "FROM_REV_DIR = os.path.join(BASE_DIR, \"from_rev\", DATE_DIR)\n", "TO_REV_AUDIO_DIR = os.path.join(TO_REV_DIR, \"audio\")\n", "REV_MANIFEST_FILEPATH = os.path.join(TO_REV_DIR, \"rev_manifest.jsonl\")\n", "FAILED_MANIFEST_FILEPATH = os.path.join(TO_REV_DIR, \"failed_manifest.jsonl\")\n", "\n", "PIPELINE_DATE_DIR = \"2022_09_08\"\n", "\n", "SEGMENTS_DIR = os.path.join(BASE_DIR, \"pipeline\", PIPELINE_DATE_DIR, \"segments\")\n", "ARTIFACTS_DIR = os.path.join(BASE_DIR, \"pipeline\", PIPELINE_DATE_DIR, \"artifacts\")" ] }, { "cell_type": "code", "execution_count": 4, "id": "e22f6da7", "metadata": {}, "outputs": [], "source": [ "# shutil.rmtree(FROM_REV_DIR, ignore_errors=True)\n", "# shutil.rmtree(TO_REV_DIR, ignore_errors=True)\n", "# shutil.rmtree(TO_REV_AUDIO_DIR, ignore_errors=True)\n", "os.makedirs(FROM_REV_DIR, exist_ok=True)\n", "os.makedirs(TO_REV_DIR, exist_ok=True)\n", "os.makedirs(TO_REV_AUDIO_DIR, exist_ok=True)\n", "os.makedirs(SEGMENTS_DIR, exist_ok=True)\n", "os.makedirs(ARTIFACTS_DIR, exist_ok=True)" ] }, { "cell_type": "markdown", "id": "499ca712", "metadata": {}, "source": [ "## Make rev data" ] }, { "cell_type": "code", "execution_count": 5, "id": "ac737816", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0 failed\n", "1 files - 0.7 hours\n" ] } ], "source": [ "filepaths = [\n", " os.path.join(RAW_AUDIO_DIR, \"sample_audio.mp3\"),\n", "]\n", "\n", "l1 = []\n", "e1 = []\n", "for fp in filepaths:\n", " try:\n", " l1.append(get_audio_properties(fp)[\"duration_s\"])\n", " except:\n", " e1.append(fp)\n", " \n", "random.seed(6006)\n", "random.shuffle(filepaths)\n", " \n", "print(len(e1), \"failed\")\n", "print(len(l1), \"files -\", round(sum(l1) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": 36, "id": "7e83d447", "metadata": {}, "outputs": [], "source": [ "import portion\n", "from suno_utils.utils.numbers import safe_round\n", "from suno_utils.tasks.music import remove_music\n", " \n", "def prep_annotation_audio(audio, clean_audio=False, remove_silence=False, keep_all_lengths=True):\n", " min_silence_gap = 0.05\n", " min_segment_dur_s = 2.0\n", " max_segment_dur_s = 10.0\n", " # TODO: remove background noise?\n", " if clean_audio:\n", " cleaned_audio = remove_music(audio, use_gpu=True)\n", " else:\n", " cleaned_audio = audio\n", " vad_gaps = get_vad_gaps(cleaned_audio, min_gap_s=min_silence_gap, vad_level=3, silent=True)\n", " silence_frac = np.sum([g.upper - g.lower for g in vad_gaps]) / audio.duration_s\n", "# print(\"{}% silence\".format(round(silence_frac * 100, 1)))\n", "\n", " # anything longer than a second should get excluded with pad\n", " silence_intervals = portion.empty()\n", " if remove_silence:\n", " max_silence_s = 1\n", " max_silence_pad = max_silence_s / 2\n", " for vad_gap in vad_gaps:\n", " excluded_lower_s = safe_round(vad_gap.lower + max_silence_pad)\n", " excluded_upper_s = safe_round(vad_gap.upper - max_silence_pad)\n", " if excluded_upper_s - excluded_lower_s > 0:\n", " silence_intervals |= portion.openclosed(excluded_lower_s, excluded_upper_s)\n", " \n", " reliable_speech_segments = get_intervals_from_gaps(\n", " vad_gaps,\n", " excluded_intervals=silence_intervals,\n", " min_dur_s=min_segment_dur_s,\n", " max_dur_s=max_segment_dur_s,\n", " total_duration_s=audio.duration_s,\n", " )\n", "\n", " tot_segment_duration_s = np.sum([end_s - start_s for start_s, end_s in reliable_speech_segments])\n", " cov_frac = tot_segment_duration_s / audio.duration_s\n", " print(\"{}% reliable speech segment coverage\".format(round(cov_frac * 100, 1)))\n", " \n", " # add non-silent speech fragments\n", " speech_segments = reliable_speech_segments[:]\n", " if keep_all_lengths:\n", " speech_segment_interval = portion.empty()\n", " for start_s, end_s in reliable_speech_segments:\n", " speech_segment_interval |= portion.closedopen(start_s, end_s)\n", " non_covered_speech = portion.openclosed(0, audio.duration_s) - silence_intervals - speech_segment_interval\n", " for s in non_covered_speech:\n", " offs_s = safe_round(s.lower)\n", " while True:\n", " end_s = safe_round(np.min([s.upper, offs_s + max_segment_dur_s]))\n", " if end_s - offs_s <= 0:\n", " break\n", " speech_segments.append((offs_s, end_s))\n", " offs_s = end_s\n", " speech_segments = sorted(speech_segments)\n", " tot_segment_duration_s = np.sum([end_s - start_s for start_s, end_s in speech_segments])\n", " cov_frac = tot_segment_duration_s / audio.duration_s\n", " print(\"{}% segment coverage with fragments\".format(round(cov_frac * 100, 1)))\n", " \n", "# tot_segment_duration_mins = int(tot_segment_duration_s // 60)\n", "# tot_segment_duration_secs = int(np.ceil(tot_segment_duration_s % 60))\n", "# print(\"{}m {}s segment audio\".format(tot_segment_duration_mins, tot_segment_duration_secs))\n", "\n", " audios_rev = []\n", " segments_meta = []\n", " for n, (start_s, end_s) in enumerate(speech_segments):\n", "# indicator_str = \"a\" + str(n)\n", " indicator_str = str(n).zfill(2)\n", " audios_indicator = []\n", " for c in indicator_str:\n", " audios_indicator.append(Audio.from_file(f\"../sanas/audio_indicators/{c}_fast.wav\"))\n", " audios_rev.extend(audios_indicator)\n", " segments_meta.append({\n", " \"segment_number\": n,\n", " \"type\": \"indicator\",\n", " \"duration_s\": safe_round(np.sum([a.duration_s for a in audios_indicator])),\n", " \"indicator_str\": indicator_str,\n", " })\n", " audios_rev.append(audio.get_slice(start_s, end_s))\n", " segments_meta.append({\n", " \"segment_number\": n,\n", " \"type\": \"speech\",\n", " \"duration_s\": safe_round(end_s - start_s),\n", " \"orginial_audio_offset_s\": start_s,\n", " })\n", " \n", " if len(audios_rev) == 0:\n", " return None, {\n", " \"tot_silence_frac\": round(silence_frac, 3),\n", " \"segment_cov_frac\": round(cov_frac, 3),\n", " }\n", "\n", " if not keep_all_lengths and (\n", " np.min([e[\"duration_s\"] for e in segments_meta if e[\"type\"] == \"speech\"]) < min_segment_dur_s or\n", " np.max([e[\"duration_s\"] for e in segments_meta if e[\"type\"] == \"speech\"]) > max_segment_dur_s\n", " ):\n", " raise ValueError(\"incorrect segment length detected\")\n", " \n", " audio_rev = Audio.concatenate(audios_rev)\n", " stitch_overhead_frac = (audio_rev.duration_s - tot_segment_duration_s) / audio_rev.duration_s\n", "# print(\"{}% stitching overhead\".format(round(stitch_overhead_frac * 100, 1)))\n", " submission_duration_s = np.ceil(audio_rev.duration_s / 60) * 60\n", " tot_overhead_frac = (submission_duration_s - tot_segment_duration_s) / submission_duration_s\n", "# print(\"{}% total overhead\".format(round(tot_overhead_frac * 100, 1)))\n", "\n", " submission_meta = {\n", " \"tot_silence_frac\": round(silence_frac, 3),\n", " \"segment_cov_frac\": round(cov_frac, 3),\n", " \"stitch_overhead_frac\": round(stitch_overhead_frac, 3),\n", " \"tot_overhead_frac\": round(tot_overhead_frac, 3),\n", " \"segments_meta\": segments_meta,\n", " }\n", " # TODO: alert on mostly silence\n", " # TODO: alert on total time too short \n", " # TODO: alert on coverage \n", " # TODO: alert stitching overhead\n", " # TODO: alert rev rounding overhead\n", " return audio_rev, submission_meta" ] }, { "cell_type": "code", "execution_count": 26, "id": "c7f5de1e", "metadata": {}, "outputs": [], "source": [ "# filepath = filepaths[5]\n", "\n", "# audio = Audio.from_file(filepath, sample_rate=16_000, byte_width=2)\n", "# audio_rev, submission_meta = prep_annotation_audio(audio)\n", "# audio_rev.play()" ] }, { "cell_type": "code", "execution_count": 27, "id": "9b27da8b", "metadata": {}, "outputs": [], "source": [ "# !rm -rf /mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/to_rev/*" ] }, { "cell_type": "code", "execution_count": 37, "id": "96d0f32a", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\r", " 0%| | 0/1 [00:00 1.5 or rough_efficiency < 0.5:\n", " print(f\"weird coverage in {uid}\")\n", " \n", " metadata.append({\n", " \"uid\": uid,\n", " \"original_audio_filepath\": filepath,\n", " \"segments_meta\": submission_meta[\"segments_meta\"],\n", " \"audio_duration_s\": audio_rev.duration_s,\n", " \"audio_filepath\": rev_filepath,\n", " \"hotwords\": hotwords,\n", " \"misc_meta\": {\n", " \"tot_silence_frac\": submission_meta[\"tot_silence_frac\"],\n", " \"segment_cov_frac\": submission_meta[\"segment_cov_frac\"],\n", " \"stitch_overhead_frac\": submission_meta[\"stitch_overhead_frac\"],\n", " \"tot_overhead_frac\": submission_meta[\"tot_overhead_frac\"],\n", " }\n", " })\n", "\n", " if n % 100 == 0:\n", " with open(REV_MANIFEST_FILEPATH, \"w\") as f:\n", " for meta in metadata:\n", " f.write(json.dumps(meta) + \"\\n\")\n", " with open(FAILED_MANIFEST_FILEPATH, \"w\") as f:\n", " for meta in failed_metadata:\n", " f.write(json.dumps(meta) + \"\\n\")\n", " \n", "with open(REV_MANIFEST_FILEPATH, \"w\") as f:\n", " for meta in metadata:\n", " f.write(json.dumps(meta) + \"\\n\")\n", "with open(FAILED_MANIFEST_FILEPATH, \"w\") as f:\n", " for meta in failed_metadata:\n", " f.write(json.dumps(meta) + \"\\n\")\n", "print(\"{}/{} files excluded\".format(len(failed_metadata), len(failed_metadata) + len(metadata)))" ] }, { "cell_type": "code", "execution_count": 64, "id": "73860040", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1 files total\n", "0.7h total\n", "0.7h rev total\n", "0.7h segment total\n", "100.0% segment coverage\n" ] } ], "source": [ "tot_duration_s = [\n", " safe_round(get_audio_properties(m[\"original_audio_filepath\"], attempt_using_header=False)[\"duration_s\"]) \n", " for m in metadata\n", "]\n", "tot_rev_duration_s = [m[\"audio_duration_s\"] for m in metadata]\n", "tot_seg_duration_s = 0\n", "for m in metadata:\n", " for e in m[\"segments_meta\"]:\n", " if e[\"type\"] == \"speech\":\n", " tot_seg_duration_s += e[\"duration_s\"]\n", "print(len(metadata), \"files total\")\n", "print(\"{}h total\".format(round(sum(tot_duration_s) / 60 / 60, 1)))\n", "print(\"{}h rev total\".format(round(sum(tot_rev_duration_s) / 60 / 60, 1)))\n", "print(\"{}h segment total\".format(round(tot_seg_duration_s / 60 / 60, 1)))\n", "print(\"{}% segment coverage\".format(round(sum(tot_seg_duration_s) / sum(tot_duration_s) * 100, 1)))" ] }, { "cell_type": "code", "execution_count": null, "id": "b5ca4d24", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "588f5904", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5172e5ef", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "149b3b8e", "metadata": {}, "source": [ "## (optional) Investigate for mistakes" ] }, { "cell_type": "code", "execution_count": 65, "id": "143da265", "metadata": {}, "outputs": [], "source": [ "metadata = []\n", "with open(REV_MANIFEST_FILEPATH) as f:\n", " for l in f.read().strip().split(\"\\n\"):\n", " metadata.append(json.loads(l))" ] }, { "cell_type": "code", "execution_count": 66, "id": "9fe48a0d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "play_audio(metadata[0][\"audio_filepath\"])" ] }, { "cell_type": "code", "execution_count": null, "id": "1b79d2e8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1b3ca219", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6c91afe9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "0868d256", "metadata": {}, "source": [ "## (Optional) Check price" ] }, { "cell_type": "code", "execution_count": 67, "id": "37decb32", "metadata": {}, "outputs": [], "source": [ "# suno_price_per_hour = 72\n", "# suno_price_per_min = suno_price_per_hour / 60\n", "\n", "# sanas_price = 162 # dollars_per_hour\n", "# sanas_price_per_s = sanas_price / 3600 # dollars per s\n", "\n", "\n", "# costs = []\n", "# for rm in metadata:\n", "# sanas_duration_s = np.sum([sm[\"duration_s\"] for sm in rm[\"segments_meta\"] if sm[\"type\"] == \"speech\"])\n", "# sanas_cost = sanas_price_per_s * sanas_duration_s\n", " \n", "# suno_cost = np.ceil(rm[\"audio_duration_s\"] / 60.) * suno_price_per_min\n", "# costs.append({\"suno\": suno_cost, \"sanas\": sanas_cost})\n", "\n", "# print(len(metadata), \"files total\")\n", "# costs = pd.DataFrame(costs)\n", "# # suno 2770.8\n", "# # sanas 4715.9\n", "# costs.sum().round(1)" ] }, { "cell_type": "code", "execution_count": null, "id": "def684e9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fcf743da", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "88a9e9cc", "metadata": {}, "source": [ "## Send stuff to rev" ] }, { "cell_type": "code", "execution_count": 68, "id": "613bd82e", "metadata": {}, "outputs": [], "source": [ "ORDER_REF_STR = \"ar_call\"\n", "GLOBAL_HOTWORDS = []" ] }, { "cell_type": "code", "execution_count": 70, "id": "1a576034", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1 files total\n", "0.7h total\n" ] } ], "source": [ "metadata = []\n", "with open(REV_MANIFEST_FILEPATH) as f:\n", " for l in f.read().strip().split(\"\\n\"):\n", " metadata.append(json.loads(l))\n", "\n", "tot_duration_s = [m[\"audio_duration_s\"] for m in metadata]\n", "print(len(metadata), \"files total\")\n", "print(\"{}h total\".format(round(sum(tot_duration_s) / 60 / 60, 1)))" ] }, { "cell_type": "markdown", "id": "b16a0dd0", "metadata": {}, "source": [ "### 01 - upload items" ] }, { "cell_type": "code", "execution_count": 71, "id": "2453089e", "metadata": {}, "outputs": [], "source": [ "import json\n", "from suno_utils.web.rev import get_auth_string, upload_file\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 73, "id": "8c66f7f9", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:03<00:00, 3.99s/it]\n" ] } ], "source": [ "import tqdm\n", "\n", "order_items = []\n", "for audio_meta in tqdm.tqdm(metadata):\n", " media_loc = upload_file(AUTH_STR, audio_meta[\"audio_filepath\"], file_ref_str=audio_meta[\"uid\"])\n", " order_items.append({\n", " \"media_loc\": media_loc,\n", " \"hotwords\": audio_meta[\"hotwords\"] + GLOBAL_HOTWORDS,\n", "# \"accents\": [\"Australian\"],\n", " \"duration_s\": int(round(audio_meta[\"audio_duration_s\"])),\n", " \"uuid\": audio_meta[\"uid\"],\n", " })" ] }, { "cell_type": "code", "execution_count": 74, "id": "a0ee5f4f", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, \"01_order_items.json\"), \"w\") as f:\n", " json.dump(order_items, f)" ] }, { "cell_type": "markdown", "id": "9131358e", "metadata": {}, "source": [ "### 02 - make order" ] }, { "cell_type": "code", "execution_count": 75, "id": "7ccf6e0a", "metadata": {}, "outputs": [], "source": [ "import json\n", "from suno_utils.web.rev import get_auth_string, make_order\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 76, "id": "0b300ef5", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, \"01_order_items.json\"), \"r\") as f:\n", " order_items = json.load(f)" ] }, { "cell_type": "code", "execution_count": 78, "id": "8318e46f", "metadata": {}, "outputs": [], "source": [ "import funcy\n", "order_numbers = []\n", "for n, items_chunk in enumerate(funcy.chunks(500, order_items)):\n", " order_number = make_order(AUTH_STR, items_chunk, order_ref_str=ORDER_REF_STR + f\"_{n}\", add_rush=False)\n", " order_numbers.append(order_number)" ] }, { "cell_type": "code", "execution_count": 79, "id": "27549d51", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, \"02_order_numer.txt\"), \"w\") as f:\n", " f.write(\"\\n\".join(order_numbers))" ] }, { "cell_type": "code", "execution_count": null, "id": "5531cb02", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8bfcc813", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "f30acc60", "metadata": {}, "source": [ "### 04 - pull transcripts" ] }, { "cell_type": "code", "execution_count": 80, "id": "65101ca4", "metadata": {}, "outputs": [], "source": [ "import json\n", "from suno_utils.web.rev import get_auth_string, get_finished_order, get_transcript\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 81, "id": "de613d78", "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", "with open(os.path.join(TO_REV_DIR, \"02_order_numer.txt\")) as f:\n", " order_numbers = f.read().split(\"\\n\")" ] }, { "cell_type": "code", "execution_count": 82, "id": "9319c7f3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1 transcripts\n" ] } ], "source": [ "# # get intermediate results\n", "# from suno_utils.web.rev import _get_order_details, _parse_order_response\n", "# transcript_metas = []\n", "# for order_number in order_numbers:\n", "# order_details = _get_order_details(AUTH_STR, order_number)\n", "# order_status, transcript_metas_chunk = _parse_order_response(order_details)\n", "# transcript_metas.extend(transcript_metas_chunk)\n", "# print(len(transcript_metas), \"transcripts\")" ] }, { "cell_type": "code", "execution_count": 83, "id": "58ae9a27", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1 transcripts\n" ] } ], "source": [ "transcript_metas = []\n", "for order_number in order_numbers:\n", " transcript_metas.extend(get_finished_order(AUTH_STR, order_number, blocking=True))\n", "print(len(transcript_metas), \"transcripts\")" ] }, { "cell_type": "code", "execution_count": 90, "id": "71cd19bb", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:02<00:00, 2.16s/it]\n" ] } ], "source": [ "for transcript_meta in tqdm.tqdm(transcript_metas):\n", " transcript_filepath = os.path.join(FROM_REV_DIR, f\"{transcript_meta['uuid']}.json\")\n", " if os.path.exists(transcript_filepath):\n", " continue\n", " transcript_json = get_transcript(AUTH_STR, transcript_meta)\n", " with open(transcript_filepath, \"w\") as f:\n", " json.dump(transcript_json, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "2b60d3a3", "metadata": {}, "outputs": [], "source": [ " \"id\": \"some id\",\n", " \"original_audio_filepath\": \"/path/to/original/audio.wav\",\n", " \"transcript_filepath\": \"/path/to/transcript.json\"," ] }, { "cell_type": "code", "execution_count": 113, "id": "3ebcec61", "metadata": {}, "outputs": [], "source": [ "manifest_filepath = os.path.join(ARTIFACTS_DIR , \"00_manifest.json\")\n", "manifest = [\n", " {\n", " \"id\": \"acd80c3e-34ce-45e3-8a1c-acf870085691\",\n", "# \"audio_filepath\": os.path.join(RAW_AUDIO_DIR, \"sample_audio.mp3\"),\n", " \"original_audio_filepath\": metadata[0][\"original_audio_filepath\"],\n", " \"transcript_filepath\": os.path.join(FROM_REV_DIR, f\"acd80c3e-34ce-45e3-8a1c-acf870085691.json\"),\n", " \"segments_meta\": metadata[0][\"segments_meta\"],\n", " },\n", "]\n", "with open(manifest_filepath, \"w\") as f:\n", " json.dump(manifest, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "7108c966", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "29df662a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "c0f04677", "metadata": {}, "source": [ "## Run sanas pipeline" ] }, { "cell_type": "code", "execution_count": 201, "id": "3d5ba9f3", "metadata": {}, "outputs": [], "source": [ "# make sure to delete segments when rerunning\n", "shutil.rmtree(\"/mnt/data-ssd-1/data/private/customer/aiera/2022_09_08-sample_call/pipeline/2022_09_08\", ignore_errors=True)\n", "shutil.rmtree(\"/home/georg/notebooks/customers/aiera/tmp_delivery\", ignore_errors=True)\n", "os.makedirs(\"/mnt/data-ssd-1/data/private/customer/aiera/2022_09_08-sample_call/pipeline/2022_09_08\")\n", "os.makedirs(\"/home/georg/notebooks/customers/aiera/tmp_delivery\")" ] }, { "cell_type": "code", "execution_count": null, "id": "d94692e8", "metadata": {}, "outputs": [], "source": [ "CUDA_VISIBLE_DEVICES=0 python -m suno_utils.scripts.customer.sanas.run \\\n", " --output_dir=\"/mnt/data-ssd-1/data/private/customer/aiera/2022_09_08-sample_call/pipeline/2022_09_08\" \\\n", " --use_gpu \\\n", " --conda-env=\"mfa\" \\\n", " --conda_sh_filepath=\"/home/georg/anaconda3/etc/profile.d/conda.sh\" \\\n", " --mfa_dictionary_name=\"english_mfa\" \\\n", " --mfa_acoustic_model_name=\"english_mfa\" \\\n", " --num_cores=20 \\\n", " --manifest_file=\"/home/georg/notebooks/customers/aiera/00_manifest.json\" \\\n", " --data_delivery_dir=\"/home/georg/notebooks/customers/aiera/tmp_delivery\" \\\n", " --sample_rate=16000 \\\n", " --max_cer_thres=0.6 \\\n", " --skip_verify_boundaries\n", "\n", "# --min_silence_gap_s=0.04 \\\n", "# --max_segment_duration_s=10.0 \\\n", "# --n_align_success_pad=2 \\\n", "# --s3_sample_data_prefix=\"s3://suno-static-public/datasets/tmp/\"" ] }, { "cell_type": "code", "execution_count": null, "id": "3c5f1084", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1e5e76ea", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "13f1f39c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b721d110", "metadata": {}, "source": [ "## QA" ] }, { "cell_type": "code", "execution_count": null, "id": "65ada2a2", "metadata": {}, "outputs": [], "source": [ "# TODO: partial words: the ca-, the, the, the" ] }, { "cell_type": "code", "execution_count": 202, "id": "9b4260c1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.7 hours\n" ] } ], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"03_segment_meta_post_timestamp.json\")) as f:\n", " segments_manifest = json.load(f)\n", "print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": 245, "id": "80a03cf2", "metadata": {}, "outputs": [], "source": [ "# for m in segments_manifest:\n", "# if m[\"ignore_segment\"]:\n", "# print(m[\"ignore_reason\"])" ] }, { "cell_type": "code", "execution_count": 203, "id": "0351726a", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "114\n", "And then in terms of -- you've [inaudible] to the BOI margin this year, but when will you get the real benefits in the manufacturing chain, uh, process, process chain? Is that more next year?\n", "and then in terms of -- you've [inaudible] to the b o i margin this year but when will you get the real benefits in the manufacturing chain uh process process chain is that more next year\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "57\n", "Uh, pricing policy, which was, uh, pegging, uh, any price increase [inaudible] around medical inflation, so, uh, we don't, uh, have any, uh, real risk there.\n", "uh pricing policy which was uh pegging uh any price increase [inaudible] around medical inflation so uh we don't uh have any uh real risk there\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "265\n", "\n", "\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "153\n", "We are, uh, all the ca-, the, the, the very few cases of delay that we had were, you know --\n", "we are uh all the ca the the the very few cases of delay that we had were you know --\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "58\n", "Um, as it is, uh, with, uh, the direct Medicare negotiations, as you said, Dupixent certainly, uh, far down the road. We don't have --\n", "um as it is uh with uh the direct medicare negotiations as you said dupixent certainly uh far down the road we don't have --\n", "----------\n" ] } ], "source": [ "l = list(zip(range(len(segments_manifest)), segments_manifest[:]))\n", "random.shuffle(l)\n", "for n, m in l[:5]:\n", " play_audio(m[\"uri\"])\n", " print(n)\n", " print(Tokens.from_dict(m[\"transcript\"][\"tokens\"]).text)\n", " print(Tokens.from_dict(m[\"transcript_normalized\"][\"tokens\"]).text)\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 204, "id": "f1bdebcf", "metadata": {}, "outputs": [], "source": [ "# sort by cer\n", "from suno_utils.utils.metrics import get_cer\n", "\n", "l = segments_manifest[:]\n", "cer_list = []\n", "for m in l:\n", " m[\"cer\"] = get_cer(m[\"transcript_normalized\"][\"text\"], m[\"transcript_asr\"][\"transcript\"])\n", "l = sorted(l, key=lambda x: x[\"cer\"], reverse=True)" ] }, { "cell_type": "code", "execution_count": 205, "id": "3b03fd5a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1.0\n", "74554e6c-e2e3-4ff2-bb9a-fd6e760f4318\n", "[inaudible] I suppose I'm concerned that the FDA doesn't allow you to accelerate --\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "0.875\n", "49e00b7d-f42a-4896-8d3c-7b0761288410\n", "If you had to be sort of, [inaudible] out there, [inaudible] GI profile, than you had with [inaudible] What does that look like [inaudible]\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "0.8695652173913043\n", "22116e9b-527e-42c1-a774-439a0c8d4552\n", "[inaudible] I think it is [inaudible] Thank you.\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "0.8018867924528302\n", "00c80e2b-3129-4771-82f9-7d89a27d3fab\n", "Any latest, we'll hear -- it seems like you're hearing or, or [inaudible] uh, the impact or not they're having on the, on the [inaudible]\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "0.674074074074074\n", "ce83a48a-c3bd-4247-9a95-73d345d6601f\n", "I'm gonna thank Bill [inaudible] for their time. [inaudible] so, Bill, thanks very much for attending. Thank you for having me. [inaudible] thanks very much. Thank you. Thank you.\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "0.6454545454545455\n", "a15f0717-6148-469e-8807-f41439ca73a3\n", "Um, [inaudible] a billion [inaudible] product. I realize you're not gonna give pre-sale guidance, but talk, talk to the [inaudible] disease as a whole --\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "0.6216216216216216\n", "9644b63d-caee-49c8-aedc-150c08009db1\n", "As -- you know, make sure there are no hands up. No. Okay. Uh, can we move to multiple sclerosis, please? The, the Aubagio [inaudible]\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "0.6153846153846154\n", "4f3a7e9e-e072-4fc2-a284-9b18b881902b\n", "Very much BOI margin [inaudible] and, you know, you told me a story about you smashing the margins, you've been investing in the business [inaudible] through the pipeline.\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "0.5797101449275363\n", "e154f2c9-2058-432f-a8fb-60bc1c23cc03\n", "You know, uh, we're working on a [inaudible] data driven, uh, here. [inaudible] is what, uh, is what I'd say. All right. Okay. Well, let's get into your wheelhouse. Let's get [inaudible]\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "0.5652173913043478\n", "6bd26be0-4f2e-42a0-8b91-36e560d952b3\n", "But at the same time, Sanofi's not getting any credit or benefit because [inaudible] So anything you can say as to when you might see data points there?\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n" ] } ], "source": [ "for e in l[:10]:\n", " print(e[\"cer\"])\n", " print(e[\"id\"])\n", " print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n", " Audio.from_file(e[\"uri\"]).play()\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": null, "id": "28bcdd8c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "32c3f559", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "ae98f9e9", "metadata": {}, "source": [ "## make long form and deliverable" ] }, { "cell_type": "code", "execution_count": 211, "id": "e63897bc", "metadata": {}, "outputs": [], "source": [ "DELIVERY_DIR = \"/home/georg/notebooks/customers/aiera/tmp_delivery\"\n", "DELIVERY_SEGMENTS_DIR = os.path.join(DELIVERY_DIR, \"segments\")\n", "\n", "shutil.rmtree(DELIVERY_DIR, ignore_errors=True)\n", "os.makedirs(DELIVERY_DIR)\n", "os.makedirs(DELIVERY_SEGMENTS_DIR)" ] }, { "cell_type": "code", "execution_count": 210, "id": "91c17585", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.7 hours\n" ] } ], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"03_segment_meta_post_timestamp.json\")) as f:\n", " segments_manifest = json.load(f)\n", "print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": 240, "id": "83d33b88", "metadata": {}, "outputs": [], "source": [ "speakers_map = {\n", " \"Speaker 1\": \"Pete\",\n", " \"Speaker 2\": \"Bill Sibold\",\n", "}\n", "\n", "# compile long form pretty print json\n", "full_transcript_tokens = []\n", "for m in segments_manifest:\n", " full_transcript_tokens.extend(m[\"transcript\"][\"tokens\"])\n", "full_transcript_tokens = Tokens([Token.from_dict(d).remove_timestamps() for d in full_transcript_tokens])\n", "full_transcript = []\n", "for d in full_transcript_tokens.speaker_turns:\n", " text = d[\"text\"]\n", " text = re.sub(r\"-- [A-Z][a-z]\", lambda m: m.group(0).lower(), text)\n", " full_transcript.append({\n", " \"speaker_id\": speakers_map[d[\"speaker_id\"]],\n", " \"text\": text,\n", " })\n", "with open(os.path.join(DELIVERY_DIR, \"fulltext.json\"), \"w\") as f:\n", " json.dump(full_transcript, f, indent=2)\n", " \n", "# make output segments meta and write segments\n", "output_segments = []\n", "for m in segments_manifest:\n", " new_uri = os.path.join(DELIVERY_SEGMENTS_DIR, m[\"id\"] + \".wav\")\n", " shutil.copy(m[\"uri\"], new_uri)\n", " transcript = {\n", " \"tokens\": [\n", " Token.from_dict(d).add_speaker_id(speakers_map[d[\"speaker_id\"]]).as_dict() \n", " for d in m[\"transcript\"][\"tokens\"]\n", " ],\n", " \"text\": m[\"transcript\"][\"text\"],\n", " }\n", " transcript_normalized = {\n", " \"tokens\": [\n", " Token.from_dict(d).add_speaker_id(speakers_map[d[\"speaker_id\"]]).as_dict() \n", " for d in m[\"transcript_normalized\"][\"tokens\"]\n", " ],\n", " \"text\": m[\"transcript_normalized\"][\"text\"],\n", " }\n", " output_segments.append({\n", " \"id\": m[\"id\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " \"uri\": os.path.join(\"segments\", m[\"id\"] + \".wav\"),\n", " \"transcript\": transcript,\n", " \"transcript_normalized\": transcript_normalized,\n", " })\n", "with open(os.path.join(DELIVERY_DIR, \"metadata.jsonl\"), \"w\") as f:\n", " for m in output_segments:\n", " f.write(json.dumps(m) + \"\\n\")" ] }, { "cell_type": "code", "execution_count": 242, "id": "826e0b06", "metadata": {}, "outputs": [], "source": [ "# !cat /home/georg/notebooks/customers/aiera/tmp_delivery/fulltext.json" ] }, { "cell_type": "code", "execution_count": 247, "id": "3464757f", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'/home/georg/notebooks/customers/aiera/tmp_delivery'" ] }, "execution_count": 247, "metadata": {}, "output_type": "execute_result" } ], "source": [ "DELIVERY_DIR" ] }, { "cell_type": "code", "execution_count": 248, "id": "62196513", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "fulltext.json segments segments_manifest.jsonl\r\n" ] } ], "source": [ "!ls /home/georg/notebooks/customers/aiera/tmp_delivery" ] }, { "cell_type": "code", "execution_count": 262, "id": "62e8968d", "metadata": {}, "outputs": [], "source": [ "!cp -R /home/georg/notebooks/customers/aiera/tmp_delivery /home/georg/notebooks/customers/aiera/tmp/sample_data" ] }, { "cell_type": "code", "execution_count": 263, "id": "06da3342", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "sample_data sample_data.tar.gz\r\n" ] } ], "source": [ "!ls /home/georg/notebooks/customers/aiera/tmp/sample_data.tar.gz" ] }, { "cell_type": "code", "execution_count": null, "id": "01aa5d2d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c52c6621", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "26d22b4c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6a6753d9", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }