{ "cells": [ { "cell_type": "code", "execution_count": 120, "id": "bf70f34f", "metadata": {}, "outputs": [], "source": [ "import os\n", "import re\n", "import json\n", "import tqdm\n", "import uuid\n", "import random\n", "import funcy\n", "import copy\n", "import shutil\n", "import numpy as np\n", "from suno_utils.audio import Audio, Token, Tokens\n", "from suno_utils.audio.conversion import get_audio_properties, play_audio\n", "from suno_utils.utils.numbers import safe_round\n", "from suno_utils.utils.text import normalize_whitespace\n", "from suno_utils.web.harvest import get_file_ext, get_filename\n", "\n", "SAMPLE_RATE = 16_000\n", "\n", "BASE_DIR = \"/mnt/data-ssd-1/data/private/customer/ring_central/2022-10-13-poc/\"\n", "RAW_DATA_DIR = os.path.join(BASE_DIR, \"raw_data\")\n", "RAW_AUDIO_DIR = os.path.join(RAW_DATA_DIR, \"audio\")\n", "\n", "AUDIO_DIR = os.path.join(BASE_DIR, \"audio\")\n", "TO_REV_DIR = os.path.join(BASE_DIR, \"to_rev\")\n", "TO_REV_AUDIO_DIR = os.path.join(TO_REV_DIR, \"audio\")\n", "FROM_REV_DIR = os.path.join(BASE_DIR, \"from_rev\")\n", "\n", "SEGMENT_DATE_DIR = \"2022_10_13\"\n", "SEGMENTS_DIR = os.path.join(BASE_DIR, \"pipeline\", SEGMENT_DATE_DIR, \"segments\")\n", "ARTIFACTS_DIR = os.path.join(BASE_DIR, \"pipeline\", SEGMENT_DATE_DIR, \"artifacts\")" ] }, { "cell_type": "code", "execution_count": 123, "id": "c769f21a", "metadata": {}, "outputs": [], "source": [ "os.makedirs(AUDIO_DIR, exist_ok=True)\n", "os.makedirs(TO_REV_DIR, exist_ok=True)\n", "os.makedirs(TO_REV_AUDIO_DIR, exist_ok=True)\n", "os.makedirs(FROM_REV_DIR, exist_ok=True)\n", "os.makedirs(SEGMENTS_DIR, exist_ok=True)\n", "os.makedirs(ARTIFACTS_DIR, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": 12, "id": "c3cd79ed", "metadata": {}, "outputs": [], "source": [ "GLOBAL_HOTWORDS = []" ] }, { "cell_type": "markdown", "id": "3bf11bab", "metadata": {}, "source": [ "## Prepare rev data" ] }, { "cell_type": "code", "execution_count": 13, "id": "4653582f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1426 files\n" ] } ], "source": [ "# some basic checks (use filename UID if available)\n", "uids = []\n", "for fn in os.listdir(RAW_AUDIO_DIR):\n", " if get_file_ext(fn) != \"wav\":\n", " print(\"found unknown file:\", fn)\n", " continue\n", " uid = get_filename(fn)\n", " assert(len(uid) == 36)\n", " uids.append(uid)\n", "assert(len(uids) == len(set(uids)))\n", "print(len(uids), \"files\")" ] }, { "cell_type": "code", "execution_count": 126, "id": "9ff97b1c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 1426/1426 [00:00<00:00, 4122.29it/s]\n" ] } ], "source": [ "# save in dir with known sample rate as wavs\n", "filepaths_info = []\n", "for fn in tqdm.tqdm(os.listdir(RAW_AUDIO_DIR)):\n", " uid = get_filename(fn)\n", " from_fp = os.path.join(RAW_AUDIO_DIR, fn)\n", " to_fp = os.path.join(RAW_AUDIO_DIR, fn)\n", " audio = Audio.from_file(os.path.join(RAW_AUDIO_DIR, fn), sample_rate=SAMPLE_RATE, byte_width=2)\n", " to_filepath = os.path.join(AUDIO_DIR, uid + \".wav\")\n", " audio.to_wav(to_filepath)\n", " filepaths_info.append((uid, from_fp, to_filepath))" ] }, { "cell_type": "code", "execution_count": 131, "id": "deb0bebc", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(BASE_DIR, \"raw_data_manifest.json\"), \"w\") as f:\n", " json.dump(filepaths_info, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "f1d8b3b8", "metadata": {}, "outputs": [], "source": [ "# TODO: do we want to normalize the volume?\n", "# from pydub import AudioSegment, effects \n", "\n", "# audio_seg = AudioSegment(\n", "# data=audio.array, \n", "# frame_rate=audio.sample_rate, \n", "# sample_width=audio.byte_width, \n", "# channels=audio.n_channels,\n", "# )\n", "# norm_seg = effects.normalize(audio_seg) \n", "# norm_audio = Audio(norm_seg.raw_data, 16_000, 2, 1)" ] }, { "cell_type": "code", "execution_count": 97, "id": "4da818da", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "45.1 mins of audio (16.6% overhead)\n", "45.1 mins of audio (17.5% overhead)\n", "45.0 mins of audio (17.7% overhead)\n", "45.0 mins of audio (18.2% overhead)\n", "20.6 mins of audio (19.1% overhead)\n" ] } ], "source": [ "TARGET_REV_FILE_DURATION_S = 45 * 60\n", "INDICATOR_AUDIO_DIR = \"/mnt/data-ssd-1/data/custom/segment_indicators\"\n", "SILENCE_DURATION_S = 0.25\n", "\n", "audio_rev_metas = []\n", "tmp_segments_audio = []\n", "tmp_segments_meta = []\n", "tmp_n = 0\n", "for n_step, (uid, _, filepath) in enumerate(filepaths_info):\n", " # add indicator\n", " indicator_str = str(tmp_n).zfill(3)\n", " tmp_indicator_audios = []\n", " for c in indicator_str:\n", " tmp_indicator_audios.append(\n", " Audio.from_file(\n", " os.path.join(INDICATOR_AUDIO_DIR, f\"{c}_fast.wav\"), \n", " sample_rate=SAMPLE_RATE, \n", " byte_width=2,\n", " )\n", " )\n", " indicator_audio = Audio.concatenate(tmp_indicator_audios)\n", " tmp_segments_audio.append(indicator_audio)\n", " tmp_segments_meta.append({\n", " \"segment_number\": tmp_n,\n", " \"type\": \"indicator\",\n", " \"duration_s\": indicator_audio.duration_s,\n", " \"indicator_str\": indicator_str,\n", " })\n", " # add silence\n", " if SILENCE_DURATION_S > 0:\n", " silence_audio = Audio.from_array(\n", " np.zeros(int(SAMPLE_RATE * SILENCE_DURATION_S), dtype=np.int16), SAMPLE_RATE\n", " )\n", " tmp_segments_audio.append(silence_audio)\n", " tmp_segments_meta.append({\n", " \"segment_number\": tmp_n,\n", " \"type\": \"silence\",\n", " \"duration_s\": SILENCE_DURATION_S,\n", " })\n", " # add audio\n", " audio = Audio.from_file(filepath, sample_rate=SAMPLE_RATE, byte_width=2)\n", "# audio = remove_music(audio, wet=1.0, model_name=\"mdx_extra_q\")\n", " tmp_segments_audio.append(audio)\n", " tmp_segments_meta.append({\n", " \"segment_number\": tmp_n,\n", " \"type\": \"speech\",\n", " \"duration_s\": audio.duration_s,\n", " \"original_audio_filepath\": os.path.join(AUDIO_DIR, uid + \".wav\"),\n", " \"original_audio_id\": uid,\n", " })\n", " # add silence\n", " if SILENCE_DURATION_S > 0:\n", " silence_audio = Audio.from_array(\n", " np.zeros(int(SAMPLE_RATE * SILENCE_DURATION_S), dtype=np.int16), SAMPLE_RATE\n", " )\n", " tmp_segments_audio.append(silence_audio)\n", " tmp_segments_meta.append({\n", " \"segment_number\": tmp_n,\n", " \"type\": \"silence\",\n", " \"duration_s\": SILENCE_DURATION_S,\n", " })\n", " # prep for next step and decide if we make new file\n", " tmp_n += 1\n", " running_duration_s = np.sum([e[\"duration_s\"] for e in tmp_segments_meta])\n", " if (\n", " (running_duration_s >= TARGET_REV_FILE_DURATION_S) or \n", " (n_step == len(filepaths_info) - 1 and running_duration_s > 0)\n", " ):\n", " audio_rev = Audio.concatenate(tmp_segments_audio)\n", " running_speech_duration_s = np.sum([e[\"duration_s\"] for e in tmp_segments_meta if e[\"type\"] == \"speech\"])\n", " print(\"{} mins of audio ({}% overhead)\".format(\n", " round(running_duration_s / 60, 1),\n", " round((1 - running_speech_duration_s / running_duration_s) * 100, 1)\n", " )) \n", " uid = str(uuid.uuid4())\n", " rev_audio_filepath = os.path.join(TO_REV_AUDIO_DIR, f\"{uid}.mp3\")\n", " audio_rev.to_mp3(rev_audio_filepath)\n", " audio_rev_metas.append({\n", " \"uid\": uid,\n", " \"audio_filepath\": rev_audio_filepath,\n", " \"segments_meta\": tmp_segments_meta,\n", " \"duration_s\": audio_rev.duration_s,\n", " })\n", " tmp_segments_audio = []\n", " tmp_segments_meta = []\n", " tmp_n = 0" ] }, { "cell_type": "code", "execution_count": 141, "id": "4d16d48e", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\"), \"w\") as f:\n", " json.dump(audio_rev_metas, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "a5f41c23", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a5dea8f6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "5fc18307", "metadata": {}, "source": [ "## (optional) investigate" ] }, { "cell_type": "code", "execution_count": 19, "id": "dc409c9d", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", " audio_rev_metas = json.load(f)" ] }, { "cell_type": "code", "execution_count": 21, "id": "3604b3cb", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "Audio.from_file(audio_rev_metas[0][\"audio_filepath\"]).play()" ] }, { "cell_type": "code", "execution_count": 22, "id": "40f5c4a2", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "Audio.from_file(audio_rev_metas[-1][\"audio_filepath\"]).play()" ] }, { "cell_type": "code", "execution_count": null, "id": "869c1000", "metadata": {}, "outputs": [], "source": [ "# Questions\n", "# very quiet --> background speech or try to transcribe?\n", "# long form audio? helps with background and context?" ] }, { "cell_type": "code", "execution_count": null, "id": "92a8e1ba", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "491772f5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "426ba2c3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "dfc212cd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "88a9e9cc", "metadata": {}, "source": [ "## Send stuff to rev" ] }, { "cell_type": "code", "execution_count": 23, "id": "05fbe672", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.rev import get_auth_string, upload_file, make_order, make_cc_order\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 24, "id": "2db71de1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3.3 hours total\n" ] } ], "source": [ "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", " audio_rev_metas = json.load(f)\n", "\n", "ORDER_REF_STR = \"rc_poc\"\n", "\n", "tot_duration_s = np.sum([m[\"duration_s\"] for m in audio_rev_metas])\n", "print(round(tot_duration_s / 60 / 60, 1), \"hours total\")" ] }, { "cell_type": "code", "execution_count": 25, "id": "8c66f7f9", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [00:22<00:00, 4.52s/it]\n" ] } ], "source": [ "order_items = []\n", "for m in tqdm.tqdm(audio_rev_metas):\n", " media_loc = upload_file(AUTH_STR, m[\"audio_filepath\"], file_ref_str=m[\"uid\"])\n", " order_items.append(\n", " {\n", " \"media_loc\": media_loc,\n", " \"hotwords\": GLOBAL_HOTWORDS,\n", " \"duration_s\": int(np.ceil(m[\"duration_s\"])),\n", " \"uuid\": m[\"uid\"],\n", " }\n", " )" ] }, { "cell_type": "code", "execution_count": 26, "id": "a0ee5f4f", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, \"01_order_items.json\"), \"w\") as f:\n", " json.dump(order_items, f)" ] }, { "cell_type": "code", "execution_count": 27, "id": "8318e46f", "metadata": {}, "outputs": [], "source": [ "order_number = make_order(AUTH_STR, order_items, order_ref_str=ORDER_REF_STR)\n", "# order_number = make_cc_order(AUTH_STR, order_items, order_ref_str=ORDER_REF_STR)" ] }, { "cell_type": "code", "execution_count": 28, "id": "27549d51", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, \"02_order_numer.txt\"), \"w\") as f:\n", " f.write(order_number)" ] }, { "cell_type": "code", "execution_count": null, "id": "0096715b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "4b13bb34", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "576b2bef", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b33dc8e0", "metadata": {}, "source": [ "## Get rev stuff" ] }, { "cell_type": "code", "execution_count": 65, "id": "e8b59ece", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.rev import get_auth_string, get_finished_order, get_transcript\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 70, "id": "9e4e40af", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, \"01_order_items.json\")) as f:\n", " order_items = json.load(f)\n", "with open(os.path.join(TO_REV_DIR, \"02_order_numer.txt\")) as f:\n", " order_number = f.read()" ] }, { "cell_type": "code", "execution_count": 246, "id": "c4d6d755", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "5/5 transcripts pulled\n" ] } ], "source": [ "# # grab while still in progress\n", "# from suno_utils.web.rev import _get_order_details, _parse_order_response\n", "# order_details = _get_order_details(AUTH_STR, order_number)\n", "# _, transcript_metas = _parse_order_response(order_details)\n", "# print(f\"{len(transcript_metas)}/{len(order_items)} transcripts pulled\")" ] }, { "cell_type": "code", "execution_count": 269, "id": "9275ae0d", "metadata": {}, "outputs": [], "source": [ "# grab all of it\n", "transcript_metas = get_finished_order(AUTH_STR, order_number, blocking=False)" ] }, { "cell_type": "code", "execution_count": 256, "id": "df28d887", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [00:09<00:00, 1.81s/it]\n" ] } ], "source": [ "# get json\n", "for transcript_meta in tqdm.tqdm(transcript_metas):\n", " transcript_filepath = os.path.join(FROM_REV_DIR, transcript_meta[\"uuid\"] + \".json\")\n", " if os.path.exists(transcript_filepath):\n", " continue\n", " transcript_json = get_transcript(AUTH_STR, transcript_meta)\n", " with open(transcript_filepath, \"w\") as f:\n", " json.dump(transcript_json, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "6fcfda29", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9be84dd0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "9cffb6b3", "metadata": {}, "source": [ "## Make manifest and run pipeline" ] }, { "cell_type": "code", "execution_count": 480, "id": "7d75b028", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", " audio_rev_metas = json.load(f)" ] }, { "cell_type": "code", "execution_count": 481, "id": "db8a1a08", "metadata": {}, "outputs": [], "source": [ "# make sure to delete segments when rerunning\n", "shutil.rmtree(ARTIFACTS_DIR, ignore_errors=True)\n", "shutil.rmtree(SEGMENTS_DIR, ignore_errors=True)\n", "os.makedirs(ARTIFACTS_DIR)\n", "os.makedirs(SEGMENTS_DIR)" ] }, { "cell_type": "code", "execution_count": 482, "id": "e1cfe9df", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [00:00<00:00, 15363.75it/s]\n" ] } ], "source": [ "manifest_filepath = os.path.join(ARTIFACTS_DIR , \"00_manifest.json\")\n", "manifest = []\n", "for audio_rev_meta in tqdm.tqdm(audio_rev_metas):\n", " transcript_fp = os.path.join(FROM_REV_DIR, audio_rev_meta[\"uid\"] + \".json\")\n", " if not os.path.exists(transcript_fp):\n", " continue\n", " manifest.append({\n", " \"id\": audio_rev_meta[\"uid\"],\n", " \"audio_filepath\": audio_rev_meta[\"audio_filepath\"],\n", " \"transcript_filepath\": transcript_fp,\n", " \"segments_meta\": audio_rev_meta[\"segments_meta\"],\n", " })\n", "with open(manifest_filepath, \"w\") as f:\n", " json.dump(manifest, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "2f53c224", "metadata": {}, "outputs": [], "source": [ "CUDA_VISIBLE_DEVICES=1 python -m suno_utils.scripts.customer.sanas.run \\\n", " --output_dir=\"/mnt/data-ssd-1/data/private/customer/ring_central/2022-10-13-poc/pipeline/2022_10_13\" \\\n", " --use_gpu \\\n", " --conda-env=\"mfa\" \\\n", " --conda_sh_filepath=\"/home/georg/anaconda3/etc/profile.d/conda.sh\" \\\n", " --mfa_dictionary_name=\"english_mfa\" \\\n", " --mfa_acoustic_model_name=\"english_mfa\" \\\n", " --num_cores=20 \\\n", " --manifest_file=\"/mnt/data-ssd-1/data/private/customer/ring_central/2022-10-13-poc/pipeline/2022_10_13/artifacts/00_manifest.json\" \\\n", " --data_delivery_dir=\"/mnt/data-ssd-1/data/private/customer/ring_central/2022-10-13-poc/pipeline/2022_10_13/artifacts/tmp_delivery\" \\\n", " --sample_rate=16000 \\\n", " --max_cer_thres=0.6 \\\n", " --anonymize_speakers \\\n", " --skip_verify_boundaries" ] }, { "cell_type": "code", "execution_count": null, "id": "caa34e53", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "13f1f39c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b721d110", "metadata": {}, "source": [ "## QA" ] }, { "cell_type": "code", "execution_count": 483, "id": "9b4260c1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2.8 hours\n" ] } ], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"03_segment_meta_post_timestamp.json\")) as f:\n", " segments_manifest = json.load(f)\n", "print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": 484, "id": "14d34ae2", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "294\n", "Bye. [inaudible]\n", "bye [inaudible]\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "242\n", "Where do I see your screen, though? One --\n", "where do i see your screen though one --\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "1147\n", "So I need to respond to her, but um, you know, I think this agreement is --\n", "so i need to respond to her but um you know i think this agreement is --\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "829\n", "Just take one more product line. Yeah. Yeah. Whichever. Create a product line. Because we're gonna take it -- But that's more --\n", "just take one more product line yeah yeah whichever create a product line because we're gonna take it -- but that's more --\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "121\n", "Okay. Mm-hmm.\n", "okay mm hmm\n", "----------\n" ] } ], "source": [ "# look at random ones\n", "l = list(zip(range(len(segments_manifest)), segments_manifest[:]))\n", "random.shuffle(l)\n", "for n, m in l[:5]:\n", " play_audio(m[\"uri\"])\n", " print(n)\n", " print(Tokens.from_dict(m[\"transcript\"][\"tokens\"]).text)\n", " print(Tokens.from_dict(m[\"transcript_normalized\"][\"tokens\"]).text)\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 486, "id": "f17c413a", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "712\n", "The API patch. That would be in August. So [inaudible] right? But -- So, what's the action item here? Um, we're gonna circle the wagons. We're gonna see where you are with AT&T. There's no outstanding deliverables from -- [inaudible] W or platform things. At this time.\n", "the api patch that would be in august so [inaudible] right but -- so what's the action item here um we're gonna circle the wagons we're gonna see where you are with a t and t there's no outstanding deliverables from -- [inaudible] w or platform things at this time\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "115\n", "So, this document could be completely unnecessary if AT&T makes certain -- So, have you talked to -- our desired outco --\n", "so this document could be completely unnecessary if a t and t makes certain -- so have you talked to -- our desired outco --\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "840\n", "Then Mohsin should talk? Which is the AT&T guy?\n", "then mohsin should talk which is the a t and t guy\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "711\n", "No. Well, we were all on the same page, that we needed to go back to AT&T; we were very clear. If they agree to X, Y, and Z, then --\n", "no well we were all on the same page that we needed to go back to a t and t we were very clear if they agree to x y and z then --\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "1226\n", "Um -- I'm [inaudible] the, the open items here are, you know, AT&T approval. Right? Approval or acceptance. [inaudible] Right. And then, um -- [inaudible] integration, I [inaudible] strategy, [inaudible] right?\n", "um -- i'm [inaudible] the the open items here are you know a t and t approval right approval or acceptance [inaudible] right and then um -- [inaudible] integration i [inaudible] strategy [inaudible] right\n", "----------\n" ] } ], "source": [ "# look at tough numbers\n", "l = [\n", " (n, m) for n, m in enumerate(segments_manifest)\n", "# if re.search(r\"[^\\s0-9][0-9]+[^\\s0-9]\", Tokens.from_dict(m[\"transcript\"][\"tokens\"]).text)\n", " if re.search(r\"AT\\&T\", Tokens.from_dict(m[\"transcript\"][\"tokens\"]).text)\n", "]\n", "random.shuffle(l)\n", "for n, m in l[:5]:\n", " play_audio(m[\"uri\"])\n", " print(n)\n", " print(Tokens.from_dict(m[\"transcript\"][\"tokens\"]).text)\n", " print(Tokens.from_dict(m[\"transcript_normalized\"][\"tokens\"]).text)\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 487, "id": "a0609f44", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "221\n", "This is basically between, uh, August and September so far? Kind of, uh, yeah. Great.\n", "this is basically between uh august and september so far kind of uh yeah great\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "218\n", "Hmm. Shocking voice, right? And the --\n", "hmm shocking voice right and the --\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "582\n", "[inaudible] Have, there's one [inaudible]\n", "[inaudible] have there's one [inaudible]\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "183\n", "Okay, this -- No, just --\n", "okay this -- no just --\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "1343\n", "RTB now [inaudible]\n", "rtb now [inaudible]\n", "----------\n" ] } ], "source": [ "# look at high cer tagged ones\n", "l = [(n, m) for n, m in enumerate(segments_manifest) if \"high asr cer\" in m.get(\"ignore_reason\", \"\")]\n", "random.shuffle(l)\n", "for n, m in l[:5]:\n", " play_audio(m[\"uri\"])\n", " print(n)\n", " print(Tokens.from_dict(m[\"transcript\"][\"tokens\"]).text)\n", " print(Tokens.from_dict(m[\"transcript_normalized\"][\"tokens\"]).text)\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 208, "id": "b608a979", "metadata": {}, "outputs": [], "source": [ "# # find specific one:\n", "# for m in segments_manifest:\n", "# if \"11-year\" in Tokens.from_dict(m[\"transcript\"][\"tokens\"]).text:\n", "# print(m)\n", "# break" ] }, { "cell_type": "code", "execution_count": null, "id": "9352a470", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5f51345f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "33ae5fab", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "32c3f559", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "ae98f9e9", "metadata": {}, "source": [ "## make deliverable" ] }, { "cell_type": "code", "execution_count": null, "id": "e06a20bb", "metadata": {}, "outputs": [], "source": [ "# https://drive.google.com/drive/folders/1cNcvfkdd8TlLTBy0ib2rLpLUwoHfFgkE\n", "# fix:\n", " # 11-year -> eleven dash year - (no dash)\n", " # 221-704 -> two two one seven zero four (should be oh)" ] }, { "cell_type": "code", "execution_count": 489, "id": "b1795289", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"03_segment_meta_post_timestamp.json\")) as f:\n", " segments_manifest = json.load(f)" ] }, { "cell_type": "code", "execution_count": 542, "id": "e003a759", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "fixed: string say it again it's uh four four six two two one seven zero four [inaudible]\n", "fixed: Talking to us. Yeah. [inaudible] I'll just add that. I think that's going be part of [inaudible]\n", "fixed: [inaudible] So this, [inaudible] Jose [inaudible] No, sorry, Pauli.\n", "fixed: two two one seven oh four [inaudible] yeah cause we don't have a screen\n", "fixed: [inaudible] So, [inaudible] [laughter]\n", "fixed: [inaudible] This list needs to move to an actual list vs. a wishlist. So if that means removing, that's fine but we need an actual list, so I can go to market with that. And same with integrations. Harshan, this is where you would come into play. Mm-hmm. Okay. I need to know, what are the integrated products we have? I could look at your integrations roadmap, but your roadmap only goes out to Q3.\n", "fixed: A -- also [inaudible] Why don't you go cover the [inaudible]\n", "fixed: [inaudible] have what's index for [inaudible] I don't know if [inaudible]\n", "fixed: [inaudible] what's in [inaudible] I don't know [inaudible]\n", "fixed: [inaudible] of where he has [inaudible] but sometimes he [inaudible] That it's worse [inaudible]\n" ] } ], "source": [ "output_segments_manifest = []\n", "for m in segments_manifest:\n", " uid = m[\"source_audio_uri\"].split(\"/\")[-1].split(\".\")[0]\n", " tokens = m[\"transcript\"][\"tokens\"]\n", " tokens_norm = m[\"transcript_normalized\"][\"tokens\"]\n", " tokens_fixed = tokens[:]\n", " tokens_norm_fixed = tokens_norm[:]\n", " if uid == \"218a1b6e-55e7-49f3-8d84-d0ef03ba3bf6\":\n", " tokens_norm_fixed = [\n", " {'value': 'string', 'type': 'text', 'speaker_id': 'Speaker 1', 'start_s': 0.0, 'end_s': 0.52}, \n", " {'value': 'say', 'type': 'text', 'speaker_id': 'Speaker 1', 'start_s': 1.5, 'end_s': 1.56}, \n", " {'value': 'it', 'type': 'text', 'speaker_id': 'Speaker 1', 'start_s': 1.61, 'end_s': 1.77}, \n", " {'value': 'again', 'type': 'text', 'speaker_id': 'Speaker 1', 'start_s': 1.77, 'end_s': 2.48}, \n", " {'value': \"it's\", 'type': 'text', 'speaker_id': 'Speaker 1', 'start_s': 4.11, 'end_s': 4.3}, \n", " {'value': 'uh', 'type': 'text', 'speaker_id': 'Speaker 1', 'start_s': 4.3, 'end_s': 4.55}, \n", " {'value': 'four', 'type': 'text', 'speaker_id': 'Speaker 1', 'start_s': 4.55, 'end_s': 4.95}, \n", " {'value': 'four', 'type': 'text', 'speaker_id': 'Speaker 1', 'start_s': 5.34, 'end_s': 5.75}, \n", " {'value': 'six', 'type': 'text', 'speaker_id': 'Speaker 1', 'start_s': 6.61, 'end_s': 6.89}, \n", " {'value': 'two', 'type': 'text', 'speaker_id': 'Speaker 1', 'start_s': 6.89, 'end_s': 7.22}, \n", " {'value': 'two', 'type': 'text', 'speaker_id': 'Speaker 1', 'start_s': 7.56, 'end_s': 7.76}, \n", " {'value': 'one', 'type': 'text', 'speaker_id': 'Speaker 1', 'start_s': 7.76, 'end_s': 8.14}, \n", " {'value': 'seven', 'type': 'text', 'speaker_id': 'Speaker 1', 'start_s': 8.53, 'end_s': 8.8}, \n", " {'value': 'zero', 'type': 'text', 'speaker_id': 'Speaker 1', 'start_s': 8.8, 'end_s': 9.21}, \n", " {'value': 'four', 'type': 'text', 'speaker_id': 'Speaker 1', 'start_s': 9.21, 'end_s': 9.46}, \n", " {'value': '[inaudible]', 'type': 'tag'},\n", " ]\n", " print(\"fixed:\", Tokens.from_dict(tokens_norm_fixed).text)\n", " elif uid == \"e28846c0-9d10-4179-a7f9-c7f8ee0ae904\":\n", " tokens_norm_fixed = []\n", " for t in tokens_norm:\n", " if t[\"value\"] == \"zero\":\n", " t_fixed = copy.deepcopy(t)\n", " t_fixed[\"value\"] = \"oh\"\n", " tokens_norm_fixed.append(t_fixed)\n", " else:\n", " tokens_norm_fixed.append(t)\n", " print(\"fixed:\", Tokens.from_dict(tokens_norm_fixed).text)\n", " elif uid == \"4eaf1934-1108-45ae-9aee-0ad68c0411f9\":\n", " tokens_fixed = []\n", " for t in tokens:\n", " if t[\"value\"] == \",but\":\n", " t_fixed = copy.deepcopy(t)\n", " t_fixed[\"value\"] = \"but\"\n", " tokens_fixed.append(t_fixed)\n", " else:\n", " tokens_fixed.append(t)\n", " print(\"fixed:\", Tokens.from_dict(tokens_fixed).text)\n", " elif uid == \"61ab3fae-92a1-4e86-93a4-9d7d7f86bdaf\":\n", " tokens_fixed = []\n", " for t in tokens:\n", " if t[\"value\"] == \".\":\n", " continue\n", " else:\n", " tokens_fixed.append(t)\n", " print(\"fixed:\", Tokens.from_dict(tokens_fixed).text)\n", " elif uid in (\n", " \"36de2fa8-dd44-4ecc-9b1b-c3f2865f511d\",\n", " \"586ff8bf-4c26-485c-934a-6c4c955c9c8d\",\n", " \"66597f84-32f9-454b-a8ef-82dd50b92bf6\",\n", " \"2bffd91a-a65a-4de0-abf0-d90035301b80\",\n", " \"baca3eb0-09ba-4fb0-bfbd-75923f3daebb\",\n", " \"d1e92357-56ba-49c1-92e0-6d4de1df860e\",\n", " ):\n", " tokens_fixed = []\n", " for t in tokens:\n", " if t[\"value\"] == \".\":\n", " tokens_fixed.append({\"value\": \"[inaudible]\", \"type\": \"tag\"})\n", " else:\n", " tokens_fixed.append(t)\n", " print(\"fixed:\", Tokens.from_dict(tokens_fixed).text)\n", " \n", " # remove tags from normalized\n", " tokens_norm_fixed = [t for t in tokens_norm_fixed if t[\"type\"] == \"text\"]\n", " \n", " out_m = {\n", " \"uid\": uid,\n", "# \"uid\": m[\"source_audio_id\"],\n", " \"transcript\": {\n", " \"tokens\": tokens_fixed,\n", " \"text\": Tokens.from_dict(tokens_fixed).text,\n", " },\n", " \"transcript_normalized\": {\n", " \"tokens\": tokens_norm_fixed,\n", " \"text\": Tokens.from_dict(tokens_norm_fixed).text,\n", " },\n", " }\n", " output_segments_manifest.append(out_m)" ] }, { "cell_type": "code", "execution_count": 543, "id": "8e2de195", "metadata": {}, "outputs": [], "source": [ "# why no inaudible on norm!!!! inaudible in exception" ] }, { "cell_type": "code", "execution_count": 544, "id": "0d95688d", "metadata": {}, "outputs": [], "source": [ "random.seed(6006)\n", "random.shuffle(output_segments_manifest)" ] }, { "cell_type": "code", "execution_count": 545, "id": "c7d1afd8", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"delivery_output_metadata.jsonl\"), \"w\") as f:\n", " for m in output_segments_manifest:\n", " f.write(json.dumps(m) + \"\\n\")\n", "# should be 1426" ] }, { "cell_type": "code", "execution_count": 409, "id": "3deb6c62", "metadata": {}, "outputs": [], "source": [ "# !head -3 /mnt/data-ssd-1/data/private/customer/ring_central/2022-10-13-poc/pipeline/2022_10_13/artifacts/delivery_output_metadata.jsonl" ] }, { "cell_type": "code", "execution_count": 552, "id": "7330821b", "metadata": {}, "outputs": [], "source": [ "!cp /mnt/data-ssd-1/data/private/customer/ring_central/2022-10-13-poc/pipeline/2022_10_13/artifacts/delivery_output_metadata.jsonl ." ] }, { "cell_type": "code", "execution_count": null, "id": "c5d360b0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bfc24b7c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "716aa58d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b3df6bbd", "metadata": {}, "source": [ "## double check output items" ] }, { "cell_type": "code", "execution_count": 504, "id": "f0a82007", "metadata": {}, "outputs": [], "source": [ "orig_audio_dir = \"/mnt/data-ssd-1/data/private/customer/ring_central/2022-10-13-poc/raw_data/audio/\"" ] }, { "cell_type": "code", "execution_count": 551, "id": "78e2d08f", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Well, you are on, we are recording, and, uh, let's roll from here. So, hey, we're regrouping again after that good session we had a couple, um, probably a couple Thursdays ago. And, uh, oh, who's Sharon?\n", "well you are on we are recording and uh let's roll from here so hey we're regrouping again after that good session we had a couple um probably a couple thursdays ago and uh oh who's sharon\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "All right. So, we can just put in the headings, basically.\n", "all right so we can just put in the headings basically\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "And, um, [inaudible]\n", "and um\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Oh, wow, that's [inaudible] Is this [inaudible] Oh yeah, this is, this is everything put together, not just, n-n-, uh, meeting. So this is the meeting minutes, and this is everything put together. Because there's also --\n", "oh wow that's is this oh yeah this is this is everything put together not just n n uh meeting so this is the meeting minutes and this is everything put together because there's also\n", "----------\n" ] } ], "source": [ "for m in output_segments_manifest[1:5]:\n", " Audio.from_file(orig_audio_dir + m[\"uid\"] + \".wav\").play()\n", " print(m[\"transcript\"][\"text\"])\n", " print(m[\"transcript_normalized\"][\"text\"])\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": null, "id": "20c7f5e9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "63589df0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "56bba97f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "55d92f14", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "242b81d9", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 211, "id": "e63897bc", "metadata": {}, "outputs": [], "source": [ "DELIVERY_DIR = \"/home/georg/notebooks/customers/aiera/tmp_delivery\"\n", "DELIVERY_SEGMENTS_DIR = os.path.join(DELIVERY_DIR, \"segments\")\n", "\n", "shutil.rmtree(DELIVERY_DIR, ignore_errors=True)\n", "os.makedirs(DELIVERY_DIR)\n", "os.makedirs(DELIVERY_SEGMENTS_DIR)" ] }, { "cell_type": "code", "execution_count": 210, "id": "91c17585", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.7 hours\n" ] } ], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"03_segment_meta_post_timestamp.json\")) as f:\n", " segments_manifest = json.load(f)\n", "print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": 240, "id": "83d33b88", "metadata": {}, "outputs": [], "source": [ "speakers_map = {\n", " \"Speaker 1\": \"Pete\",\n", " \"Speaker 2\": \"Bill Sibold\",\n", "}\n", "\n", "# compile long form pretty print json\n", "full_transcript_tokens = []\n", "for m in segments_manifest:\n", " full_transcript_tokens.extend(m[\"transcript\"][\"tokens\"])\n", "full_transcript_tokens = Tokens([Token.from_dict(d).remove_timestamps() for d in full_transcript_tokens])\n", "full_transcript = []\n", "for d in full_transcript_tokens.speaker_turns:\n", " text = d[\"text\"]\n", " text = re.sub(r\"-- [A-Z][a-z]\", lambda m: m.group(0).lower(), text)\n", " full_transcript.append({\n", " \"speaker_id\": speakers_map[d[\"speaker_id\"]],\n", " \"text\": text,\n", " })\n", "with open(os.path.join(DELIVERY_DIR, \"fulltext.json\"), \"w\") as f:\n", " json.dump(full_transcript, f, indent=2)\n", " \n", "# make output segments meta and write segments\n", "output_segments = []\n", "for m in segments_manifest:\n", " new_uri = os.path.join(DELIVERY_SEGMENTS_DIR, m[\"id\"] + \".wav\")\n", " shutil.copy(m[\"uri\"], new_uri)\n", " transcript = {\n", " \"tokens\": [\n", " Token.from_dict(d).add_speaker_id(speakers_map[d[\"speaker_id\"]]).as_dict() \n", " for d in m[\"transcript\"][\"tokens\"]\n", " ],\n", " \"text\": m[\"transcript\"][\"text\"],\n", " }\n", " transcript_normalized = {\n", " \"tokens\": [\n", " Token.from_dict(d).add_speaker_id(speakers_map[d[\"speaker_id\"]]).as_dict() \n", " for d in m[\"transcript_normalized\"][\"tokens\"]\n", " ],\n", " \"text\": m[\"transcript_normalized\"][\"text\"],\n", " }\n", " output_segments.append({\n", " \"id\": m[\"id\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " \"uri\": os.path.join(\"segments\", m[\"id\"] + \".wav\"),\n", " \"transcript\": transcript,\n", " \"transcript_normalized\": transcript_normalized,\n", " })\n", "with open(os.path.join(DELIVERY_DIR, \"metadata.jsonl\"), \"w\") as f:\n", " for m in output_segments:\n", " f.write(json.dumps(m) + \"\\n\")" ] }, { "cell_type": "code", "execution_count": 242, "id": "826e0b06", "metadata": {}, "outputs": [], "source": [ "# !cat /home/georg/notebooks/customers/aiera/tmp_delivery/fulltext.json" ] }, { "cell_type": "code", "execution_count": null, "id": "e33e0a11", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9b2137aa", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ff7c5d47", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }