{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "b304dfff",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
"%pylab inline"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "f955e852",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\""
]
},
{
"cell_type": "code",
"execution_count": 34,
"id": "e9716000",
"metadata": {},
"outputs": [],
"source": [
"import time\n",
"import os\n",
"import tqdm\n",
"import shutil\n",
"import random\n",
"import funcy\n",
"import json\n",
"import uuid\n",
"import numpy as np\n",
"import multiprocessing\n",
"import pandas as pd\n",
"from suno_utils.audio import Audio\n",
"from suno_utils.audio.conversion import play_audio, get_audio_properties\n",
"from suno_utils.utils.slicer import get_vad_gaps, get_intervals_from_gaps\n",
"from suno_utils.audio import Tokens\n",
"\n",
"\n",
"DATE_DIR = \"2022_08_11\"\n",
"\n",
"BASE_DIR = \"/mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/\"\n",
"TO_REV_DIR = os.path.join(BASE_DIR, \"to_rev\", DATE_DIR)\n",
"FROM_REV_DIR = os.path.join(BASE_DIR, \"from_rev\", DATE_DIR)\n",
"TO_REV_AUDIO_DIR = os.path.join(TO_REV_DIR, \"audio\")\n",
"REV_MANIFEST_FILEPATH = os.path.join(TO_REV_DIR, \"rev_manifest.jsonl\")\n",
"FAILED_MANIFEST_FILEPATH = os.path.join(TO_REV_DIR, \"failed_manifest.jsonl\")\n",
"\n",
"SEGMENT_DATE_DIR = \"2022_08_11_2\"\n",
"\n",
"SEGMENTS_DIR = os.path.join(BASE_DIR, \"pipeline\", SEGMENT_DATE_DIR, \"segments\")\n",
"ARTIFACTS_DIR = os.path.join(BASE_DIR, \"pipeline\", SEGMENT_DATE_DIR, \"artifacts\")"
]
},
{
"cell_type": "code",
"execution_count": 59,
"id": "a88bf02b",
"metadata": {},
"outputs": [],
"source": [
"os.makedirs(SEGMENTS_DIR, exist_ok=True)\n",
"os.makedirs(ARTIFACTS_DIR, exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "2d612fd7",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"433.3h of segments, 482.2h of rev submission\n"
]
}
],
"source": [
"with open(REV_MANIFEST_FILEPATH) as f:\n",
" rev_submission_meta = []\n",
" for p in f.read().strip().split(\"\\n\"):\n",
" rev_submission_meta.append(json.loads(p))\n",
"seg_tot_s = 0\n",
"tot_s = 0\n",
"for m in rev_submission_meta:\n",
" seg_tot_s += np.sum([s[\"duration_s\"] for s in m[\"segments_meta\"] if s[\"type\"] == \"speech\"])\n",
" tot_s += np.sum([s[\"duration_s\"] for s in m[\"segments_meta\"]])\n",
"seg_tot_s = round(seg_tot_s / 60 / 60, 1)\n",
"tot_s = round(tot_s / 60 / 60, 1)\n",
"print(\"{}h of segments, {}h of rev submission\".format(seg_tot_s, tot_s))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3c451ead",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "9a054d3d",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "c9c31c96",
"metadata": {},
"source": [
"## Parse"
]
},
{
"cell_type": "code",
"execution_count": 62,
"id": "6c298d93",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"917 filenames\n"
]
}
],
"source": [
"# with open(os.path.join(TO_REV_DIR, \"batch_1_fns.txt\")) as f:\n",
"with open(os.path.join(TO_REV_DIR, \"batch_2_fns.txt\")) as f:\n",
" batch_fns = f.read().split(\"\\n\")\n",
"print(len(batch_fns), \"filenames\")"
]
},
{
"cell_type": "code",
"execution_count": 63,
"id": "3a4802e1",
"metadata": {},
"outputs": [],
"source": [
"from suno_utils.web.rev import parse_transcript\n",
"\n",
"rev_transcripts = []\n",
"for fn in os.listdir(FROM_REV_DIR):\n",
" if fn not in set(batch_fns):\n",
" continue\n",
" with open(os.path.join(FROM_REV_DIR, fn)) as f:\n",
" raw_transcript = json.load(f)\n",
" transcript = parse_transcript(raw_transcript, anonymize_speakers=True)\n",
" rev_transcripts.append({\n",
" \"id\": fn[:-5],\n",
" \"transcript\": transcript,\n",
" })\n",
"print(len(rev_transcripts), \"transcripts\")"
]
},
{
"cell_type": "code",
"execution_count": 65,
"id": "514e7fe3",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"10372 meta data entries\n"
]
}
],
"source": [
"# find most egregious mistakes\n",
"from collections import defaultdict\n",
"\n",
"duration_info = defaultdict(dict)\n",
"\n",
"for m in rev_submission_meta:\n",
" duration_info[m[\"uid\"]][\"available_duration_s\"] = np.sum([\n",
" s[\"duration_s\"] for s in m[\"segments_meta\"] if s[\"type\"] == \"speech\"\n",
" ])\n",
"print(len(duration_info), \"meta data entries\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9ff03ea4",
"metadata": {},
"outputs": [],
"source": [
"# for m in segments_meta:\n",
"# uid = m[\"source_audio_id\"]\n",
"# if \"retained_duration_s\" not in duration_info[uid]:\n",
"# duration_info[uid][\"retained_duration_s\"] = 0\n",
"# duration_info[uid][\"retained_duration_s\"] += m[\"duration_s\"]\n",
" \n",
"# df = pd.DataFrame(duration_info).T.rename_axis(\"uid\").fillna(0)\n",
"# df[\"missing_frac\"] = 1 - df[\"retained_duration_s\"] / df[\"available_duration_s\"]\n",
"# df[\"missing_duration_s\"] = df[\"available_duration_s\"] - df[\"retained_duration_s\"]\n",
"# df.sort_values(\"missing_duration_s\", ascending=False).head()"
]
},
{
"cell_type": "code",
"execution_count": 66,
"id": "90a7f695",
"metadata": {},
"outputs": [],
"source": [
"from statistics import mode\n",
"\n",
"digits = list(\"0123456789\")\n",
"numbers = \"zero one two three four five six seven eight nine\".split()\n",
"\n",
"def _is_numeric(text):\n",
" text_bits = re.sub(r\"[^\\sa-z0-9]\", \"\", text.lower()).strip().split()\n",
" return all([s in set(numbers) or re.match(r\"[0-9]+\", s) for s in text_bits])\n",
"\n",
"def _digitify(text):\n",
" for d, n in zip(digits, numbers):\n",
" text = text.lower().replace(n, d)\n",
" text = re.sub(r\"[^0-9]\", \"\", text)\n",
" return text\n",
"\n",
"def _find_subs(s1, s2):\n",
" subs = []\n",
" loc = 0\n",
" while s1:\n",
" s1_copy = s1\n",
" while s1_copy:\n",
" while s1_copy and s1_copy not in s2:\n",
" s1_copy = s1_copy[:-1]\n",
" if s1_copy:\n",
" subs.append((loc, s2.index(s1_copy), s1_copy))\n",
" loc += len(s1_copy)\n",
" s1 = s1[len(s1_copy):]\n",
" else:\n",
" s1 = s1[1:]\n",
" loc += 1\n",
" s1_copy = s1 \n",
" return subs\n",
"\n",
"def get_segments(rev_transcript, n_segments):\n",
" \n",
" # find indicator speaker\n",
" indicator_segments = [\n",
" t[\"speaker_id\"] for t in rev_transcript.speaker_turns if _is_numeric(t[\"plaintext\"])\n",
" ]\n",
" if len(indicator_segments) == 0:\n",
" print(\"no indicators found\")\n",
" return []\n",
" speaker_id = mode(indicator_segments)\n",
" # parse numbers for indicator speaker\n",
" indicator_tokens = []\n",
" prev_n = -2\n",
" for n, token in enumerate(rev_transcript.tokens):\n",
" if token.speaker_id != speaker_id:\n",
" continue\n",
" if n - prev_n == 1:\n",
" indicator_tokens[-1] = (indicator_tokens[-1][0], indicator_tokens[-1][1] + _digitify(token.value))\n",
" else:\n",
" indicator_tokens.append((n, _digitify(token.value)))\n",
" prev_n = n\n",
"\n",
" # see which of the indicator tokens got matched\n",
" # list of [(indicator_index, token_index)]\n",
"\n",
" match_str = \"*\".join([s for _, s in indicator_tokens])\n",
" known_idx_2_segment_nr = {}\n",
" indicator_texts = []\n",
" offs_idx = 0\n",
" for n in range(n_segments):\n",
" text = str(n).zfill(2)\n",
" for nn in range(len(text)):\n",
" known_idx_2_segment_nr[offs_idx + nn] = n\n",
" offs_idx += len(text) + 1\n",
" indicator_texts.append(text)\n",
" known_str = \"|\".join(indicator_texts)\n",
" matched_idx_set = {}\n",
"# from difflib import SequenceMatcher\n",
"# matcher = SequenceMatcher(\n",
"# a=match_str,\n",
"# b=known_str,\n",
"# autojunk=False,\n",
"# )\n",
"# for block in matcher.get_matching_blocks():\n",
"# if block.size == 0:\n",
"# continue\n",
"# l = []\n",
"# for n in range(block.b, block.b + block.size):\n",
"# l.append(known_idx_2_segment_nr[n])\n",
"# if len(set(l)) != 1:\n",
"# continue\n",
"# matched_idx_set[block.a] = l[0]\n",
" \n",
" for a_idx, b_idx, match_g in _find_subs(match_str, known_str):\n",
" l = []\n",
" for n in range(b_idx, b_idx + len(match_g)):\n",
" l.append(known_idx_2_segment_nr[n])\n",
" if len(set(l)) != 1:\n",
" continue\n",
" matched_idx_set[a_idx] = l[0]\n",
" \n",
" \n",
" offs_idx = 0\n",
" correct_indicators = [] # indicator_index, token_index\n",
" for token_idx, text in indicator_tokens:\n",
" for n in range(len(text)):\n",
" if offs_idx + n in matched_idx_set:\n",
" correct_indicators.append((matched_idx_set[offs_idx + n], token_idx)) \n",
" offs_idx += len(text) + 1\n",
"\n",
" # get relevant tokens for all consecutive ones\n",
" token_segments = []\n",
" for (indicator_index_l, token_index_l), (indicator_index_r, token_index_r) in zip(\n",
" correct_indicators[:-1], correct_indicators[1:]\n",
" ):\n",
" if indicator_index_l == indicator_index_r - 1:\n",
" relevant_tokens = [\n",
" t for t in rev_transcript.tokens[token_index_l:token_index_r] if t.speaker_id != speaker_id\n",
" ]\n",
" token_segments.append((indicator_index_l, relevant_tokens))\n",
" # check last one\n",
" indicator_index_l, token_index_l = correct_indicators[-1]\n",
" if indicator_index_l == n_segments - 1:\n",
" relevant_tokens = [\n",
" t for t in rev_transcript.tokens[token_index_l:] if t.speaker_id != speaker_id\n",
" ]\n",
" token_segments.append((indicator_index_l, relevant_tokens))\n",
" \n",
" return token_segments"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "6ae43490",
"metadata": {},
"outputs": [],
"source": [
"# for m in rev_transcripts:\n",
"# if m[\"id\"] == \"f26fa631-f017-402a-8e7c-9da308e910bf\":\n",
"# rev_transcript = m[\"transcript\"]\n",
"# break\n",
"# for m in rev_submission_meta:\n",
"# if m[\"uid\"] == \"f26fa631-f017-402a-8e7c-9da308e910bf\":\n",
"# segments_meta = m[\"segments_meta\"]\n",
"# break\n",
" \n",
"# n_segments = segments_meta[-1][\"segment_number\"] + 1\n",
"# token_segments = get_segments(rev_transcript, n_segments)\n",
"# segments_retained = set([n for n, _ in token_segments])\n",
"# dur_s = 0\n",
"# for m in segments_meta:\n",
"# if m[\"type\"] == \"speech\" and m[\"segment_number\"] in segments_retained:\n",
"# dur_s += m[\"duration_s\"]\n",
"# print(round(dur_s / 60, 1), \"minutes\")"
]
},
{
"cell_type": "code",
"execution_count": 316,
"id": "809fac82",
"metadata": {},
"outputs": [],
"source": [
"# segments_meta_map = {}\n",
"# for m in rev_submission_meta:\n",
"# segments_meta_map[m[\"uid\"]] = m[\"segments_meta\"]\n",
"# durs_list = []\n",
"# for m in tqdm.tqdm(rev_transcripts):\n",
"# segments_meta = segments_meta_map[m[\"id\"]]\n",
"# rev_transcript = m[\"transcript\"]\n",
"# n_segments = segments_meta[-1][\"segment_number\"] + 1\n",
"# token_segments = get_segments(rev_transcript, n_segments)\n",
"# segments_retained = set([n for n, _ in token_segments])\n",
"# dur_s = 0\n",
"# for m in segments_meta:\n",
"# if m[\"type\"] == \"speech\" and m[\"segment_number\"] in segments_retained:\n",
"# dur_s += m[\"duration_s\"]\n",
"# durs_list.append(dur_s)\n",
"# print(round(np.sum(durs_list) / 60 / 60, 1), \"hours\")\n",
"# # 28.8 hours"
]
},
{
"cell_type": "code",
"execution_count": 72,
"id": "ca3521fe",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
" 1%|██ | 12/917 [00:06<07:29, 2.01it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"low segment coverage for 1d236525-c6ae-44f2-a7e6-72ccb9bf9779, skipping...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 21%|███████████████████████████████▊ | 192/917 [01:46<05:03, 2.39it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"low segment coverage for 8c9dafec-baf9-4bff-ad6b-a0c54f0bf50b, skipping...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 25%|█████████████████████████████████████▊ | 228/917 [02:03<06:01, 1.91it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"low segment coverage for eb780034-7672-4b1f-8f2a-b3cf4884dc39, skipping...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 27%|████████████████████████████████████████▉ | 247/917 [02:14<08:13, 1.36it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"low segment coverage for 3c925f48-6e88-4ec5-b43f-7d15b0a993ad, skipping...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 27%|█████████████████████████████████████████▌ | 251/917 [02:17<06:14, 1.78it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"low segment coverage for 7f033437-b4ab-43a4-918c-86b6efd0e0f5, skipping...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 28%|███████████████████████████████████████████▎ | 261/917 [02:22<05:07, 2.13it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"low segment coverage for f8fd682f-a38e-4853-8a76-9d9292f6e047, skipping...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 47%|███████████████████████████████████████████████████████████████████████▉ | 434/917 [03:50<02:56, 2.74it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"low segment coverage for 85364b25-5997-4153-aeaa-45f3e6626032, skipping...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 69%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 637/917 [05:34<02:21, 1.98it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"low segment coverage for aeb7b9b6-2aba-49e2-a69a-ba5c3b08ad3a, skipping...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 816/917 [07:08<00:47, 2.14it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"low segment coverage for e87a8b64-e59c-4ab7-969f-934a224c1e1b, skipping...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 917/917 [08:01<00:00, 1.90it/s]\n"
]
}
],
"source": [
"# write the manifest\n",
"segments_meta_map = {}\n",
"for m in rev_submission_meta:\n",
" segments_meta_map[m[\"uid\"]] = m\n",
"manifest = []\n",
"for m in tqdm.tqdm(rev_transcripts):\n",
" # skip if already done\n",
" segments_meta = segments_meta_map[m[\"id\"]][\"segments_meta\"]\n",
" rev_transcript = m[\"transcript\"]\n",
" n_segments = segments_meta[-1][\"segment_number\"] + 1\n",
" token_segments = get_segments(rev_transcript, n_segments)\n",
" meta_info_map = {}\n",
" for mm in segments_meta:\n",
" if mm[\"type\"] != \"speech\":\n",
" continue\n",
" meta_info_map[mm[\"segment_number\"]] = mm\n",
" source_audio_uri = segments_meta_map[m[\"id\"]][\"original_audio_filepath\"]\n",
" audio = Audio.from_file(source_audio_uri, sample_rate=16_000, byte_width=2)\n",
" if len(token_segments) / n_segments < 0.6:\n",
" print(\"low segment coverage for {}, skipping...\".format(m[\"id\"]))\n",
" continue\n",
" for n, tokens in token_segments:\n",
" segment_id = str(uuid.uuid4())\n",
" new_segment_uri = os.path.join(SEGMENTS_DIR, \"{}.wav\".format(segment_id))\n",
" from_s = meta_info_map[n][\"orginial_audio_offset_s\"]\n",
" to_s = from_s + meta_info_map[n][\"duration_s\"]\n",
" audio.get_slice(from_s, to_s).to_wav(new_segment_uri)\n",
" manifest.append({\n",
" \"source_audio_offset_s\": from_s,\n",
" \"id\": segment_id,\n",
" \"uri\": new_segment_uri,\n",
" \"duration_s\": meta_info_map[n][\"duration_s\"],\n",
" \"transcript\": {\"tokens\": Tokens(tokens).as_dict()},\n",
" \"ignore_segment\": False,\n",
" \"ignore_reason\": \"\",\n",
" \"source_audio_uri\": source_audio_uri,\n",
" \"source_audio_id\": m[\"id\"],\n",
" \"source_transcript_id\": m[\"id\"],\n",
" \"source_transcript_uri\": os.path.join(BASE_DIR, \"from_rev/2022_08_07/{}.json\".format(m[\"id\"])),\n",
" })"
]
},
{
"cell_type": "code",
"execution_count": 74,
"id": "4d4a7723",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"48.5h of segments, 48.7h of rev submission\n"
]
}
],
"source": [
"# show what we retained\n",
"tmp_uids = set([m[\"source_audio_id\"] for m in manifest])\n",
"sub_dur_s = round(np.sum([duration_info[tmp_uid][\"available_duration_s\"] for tmp_uid in tmp_uids]) / 60 / 60, 1)\n",
"seg_dur_s = round(np.sum([m[\"duration_s\"] for m in manifest]) / 60 / 60, 1)\n",
"print(\"{}h of segments, {}h of rev submission\".format(seg_dur_s, sub_dur_s))\n",
"# 129.1h of segments, 130.0h of rev submission"
]
},
{
"cell_type": "code",
"execution_count": 76,
"id": "cee8873a",
"metadata": {},
"outputs": [],
"source": [
"with open(os.path.join(ARTIFACTS_DIR, \"01_segment_meta.json\"), \"w\") as f:\n",
" json.dump(manifest, f)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7da8debe",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "dc5e491f",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "acce0257",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "02c45969",
"metadata": {},
"source": [
"## QA"
]
},
{
"cell_type": "code",
"execution_count": 77,
"id": "e1333bc9",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"48.5 hours\n"
]
}
],
"source": [
"with open(os.path.join(ARTIFACTS_DIR, \"01_segment_meta.json\")) as f:\n",
" segments_manifest = json.load(f)\n",
"print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")"
]
},
{
"cell_type": "code",
"execution_count": 78,
"id": "aa498657",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"6f1a770c-1e0e-4cc2-b078-05b84a43e641\n",
"And I'll send you the list of supported headsets and the list of not supported headsets.\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------\n",
"6520829e-7465-425c-b728-d8374e4167b8\n",
"Hello. How are you doing? May I know your WinID for identification?\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------\n",
"10930e10-7341-4c7e-aada-829217a0885e\n",
"Okay, you're welcome. Thank you for calling. Have a nice day, and welcome to Continuum. Bye -- bye.\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------\n",
"a2bcb6b3-3de5-462b-a2e0-45de83329414\n",
"Yes, I can. Yes, I do have control the screen.\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------\n",
"1d788b49-35cb-4f8a-9b42-091efc7bef76\n",
"The first letter is a capitalized, and then followed by the number 44. Can you please try to log in now?\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------\n",
"3dd6776b-d9ab-4e8a-a4b2-78ff66cd1d3b\n",
"Let me try to call you again. Maybe, it's on mute. That's why I can't hear you?\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------\n",
"312b513b-d698-44fc-86f3-ae4e99c9ac8e\n",
"So, there is a link there. You can access that link using your phone or your, um, personal computer.\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------\n",
"8c07ad3f-2b74-42bd-8ebb-4cb3daef5977\n",
"Okay, and there will be another, uh, popup in your screen and, uh, click okay and then yes.\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------\n",
"0dbdbf3c-47f2-4e0c-a232-79cb796c71e4\n",
"Okay, um, so you logged in? You are still on the log-in window, correct?\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------\n",
"caab6b6f-4f50-44cc-878f-fd0410faed0d\n",
"We'll try to run a speed test once my remote control is back, so we can check your connection.\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------\n"
]
}
],
"source": [
"# random ones\n",
"# l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n",
"l = segments_manifest[:]\n",
"random.shuffle(l)\n",
"for e in l[:10]:\n",
" print(e[\"id\"])\n",
" print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n",
" Audio.from_file(e[\"uri\"]).play()\n",
" print(\"-\"*10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "40936b1f",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "650a11cd",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "bccfbf76",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "0f3f2a83",
"metadata": {},
"source": [
"## run whole pipeline"
]
},
{
"cell_type": "code",
"execution_count": 75,
"id": "a1b9475c",
"metadata": {},
"outputs": [],
"source": [
"# cd /home/georg/notebooks/customers/sanas"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bdd8803d",
"metadata": {},
"outputs": [],
"source": [
"# CUDA_VISIBLE_DEVICES=1 python run_modified_pipeline.py\n",
"# CUDA_VISIBLE_DEVICES=2 python run_modified_pipeline_2.py"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2abbdfe1",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "16275db3",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "59b62270",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "b721d110",
"metadata": {},
"source": [
"## QA"
]
},
{
"cell_type": "code",
"execution_count": 425,
"id": "9b4260c1",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"28.8 hours\n"
]
}
],
"source": [
"with open(os.path.join(ARTIFACTS_DIR, \"03_segment_meta_post_timestamp.json\")) as f:\n",
" segments_manifest = json.load(f)\n",
"print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f1bdebcf",
"metadata": {},
"outputs": [],
"source": [
"# sort by cer\n",
"from suno_utils.utils.metrics import get_cer\n",
"\n",
"l = segments_manifest[:]\n",
"cer_list = []\n",
"for m in l:\n",
" m[\"cer\"] = get_cer(m[\"transcript_normalized\"][\"text\"], m[\"transcript_asr\"][\"transcript\"])\n",
"l = sorted(l, key=lambda x: x[\"cer\"], reverse=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bdc9bc66",
"metadata": {},
"outputs": [],
"source": [
"for e in l[:10]:\n",
" print(e[\"cer\"])\n",
" print(e[\"id\"])\n",
" print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n",
" Audio.from_file(e[\"uri\"]).play()\n",
" print(\"-\"*10)"
]
},
{
"cell_type": "code",
"execution_count": 423,
"id": "0cfa5625",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"28.8 hours\n"
]
}
],
"source": [
"with open(os.path.join(BASE_DIR, \"pipeline/2022_08_07__2/artifacts/04_segment_metadata.jsonl\")) as f:\n",
" segments_manifest = [json.loads(e) for e in f.read().strip().split(\"\\n\")]\n",
"print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")"
]
},
{
"cell_type": "code",
"execution_count": 424,
"id": "8ba34e21",
"metadata": {},
"outputs": [],
"source": [
"# d = \"/mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/to_sanas/2022-08-12/\"\n",
"\n",
"# # l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n",
"# l = segments_manifest[:]\n",
"# random.shuffle(l)\n",
"# for e in l[:10]:\n",
"# print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n",
"# Audio.from_file(d + e[\"uri\"]).play()\n",
"# print(\"-\"*10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "55387924",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "2b0e5be9",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "1747220c",
"metadata": {},
"source": [
"## find dominant speaker"
]
},
{
"cell_type": "code",
"execution_count": 88,
"id": "a393ebe6",
"metadata": {},
"outputs": [],
"source": [
"with open(REV_MANIFEST_FILEPATH) as f:\n",
" rev_submission_meta = []\n",
" for p in f.read().strip().split(\"\\n\"):\n",
" rev_submission_meta.append(json.loads(p))\n",
"original_id_map = {}\n",
"for e in rev_submission_meta:\n",
" original_id_map[e[\"uid\"]] = \"/\".join(e[\"original_audio_filepath\"].split(\"/\")[-2:])"
]
},
{
"cell_type": "code",
"execution_count": 89,
"id": "fdc45b00",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4000/4000 [00:51<00:00, 77.69it/s]\n"
]
}
],
"source": [
"from suno_utils.web.rev import parse_transcript\n",
"from collections import Counter\n",
"\n",
"main_speaker_info = []\n",
"for fn in tqdm.tqdm(os.listdir(FROM_REV_DIR)):\n",
" if not fn.endswith(\".json\"):\n",
" continue\n",
" uid = fn[:-5]\n",
" original_uid = original_id_map[uid]\n",
" with open(os.path.join(FROM_REV_DIR, fn)) as f:\n",
" raw_transcript = json.load(f)\n",
" transcript = parse_transcript(raw_transcript, anonymize_speakers=True)\n",
" speech_amount = Counter()\n",
" for t in transcript.tokens:\n",
" if t.type != \"text\" or _is_numeric(t.value):\n",
" continue\n",
" speech_amount[t.speaker_id] += len(t.value)\n",
" if len(speech_amount) == 0:\n",
" main_speaker_info.append({\n",
" \"original_filename\": original_uid,\n",
" \"main_speakers\": [],\n",
" })\n",
" continue\n",
" denom = np.sum([e[-1] for e in speech_amount.most_common()])\n",
" main_speakers = [e[0] for e in speech_amount.most_common() if e[1] / denom >= 0.2]\n",
"# if uid == \"b592becf-4d7b-4df1-afa0-1502be991d76\":\n",
"# break\n",
" main_speaker_info.append({\n",
" \"original_filename\": original_uid,\n",
" \"main_speakers\": main_speakers,\n",
" })"
]
},
{
"cell_type": "code",
"execution_count": 90,
"id": "81ced604",
"metadata": {},
"outputs": [],
"source": [
"with open(os.path.join(ARTIFACTS_DIR, \"main_speaker_info.json\"), \"w\") as f:\n",
" json.dump(main_speaker_info, f)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ca5a5371",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "69ccc2cc",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "024bbeb8",
"metadata": {},
"source": [
"## Playground"
]
},
{
"cell_type": "code",
"execution_count": 333,
"id": "b66e48ad",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
" 8%|████████████▊ | 59/698 [00:00<00:03, 164.15it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"15327bfd-8b24-4b67-9b5f-8173d51669f3 0.7931034482758621\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"TEST_UUID = \"15327bfd-8b24-4b67-9b5f-8173d51669f3\"\n",
"# TEST_UUID = \"15f15e13-62af-4eed-9f62-043ce54d95e0\"\n",
"# TEST_UUID = \"ab8a3845-b192-44ae-b19f-03052ae2d49f\"\n",
"\n",
"segments_meta_map = {}\n",
"for m in rev_submission_meta:\n",
" segments_meta_map[m[\"uid\"]] = m\n",
"manifest = []\n",
"for m in tqdm.tqdm(rev_transcripts):\n",
" if m[\"id\"] != TEST_UUID:\n",
" continue\n",
" segments_meta = segments_meta_map[m[\"id\"]][\"segments_meta\"]\n",
" rev_transcript = m[\"transcript\"]\n",
" n_segments = segments_meta[-1][\"segment_number\"] + 1\n",
" token_segments = get_segments(rev_transcript, n_segments)\n",
" meta_info_map = {}\n",
" for mm in segments_meta:\n",
" if mm[\"type\"] != \"speech\":\n",
" continue\n",
" meta_info_map[mm[\"segment_number\"]] = mm\n",
" source_audio_uri = segments_meta_map[m[\"id\"]][\"original_audio_filepath\"]\n",
" audio = Audio.from_file(source_audio_uri, sample_rate=16_000, byte_width=2)\n",
" if len(token_segments) / n_segments < 0.999:\n",
" print(m[\"id\"], len(token_segments) / n_segments)\n",
" break"
]
},
{
"cell_type": "code",
"execution_count": 334,
"id": "0f772448",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.7931034482758621\n"
]
}
],
"source": [
"# find indicator speaker\n",
"indicator_segments = [\n",
" t[\"speaker_id\"] for t in rev_transcript.speaker_turns if _is_numeric(t[\"plaintext\"])\n",
"]\n",
"speaker_id = mode(indicator_segments)\n",
"# parse numbers for indicator speaker\n",
"indicator_tokens = []\n",
"prev_n = -2\n",
"for n, token in enumerate(rev_transcript.tokens):\n",
" if token.speaker_id != speaker_id:\n",
" continue\n",
" if n - prev_n == 1:\n",
" indicator_tokens[-1] = (indicator_tokens[-1][0], indicator_tokens[-1][1] + _digitify(token.value))\n",
" else:\n",
" indicator_tokens.append((n, _digitify(token.value)))\n",
" prev_n = n\n",
"\n",
"# see which of the indicator tokens got matched\n",
"# list of [(indicator_index, token_index)]\n",
"from difflib import SequenceMatcher\n",
"match_str = \"*\".join([s for _, s in indicator_tokens])\n",
"known_idx_2_segment_nr = {}\n",
"indicator_texts = []\n",
"offs_idx = 0\n",
"for n in range(n_segments):\n",
" text = str(n).zfill(2)\n",
" for nn in range(len(text)):\n",
" known_idx_2_segment_nr[offs_idx + nn] = n\n",
" offs_idx += len(text) + 1\n",
" indicator_texts.append(text)\n",
"known_str = \"|\".join(indicator_texts)\n",
"matched_idx_set = {}\n",
"for a_idx, b_idx, match_g in _find_subs(match_str, known_str):\n",
" l = []\n",
" for n in range(b_idx, b_idx + len(match_g)):\n",
" l.append(known_idx_2_segment_nr[n])\n",
" if len(set(l)) != 1:\n",
" continue\n",
" matched_idx_set[a_idx] = l[0]\n",
" \n",
"offs_idx = 0\n",
"correct_indicators = [] # indicator_index, token_index\n",
"for token_idx, text in indicator_tokens:\n",
" for n in range(len(text)):\n",
" if offs_idx + n in matched_idx_set:\n",
" correct_indicators.append((matched_idx_set[offs_idx + n], token_idx)) \n",
" offs_idx += len(text) + 1\n",
"\n",
"# get relevant tokens for all consecutive ones\n",
"token_segments = []\n",
"for (indicator_index_l, token_index_l), (indicator_index_r, token_index_r) in zip(\n",
" correct_indicators[:-1], correct_indicators[1:]\n",
"):\n",
" if indicator_index_l == indicator_index_r - 1:\n",
" relevant_tokens = [\n",
" t for t in rev_transcript.tokens[token_index_l:token_index_r] if t.speaker_id != speaker_id\n",
" ]\n",
" token_segments.append((indicator_index_l, relevant_tokens))\n",
"# check last one\n",
"indicator_index_l, token_index_l = correct_indicators[-1]\n",
"if indicator_index_l == n_segments - 1:\n",
" relevant_tokens = [\n",
" t for t in rev_transcript.tokens[token_index_l:] if t.speaker_id != speaker_id\n",
" ]\n",
" token_segments.append((indicator_index_l, relevant_tokens))\n",
" \n",
"print(len(token_segments) / n_segments)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b40e745c",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "e7bd0c79",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "a1347f2d",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "daf36b9a",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}