{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "e2119a4a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 2, "id": "602452ee", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ['CUDA_VISIBLE_DEVICES'] = ''" ] }, { "cell_type": "code", "execution_count": 15, "id": "e4c9ee94", "metadata": {}, "outputs": [], "source": [ "import json\n", "import random\n", "\n", "import funcy\n", "import pandas as pd\n", "import tqdm\n", "\n", "from suno_utils.utils.notebook import play_audio\n", "\n", "DATA_DIR = \"/mnt/data-ssd-1/data/supreme_corpus/processed_data/\"" ] }, { "cell_type": "markdown", "id": "22d78a0c", "metadata": {}, "source": [ "## Final analysis" ] }, { "cell_type": "code", "execution_count": 8, "id": "13254024", "metadata": {}, "outputs": [], "source": [ "slice_data = []\n", "with open(DATA_DIR + \"manifest.jsonl\") as f:\n", " for l in f:\n", " slice_data.append(json.loads(l))" ] }, { "cell_type": "code", "execution_count": 23, "id": "c36012b1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "163254 slices\n", "519.9 hours\n", "745 unique speakers\n", "2.0 seconds min slice len\n", "15.0 seconds max slice len\n", "11.5 seconds avg slice len\n" ] } ], "source": [ "print(\"{} slices\".format(len(slice_data)))\n", "v = round(np.sum([e[\"duration_s\"] for e in slice_data]) / 60 / 60, 1)\n", "print(\"{} hours\".format(v))\n", "v = len(set(funcy.flatten([[ee[\"speaker\"] for ee in e[\"segments\"]] for e in slice_data])))\n", "print(\"{} unique speakers\".format(v))\n", "v = round(np.min([e[\"duration_s\"] for e in slice_data]), 1)\n", "print(\"{} seconds min slice len\".format(v))\n", "v = round(np.max([e[\"duration_s\"] for e in slice_data]), 1)\n", "print(\"{} s max slice len\".format(v))\n", "v = round(np.mean([e[\"duration_s\"] for e in slice_data]), 1)\n", "print(\"{} s avg slice len\".format(v))" ] }, { "cell_type": "code", "execution_count": null, "id": "a7980b52", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f3d66ab5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "97e075ee", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "cc77429c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "71cc0801", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3af3b1e2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "dd305e71", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "0345ffde", "metadata": {}, "source": [ "## Intermediate investigate" ] }, { "cell_type": "code", "execution_count": 3, "id": "e94d8aab", "metadata": {}, "outputs": [], "source": [ "import tqdm\n", "import json\n", "import random\n", "import pandas as pd\n", "from suno_utils.utils.notebook import play_audio\n", "from suno_utils.utils.conversion import read_audio\n", "\n", "DATA_DIR = \"/mnt/data-ssd-1/data/supreme_corpus/processed_data/\"\n", "\n", "with open(DATA_DIR + \"slices_meta.jsonl\") as f:\n", " slice_data = [json.loads(e) for e in f]\n", " \n", "meta_df = pd.read_json(\"/mnt/data-ssd-1/data/supreme_corpus/raw_data/scrape_meta.jsonl\", lines=True)" ] }, { "cell_type": "code", "execution_count": 24, "id": "eefe5309", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3e7df95e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "ed64d913", "metadata": {}, "source": [ "## Find good slices" ] }, { "cell_type": "code", "execution_count": null, "id": "9f4cbdd2", "metadata": {}, "outputs": [], "source": [ "date_map = meta_df.set_index(\"uuid\")[\"date\"].dt.strftime(\"%Y/%m/%d\").to_dict()\n", "slice_texts = [\" \".join([s[\"text\"] for s in e[\"transcript\"]]) for e in slice_data]\n", "slice_dates = [date_map[e[\"uuid\"]] for e in slice_data]" ] }, { "cell_type": "markdown", "id": "db5a8881", "metadata": {}, "source": [ "#### by text" ] }, { "cell_type": "code", "execution_count": 86, "id": "2ce7a504", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "937 found\n" ] } ], "source": [ "q = r\"\\[laughter\\]\"\n", "# q = \"iphone\"\n", "\n", "idx_list = []\n", "for n, (text, date) in enumerate(zip(slice_texts, slice_dates)):\n", " if re.search(q, text.lower()):\n", " idx_list.append((n, int(date.replace(\"/\", \"0\"))))\n", "idx_list = [e for e, _ in sorted(idx_list, key=lambda x: x[-1], reverse=True)]\n", "print(len(idx_list), \"found\")" ] }, { "cell_type": "markdown", "id": "be725aba", "metadata": {}, "source": [ "#### by other" ] }, { "cell_type": "code", "execution_count": 90, "id": "710e6669", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3637 found\n" ] } ], "source": [ "# crosstalk\n", "idx_list = []\n", "for n, (text, date, e) in enumerate(zip(slice_texts, slice_dates, slice_data)):\n", " if len(e[\"transcript\"]) >= 5:\n", " idx_list.append((n, int(date.replace(\"/\", \"0\"))))\n", "idx_list = [e for e, _ in sorted(idx_list, key=lambda x: x[-1], reverse=True)]\n", "print(len(idx_list), \"found\")" ] }, { "cell_type": "code", "execution_count": 94, "id": "61ba7946", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "797 found\n" ] } ], "source": [ "# numbers\n", "idx_list = []\n", "for n, (text, date, e) in enumerate(zip(slice_texts, slice_dates, slice_data)):\n", " if len(re.findall(r\"[0-9]+\", text)) >= 4:\n", " idx_list.append((n, int(date.replace(\"/\", \"0\"))))\n", "idx_list = [e for e, _ in sorted(idx_list, key=lambda x: x[-1], reverse=True)]\n", "print(len(idx_list), \"found\")" ] }, { "cell_type": "code", "execution_count": 103, "id": "79236139", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "302 found\n" ] } ], "source": [ "# hesitation\n", "idx_list = []\n", "for n, (text, date, e) in enumerate(zip(slice_texts, slice_dates, slice_data)):\n", " if len(re.findall(r\"--\", text)) >= 3 and len(e[\"transcript\"]) == 1 and \"MS.\" in e[\"transcript\"][0][\"speaker\"]:\n", " idx_list.append((n, int(date.replace(\"/\", \"0\"))))\n", "idx_list = [e for e, _ in sorted(idx_list, key=lambda x: x[-1], reverse=True)]\n", "print(len(idx_list), \"found\")" ] }, { "cell_type": "markdown", "id": "bb3d5c19", "metadata": {}, "source": [ "#### show slices" ] }, { "cell_type": "code", "execution_count": 106, "id": "d3eec6e7", "metadata": {}, "outputs": [], "source": [ "# for idx in idx_list[30:40]:\n", "# slice_meta = slice_data[idx]\n", "# print(idx, \"-\", date_map[slice_meta[\"uuid\"]])\n", "# for e in slice_meta[\"transcript\"]:\n", "# print(\"{}: {}\".format(e[\"speaker\"], e[\"text\"]))\n", "# play_audio(DATA_DIR + slice_meta[\"path\"], sample_rate=44100)\n", "# print()" ] }, { "cell_type": "markdown", "id": "bb1061c9", "metadata": {}, "source": [ "#### found slices" ] }, { "cell_type": "code", "execution_count": null, "id": "60dc7df3", "metadata": {}, "outputs": [], "source": [ "# 50357, 10977 - quote (testing)\n", "# 51056 - apple\n", "# 115828 - netflix\n", "# 37865 - uber\n", "# 14549 - phone\n", "# 0 - normalization\n", "# 164354, 99990, 538, 84916 - medical\n", "# 144185, 37606, 6477, 16379 - funny\n", "# 4205, 4317, 4443, 4452, 4587 - covid\n", "# 171, 1157, 1356, 1475, 5578 - laughter\n", "# 37, 116, 231, 320, 345 - crosstalk\n", "# 670, 1596, 2300, 2545, 3891, 5860 - numbers\n", "# 47, 59, 147, 836, 1256, 9563, 10977 - hesitations" ] }, { "cell_type": "code", "execution_count": 123, "id": "37c54752", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "12357 - 2022/03/01\n", "MR. KULEWICZ: Well, no, we're -- we're paying -- we're paying based on $332, which is 125 percent of the Medicare rate. We pay 70 --\n", "JUSTICE SOTOMAYOR: No, no, no. My point is --\n", "MR. KULEWICZ: I'm sorry.\n", "JUSTICE SOTOMAYOR: -- if they are -- if they charge 5,000 per treatment,\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "slice_meta = slice_data[670]\n", "print(idx, \"-\", date_map[slice_meta[\"uuid\"]])\n", "for e in slice_meta[\"transcript\"]:\n", " print(\"{}: {}\".format(e[\"speaker\"], e[\"text\"]))\n", "play_audio(DATA_DIR + slice_meta[\"path\"], sample_rate=44100)\n", "print()" ] }, { "cell_type": "code", "execution_count": null, "id": "d222ef41", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "158fdd90", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0999984d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3da7773c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8d673d92", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "dcce2743", "metadata": {}, "source": [ "## investigate" ] }, { "cell_type": "markdown", "id": "b217fceb", "metadata": {}, "source": [ "#### investigate using slice meta" ] }, { "cell_type": "code", "execution_count": 27, "id": "c9599fdc", "metadata": { "scrolled": false }, "outputs": [], "source": [ "# # random sampling\n", "# random.seed(6006)\n", "# idx_list = list(range(len(slice_data)))\n", "# random.shuffle(idx_list)\n", "\n", "# for idx in idx_list[30:50]:\n", "# slice_meta = slice_data[idx]\n", "# print(idx, \"-\", date_map[slice_meta[\"uuid\"]])\n", "# for e in slice_meta[\"transcript\"]:\n", "# print(\"{}: {}\".format(e[\"speaker\"], e[\"text\"]))\n", "# play_audio(DATA_DIR + slice_meta[\"path\"], sample_rate=44100)\n", "# print()" ] }, { "cell_type": "code", "execution_count": 11, "id": "59054976", "metadata": {}, "outputs": [], "source": [ "# # shortest transcript\n", "# sorted_metas = sorted(\n", "# list(zip(list(range(len(slice_data))), slice_data)), \n", "# key=lambda x: len(\" \".join([e[\"text\"] for e in x[1][\"transcript\"]]))\n", "# )\n", "# for idx, slice_meta in sorted_metas[:10]:\n", "# print(idx)\n", "# for e in slice_meta[\"transcript\"]:\n", "# print(\"{}: {}\".format(e[\"speaker\"], e[\"text\"]))\n", "# play_audio(DATA_DIR + slice_meta[\"path\"], sample_rate=44100)\n", "# print()" ] }, { "cell_type": "markdown", "id": "84671104", "metadata": {}, "source": [ "#### use alignment dict" ] }, { "cell_type": "code", "execution_count": 12, "id": "62494362", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 0/781 [00:00\n", " \n", " Your browser does not support the audio element.\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# find ones where alignment had trouble (crosstalk)\n", "heureka = False\n", "for n, slice_meta in enumerate(slice_data):\n", " if n < 0:\n", " continue\n", " uuid = slice_meta[\"uuid\"]\n", " start_s, end_s = slice_meta[\"timestamp\"]\n", " df = alignment_dfs[uuid]\n", " _df = df[(df[\"start\"] >= start_s) & (df[\"end\"] <= end_s)]\n", " if len(_df.index) < 2:\n", " continue\n", " start_idx = _df.index[0]\n", " end_idx = _df.index[-1]\n", " n_found = 0\n", " for s in df.iloc[start_idx:end_idx][\"success\"].values:\n", " if s:\n", " n_found = 0\n", " else:\n", " n_found += 1\n", " if n_found >= 5:\n", " heureka = True\n", " if heureka:\n", " print(n)\n", " print(slice_meta[\"transcript\"])\n", " play_audio(DATA_DIR + slice_meta[\"path\"])\n", " break" ] }, { "cell_type": "code", "execution_count": null, "id": "104951e5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bdf4a126", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "640f1235", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "0d8bb665", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 272, "id": "88854dac", "metadata": {}, "outputs": [], "source": [ "norm_texts = [e[\"text_normalized\"] for e in slice_data if len(e[\"text_normalized\"]) > 0]" ] }, { "cell_type": "code", "execution_count": 277, "id": "22eb13aa", "metadata": {}, "outputs": [], "source": [ "# find 5 grams using scikit\n", "from sklearn.feature_extraction.text import CountVectorizer" ] }, { "cell_type": "code", "execution_count": 294, "id": "f8b689a8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "39178\n" ] } ], "source": [ "clf = CountVectorizer(analyzer='word', ngram_range=(5, 5), min_df=3)\n", "count_matrix = clf.fit_transform(norm_texts)\n", "n_count_by_phrase = np.array(count_matrix.sum(axis=0)).squeeze()\n", "print(len(clf.vocabulary_))\n", "df = pd.Series(clf.vocabulary_).to_frame(\"vocab_idx\").rename_axis(\"phrase\").reset_index()\n", "df[\"n_count\"] = df[\"vocab_idx\"].map({n: s for n, s in enumerate(n_count_by_phrase)})\n", "df = df.sort_values(\"n_count\", ascending=False).reset_index(drop=True)" ] }, { "cell_type": "code", "execution_count": 312, "id": "cf557538", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(39178, 3)" ] }, "execution_count": 312, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.shape" ] }, { "cell_type": "code", "execution_count": 310, "id": "40063126", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
phrasevocab_idxn_count
0may it please the court154481037
1and may it please the11931015
2chief justice and may it5408993
3justice and may it please14612992
4it seems to me that14110536
5mister chief justice and may15677508
6mr chief justice and may15794487
7if you look at the10477335
8thank you mister chief justice21871319
9counsel the case is submitted5910305
\n", "
" ], "text/plain": [ " phrase vocab_idx n_count\n", "0 may it please the court 15448 1037\n", "1 and may it please the 1193 1015\n", "2 chief justice and may it 5408 993\n", "3 justice and may it please 14612 992\n", "4 it seems to me that 14110 536\n", "5 mister chief justice and may 15677 508\n", "6 mr chief justice and may 15794 487\n", "7 if you look at the 10477 335\n", "8 thank you mister chief justice 21871 319\n", "9 counsel the case is submitted 5910 305" ] }, "execution_count": 310, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head(10)" ] }, { "cell_type": "code", "execution_count": 311, "id": "e986f01d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
phrasevocab_idxn_count
39168mean we re we re155823
39169fairness integrity and public reputation78633
39170judge can look at the144503
39171you remand it to the385003
39172do what they re supposed67273
39173think that true your honor301123
39174don think that don think72533
39175when you think about what360513
39176that the evidence in the232573
39177and you say that not24563
\n", "
" ], "text/plain": [ " phrase vocab_idx n_count\n", "39168 mean we re we re 15582 3\n", "39169 fairness integrity and public reputation 7863 3\n", "39170 judge can look at the 14450 3\n", "39171 you remand it to the 38500 3\n", "39172 do what they re supposed 6727 3\n", "39173 think that true your honor 30112 3\n", "39174 don think that don think 7253 3\n", "39175 when you think about what 36051 3\n", "39176 that the evidence in the 23257 3\n", "39177 and you say that not 2456 3" ] }, "execution_count": 311, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.tail(10)" ] }, { "cell_type": "code", "execution_count": null, "id": "b5faa4b7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e4c3b0be", "metadata": {}, "outputs": [], "source": [ "# we actually need to do this with gentle word based timestamps that are reliable and make streaks" ] }, { "cell_type": "code", "execution_count": null, "id": "50ade2d8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "31708a95", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "662b556c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a3ee101b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "079064e2", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }