{ "cells": [ { "cell_type": "code", "execution_count": 3, "id": "chicken-moral", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 4, "id": "consistent-metallic", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ['CUDA_VISIBLE_DEVICES'] = ''" ] }, { "cell_type": "code", "execution_count": 5, "id": "according-tuner", "metadata": {}, "outputs": [], "source": [ "import re\n", "import pandas as pd\n", "import numpy as np\n", "import librosa\n", "from scipy.io import wavfile\n", "import librosa.display\n", "from pydub import AudioSegment\n", "from IPython.display import display_html\n", "import torch\n", "\n", "\n", "def _convert_array(arr, force_int=False, force_float=False):\n", " if isinstance(arr, torch.Tensor):\n", " arr = arr.cpu().detach().numpy()\n", " assert(len(arr.shape) in (1, 2))\n", " if len(arr.shape) == 2:\n", " assert(arr.shape[0] == 1)\n", " arr = arr[0]\n", " if force_int and \"float\" in str(arr.dtype):\n", " arr = np.floor(arr.copy() * 32768).astype(np.int16)\n", " if force_float and \"int\" in str(arr.dtype):\n", " arr = arr.copy().astype(np.float32) / 32768\n", " return arr\n", "\n", "\n", "def plot_ft(arr, window_size=1024, hop_length=512, n_mels=128):\n", " arr = _convert_array(arr, force_float=True)\n", " window = np.hanning(window_size)\n", " stft= librosa.core.spectrum.stft(arr, n_fft=window_size, hop_length=hop_length, window=window)\n", " out = 2 * np.abs(stft) / np.sum(window)\n", " data = librosa.amplitude_to_db(out, ref=np.max)\n", "\n", " plt.figure(figsize=(16, 10))\n", " librosa.display.specshow(data, x_axis='time', y_axis='mel', sr=16000)\n", " plt.set_cmap('afmhot')\n", "\n", "\n", "def play_array(arr):\n", " arr = _convert_array(arr, force_int=True)\n", " display_html(AudioSegment(\n", " arr.tobytes(), \n", " frame_rate=16000,\n", " sample_width=2, \n", " channels=1\n", " ))" ] }, { "cell_type": "code", "execution_count": 6, "id": "presidential-fourth", "metadata": {}, "outputs": [], "source": [ "GIGA_DIR = \"/data-ssd-3/georg/data/gigaspeech/\"\n", "\n", "df = pd.read_json(GIGA_DIR + \"GigaSpeech.json\")\n", "df = pd.DataFrame.from_dict(df['audios'].tolist())" ] }, { "cell_type": "code", "execution_count": 18, "id": "current-brooklyn", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "youtube 21591\n", "podcast 15444\n", "audiobook 1096\n", "Name: source, dtype: int64" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df['source'].value_counts()" ] }, { "cell_type": "code", "execution_count": 27, "id": "surprised-hollywood", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Education 5937\n", "News and Politics 3849\n", "Entertainment 3034\n", "People and Blogs 1972\n", "Science and Technology 1731\n", "Howto and Style 1552\n", "Comedy 566\n", "Nonprofits and Activism 559\n", "Film and Animation 545\n", "Gaming 361\n", "Sports 360\n", "Music 301\n", "N/A 269\n", "Autos and Vehicles 188\n", "Travel and Events 183\n", "Pets and Animals 65\n", "Name: category, dtype: int64" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[df['source'] == 'youtube']['category'].value_counts()" ] }, { "cell_type": "code", "execution_count": 26, "id": "recovered-height", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "News and Politics 11997\n", "Science and Technology 895\n", "Kids and Family 607\n", "Arts 413\n", "People and Blogs 385\n", "Crime 333\n", "Health and Fitness 330\n", "Society and Culture 142\n", "History 126\n", "Education 75\n", "Business 53\n", "Travel and Events 32\n", "Leisure 10\n", "Music 6\n", "Name: category, dtype: int64" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[df['source'] == 'podcast']['category'].value_counts()" ] }, { "cell_type": "code", "execution_count": 17, "id": "likely-chicago", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
titleurlpathaidsourceformatsample_ratecodecchannelsmd5durationsizespeakercategorysubsetssegments
0Check Cashing Storeshttps://99percentinvisible.org/episode/episode...audio/podcast/P0001/POD0000000001.opusPOD0000000001podcastopus16000s16le1a1eb19f5bc68d4af71f65a3e8d700343629.9312555888N/AArts[{XL}, {L}][{'sid': 'POD0000000001_S0000008', 'speaker': ...
1Cold War Kidshttps://99percentinvisible.org/episode/cold-wa...audio/podcast/P0001/POD0000000002.opusPOD0000000002podcastopus16000s16le122aff12d1e852338593a6393ceb626041735.5117145217N/AArts[{XL}][{'sid': 'POD0000000002_S0000004', 'speaker': ...
\n", "
" ], "text/plain": [ " title url \\\n", "0 Check Cashing Stores https://99percentinvisible.org/episode/episode... \n", "1 Cold War Kids https://99percentinvisible.org/episode/cold-wa... \n", "\n", " path aid source format \\\n", "0 audio/podcast/P0001/POD0000000001.opus POD0000000001 podcast opus \n", "1 audio/podcast/P0001/POD0000000002.opus POD0000000002 podcast opus \n", "\n", " sample_rate codec channels md5 duration \\\n", "0 16000 s16le 1 a1eb19f5bc68d4af71f65a3e8d700343 629.931 \n", "1 16000 s16le 1 22aff12d1e852338593a6393ceb62604 1735.511 \n", "\n", " size speaker category subsets \\\n", "0 2555888 N/A Arts [{XL}, {L}] \n", "1 7145217 N/A Arts [{XL}] \n", "\n", " segments \n", "0 [{'sid': 'POD0000000001_S0000008', 'speaker': ... \n", "1 [{'sid': 'POD0000000002_S0000004', 'speaker': ... " ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head(2)" ] }, { "cell_type": "code", "execution_count": 201, "id": "comprehensive-postage", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "title The_Indicator_from_Planet_Money_Meet_Our_Junk_...\n", "url https://play.podtrac.com/npr-510325/edge1.pod....\n", "path audio/podcast/P0044/POD0000004354.opus\n", "aid POD0000004354\n", "source podcast\n", "format opus\n", "sample_rate 16000\n", "codec s16le\n", "channels 1\n", "md5 d00d5cae92a708c83555be8af1aaa375\n", "duration 551.549\n", "size 2192242\n", "speaker N/A\n", "category News and Politics\n", "subsets [{XL}]\n", "segments [{'sid': 'POD0000004354_S0000002', 'speaker': ...\n", "Name: 4114, dtype: object" ] }, "execution_count": 201, "metadata": {}, "output_type": "execute_result" } ], "source": [ "row = df[df['source'] == \"podcast\"].sample(1).iloc[0]\n", "# row = df[(df['source'] == \"youtube\") & (df['category'] == \"Gaming\")].sample(1).iloc[0]\n", "out = AudioSegment.from_file(GIGA_DIR + row[\"path\"], codec=\"opus\")\n", "row" ] }, { "cell_type": "code", "execution_count": 204, "id": "helpful-stick", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[(38.17, 40.12),\n", " (40.12, 42.73),\n", " (42.79, 58.4),\n", " (58.46, 64.79),\n", " (64.79, 76.73),\n", " (106.0, 109.5),\n", " (113.95, 119.89),\n", " (120.01, 130.64),\n", " (130.67, 135.59),\n", " (154.64, 160.6),\n", " (160.6, 162.97),\n", " (172.25, 186.19),\n", " (186.34, 194.29),\n", " (194.29, 196.3),\n", " (196.54, 200.57),\n", " (200.57, 209.3),\n", " (269.29, 271.57),\n", " (271.87, 272.71),\n", " (350.47, 366.59),\n", " (366.92, 373.64),\n", " (373.67, 381.45),\n", " (381.45, 384.27),\n", " (384.27, 397.05),\n", " (408.67, 411.16),\n", " (411.16, 415.78),\n", " (415.78, 417.67),\n", " (434.15, 437.99),\n", " (440.12, 441.71),\n", " (449.45, 466.02),\n", " (466.98, 472.59),\n", " (473.04, 479.71),\n", " (479.71, 481.31),\n", " (481.31, 485.68),\n", " (500.44, 502.8),\n", " (518.98, 532.65),\n", " (534.45, 542.22)]" ] }, "execution_count": 204, "metadata": {}, "output_type": "execute_result" } ], "source": [ "[(e[\"begin_time\"], e[\"end_time\"]) for e in row[\"segments\"]]" ] }, { "cell_type": "code", "execution_count": 206, "id": "bulgarian-pavilion", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "seg = (\n", " 77,\n", " 105,\n", ")\n", "display_html(out[int(seg[0] * 1000):int(seg[1] * 1000)])" ] }, { "cell_type": "code", "execution_count": 202, "id": "stylish-commitment", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "{'sid': 'POD0000004354_S0000044',\n", " 'speaker': 'N/A',\n", " 'begin_time': 449.45,\n", " 'end_time': 466.02,\n", " 'text_raw': '',\n", " 'text_tn': \"SOMETIMES A COMPANY CAN BE SO DOWN AND OUT THAT THEY JUST ELECT TO GO BANKRUPT AND WIPE OUT ALL OF THEIR CREDIT OR WIPE OUT ALL OF THEIR BONDS OH BUT YOU KNOW KEEP IN MIND LIKE MOST OF THESE COMPANIES ARE RUN BY FOLKS THAT WANT TO STAY EMPLOYED BUT DON'T NECESSARILY WANT TO GO BANKRUPT \",\n", " 'subsets': ['{XL}']}" ] }, "execution_count": 202, "metadata": {}, "output_type": "execute_result" } ], "source": [ "seg = random.choice(row[\"segments\"])\n", "display_html(out[int(seg[\"begin_time\"] * 1000):int(seg[\"end_time\"] * 1000)])\n", "seg" ] }, { "cell_type": "code", "execution_count": null, "id": "instant-detail", "metadata": {}, "outputs": [], "source": [ "# PODCASTS\n", "# some have music in the background\n", "# TODO: can we identify and remove clips with music?\n", "# sometimes phone calls guests or recordings are played\n", "# TODO: can we distinguish quality using pesq/stoi/frequency coverage?\n", "\n", "# YOUTUBE\n", "# most have low quality, background noise or echo\n", "# still bias towards 'formal' videos. probably because they have CC\n", "\n", "# AUDIOBOOKS\n", "# weird subject\n", "# read speech\n", "# not fantastic quality" ] }, { "cell_type": "code", "execution_count": null, "id": "impressive-bulgaria", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "stable-bonus", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "thrown-albert", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "educational-temperature", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.12" } }, "nbformat": 4, "nbformat_minor": 5 }