{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "3f85d5b3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 2, "id": "2cc271ef", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"3\"\n", "os.environ[\"OMP_NUM_THREADS\"] = \"1\" \n", "os.environ[\"MKL_NUM_THREADS\"] = \"1\" \n", "os.environ[\"WANDB_CONSOLE\"] = \"off\"\n", "os.environ[\"WAND_DISABLE\"] = \"true\"\n", "import wandb\n", "wandb.require(\"service\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "9b0dfd1f", "metadata": {}, "outputs": [], "source": [ "import tqdm\n", "import json\n", "import pandas as pd\n", "\n", "PODCAST_DATA_DIR = \"/mnt/data-ssd-1/data/podcasts/\"\n", "CUSTOM_DATA_DIR = \"/mnt/data-ssd-1/data/custom/lang_id-en_tl/\"" ] }, { "cell_type": "code", "execution_count": null, "id": "dd132d0c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "44cc11e2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "43fb8540", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "98e163e9", "metadata": {}, "source": [ "### find podcast subsets for en/tl" ] }, { "cell_type": "code", "execution_count": 12, "id": "fe2e30dc", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.podcasts import load_podcast_db" ] }, { "cell_type": "code", "execution_count": 13, "id": "97f38e83", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1858426 podcasts found\n" ] } ], "source": [ "podcast_df = load_podcast_db(PODCAST_DATA_DIR + \"meta/podcastindex_feeds.db\")\n", "podcast_df = podcast_df[\n", " (podcast_df[\"host\"] == \"anchor.fm\") &\n", " (podcast_df[\"generator\"] == \"Anchor Podcasts\")\n", "]\n", "print(len(podcast_df), \"podcasts found\")" ] }, { "cell_type": "code", "execution_count": 15, "id": "b1b249a9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "584022 english podcasts found\n", "1219 tagalog podcasts found\n" ] } ], "source": [ "english_df = podcast_df[\n", " (podcast_df[\"language\"].str[:2] == \"en\") &\n", " (podcast_df[\"episode_count\"] >= 2) &\n", " (podcast_df[\"last_update\"] >= \"2021-01-01\") \n", "]\n", "english_df = english_df.sample(frac=1.0, random_state=6006).reset_index(drop=True)\n", "tagalog_df = podcast_df[\n", " (podcast_df[\"language\"].str[:2] == \"tl\") &\n", " (podcast_df[\"episode_count\"] >= 2) &\n", " (podcast_df[\"last_update\"] >= \"2021-01-01\") \n", "]\n", "tagalog_df = tagalog_df.sample(frac=1.0, random_state=6006).reset_index(drop=True)\n", "print(len(english_df), \"english podcasts found\")\n", "print(len(tagalog_df), \"tagalog podcasts found\")" ] }, { "cell_type": "markdown", "id": "0d7f91b7", "metadata": {}, "source": [ "### download episodes" ] }, { "cell_type": "code", "execution_count": 16, "id": "74e56402", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.podcasts import load_rss_feed" ] }, { "cell_type": "code", "execution_count": 17, "id": "487ac150", "metadata": {}, "outputs": [], "source": [ "country_podcasts_df = pd.read_csv(\"/home/georg/notebooks/datasets/podcasts/country_podcasts.csv\")\n", "phl_podcasts_df = country_podcasts_df[country_podcasts_df[\"country\"] == \"Philippines\"]\n", "phl_podcasts_df = phl_podcasts_df.sample(frac=1.0, random_state=6006).reset_index(drop=True)" ] }, { "cell_type": "code", "execution_count": 18, "id": "3f3dc43b", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "499it [00:17, 29.33it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "500 english (phl accent) found\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "english_filipino_ids = []\n", "for _, row in tqdm.tqdm(phl_podcasts_df.iterrows()):\n", " feed_fp = f\"{PODCAST_DATA_DIR}bulk_rss_feeds/raw_files/{row['id']}.feed\"\n", " if not os.path.exists(feed_fp):\n", " continue\n", " try:\n", " rss_feed = load_rss_feed(feed_fp)\n", " except ValueError:\n", " continue\n", " terms = set([e.get(\"term\", \"\").strip().lower() for e in rss_feed.get(\"feed\", {}).get(\"tags\", [])])\n", " if \"music\" in terms:\n", " continue\n", " english_filipino_ids.append(row[\"id\"])\n", " if len(english_filipino_ids) == 500:\n", " break\n", "print(len(english_filipino_ids), \"english (phl accent) found\")" ] }, { "cell_type": "code", "execution_count": 20, "id": "08090217", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "532it [00:22, 23.78it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "500 english found\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "english_global_ids = []\n", "for _, row in tqdm.tqdm(english_df.iterrows()):\n", " feed_fp = f\"{PODCAST_DATA_DIR}bulk_rss_feeds/raw_files/{row['id']}.feed\"\n", " if not os.path.exists(feed_fp):\n", " continue\n", " try:\n", " rss_feed = load_rss_feed(feed_fp)\n", " except ValueError:\n", " continue\n", " terms = set([e.get(\"term\", \"\").strip().lower() for e in rss_feed.get(\"feed\", {}).get(\"tags\", [])])\n", " if \"music\" in terms:\n", " continue\n", " english_global_ids.append(row[\"id\"])\n", " if len(english_global_ids) == 500:\n", " break\n", "print(len(english_global_ids), \"english found\")" ] }, { "cell_type": "code", "execution_count": 21, "id": "716088f8", "metadata": {}, "outputs": [], "source": [ "english_ids = english_global_ids + english_filipino_ids" ] }, { "cell_type": "code", "execution_count": 22, "id": "26c92137", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "1046it [00:13, 78.21it/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ "1000 tagalog found\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "tagalog_ids = []\n", "for _, row in tqdm.tqdm(tagalog_df.iterrows()):\n", " feed_fp = f\"{PODCAST_DATA_DIR}bulk_rss_feeds/raw_files/{row['id']}.feed\"\n", " if not os.path.exists(feed_fp):\n", " continue\n", " try:\n", " rss_feed = load_rss_feed(feed_fp)\n", " except ValueError:\n", " continue\n", " terms = set([e.get(\"term\", \"\").strip().lower() for e in rss_feed.get(\"feed\", {}).get(\"tags\", [])])\n", " if \"music\" in terms:\n", " continue\n", " tagalog_ids.append(row[\"id\"])\n", " if len(tagalog_ids) == 1000:\n", " break\n", "print(len(tagalog_ids), \"tagalog found\")" ] }, { "cell_type": "code", "execution_count": null, "id": "d39e648e", "metadata": {}, "outputs": [], "source": [ "# get 1000 podcasts with 5 episodes each" ] }, { "cell_type": "code", "execution_count": 24, "id": "1e370f58", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'976036;3771666'" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\";\".join([str(e) for e in (english_ids + tagalog_ids)[:2]])" ] }, { "cell_type": "markdown", "id": "616db8d9", "metadata": {}, "source": [ "python /home/georg/code/glockenspiel/suno_utils/suno_utils/scripts/fetch_podcasts.py \\\n", " --podcast_db_filepath=\"/mnt/data-ssd-1/data/podcasts/meta/podcastindex_feeds.db\" \\\n", " --podcast_ids=\"976036;3771666\" \\\n", " --output_dir=\"/mnt/data-ssd-1/data/podcasts/audio\" \\\n", " --last_n_episodes=2" ] }, { "cell_type": "code", "execution_count": 26, "id": "8f16f60b", "metadata": {}, "outputs": [], "source": [ "podcast_ids = {\n", " \"english_ids\": english_ids,\n", " \"tagalog_ids\": tagalog_ids,\n", "}\n", "with open(CUSTOM_DATA_DIR + \"meta/01_podcast_ids.json\", \"w\") as f:\n", " json.dump(podcast_ids, f)" ] }, { "cell_type": "markdown", "id": "62cabd90", "metadata": {}, "source": [ "### (optional) listen" ] }, { "cell_type": "code", "execution_count": 69, "id": "74b86a70", "metadata": {}, "outputs": [], "source": [ "from suno_utils.audio import Audio\n", "\n", "Audio.from_file(\"/mnt/data-ssd-1/data/podcasts/audio/866122/9998e502-dd11-4e73-9149-ab8b4350914a.m4a\").play()" ] }, { "cell_type": "code", "execution_count": null, "id": "1f6992de", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "aa3d642a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "ded15a31", "metadata": {}, "source": [ "### make custom data slices" ] }, { "cell_type": "code", "execution_count": 72, "id": "97a1dbd0", "metadata": {}, "outputs": [], "source": [ "import random\n", "import shutil\n", "import funcy\n", "import multiprocessing\n", "import uuid\n", "from suno_utils.audio import Audio\n", "from suno_utils.audio.conversion import get_audio_properties\n", "from suno_utils.utils.podcasts import get_episode_audio_url\n", "from suno_utils.web.harvest import get_file_ext" ] }, { "cell_type": "code", "execution_count": 28, "id": "35caf93a", "metadata": {}, "outputs": [], "source": [ "with open(CUSTOM_DATA_DIR + \"meta/01_podcast_ids.json\") as f:\n", " podcast_ids = json.load(f)\n", "english_ids = podcast_ids[\"english_ids\"]\n", "tagalog_ids = podcast_ids[\"tagalog_ids\"]\n", "random.seed(6006)\n", "random.shuffle(english_ids)\n", "random.shuffle(tagalog_ids)" ] }, { "cell_type": "code", "execution_count": 52, "id": "098785fd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "180 english (dev) episodes\n", "184 tagalog (dev) episodes\n", "3497 english (train) episodes\n", "3073 tagalog (train) episodes\n" ] } ], "source": [ "MIN_EPISODE_DURATION_S = 5 * 60\n", "\n", "def _get_episode_urls(rss_feed):\n", " audio_urls = []\n", " for feed_entry in rss_feed[\"entries\"]:\n", " audio_url = get_episode_audio_url(feed_entry)\n", " audio_urls.append((feed_entry[\"safe_id\"], audio_url))\n", " return audio_urls\n", "\n", "\n", "def _collect_filepaths(podcast_ids, last_n_episodes=5):\n", " filepaths = []\n", " for podcast_id in podcast_ids:\n", " podcast_dir = f\"{PODCAST_DATA_DIR}audio/{podcast_id}/\"\n", " if not os.path.exists(podcast_dir):\n", " continue\n", " feed_fp = f\"{podcast_dir}raw_feed.bytes\"\n", " if not os.path.exists(feed_fp):\n", " continue\n", " try:\n", " rss_feed = load_rss_feed(feed_fp)\n", " except ValueError:\n", " continue\n", " episode_audio_urls = _get_episode_urls(rss_feed)\n", " if last_n_episodes > 0:\n", " episode_audio_urls = episode_audio_urls[:last_n_episodes]\n", " for episode_id, episode_audio_url in episode_audio_urls:\n", " file_ext = get_file_ext(episode_audio_url)\n", " episode_filepath = os.path.join(podcast_dir, episode_id + f\".{file_ext}\")\n", " if not os.path.exists(episode_filepath):\n", " continue\n", " duration_s = get_audio_properties(episode_filepath)[\"duration_s\"]\n", " if duration_s < MIN_EPISODE_DURATION_S:\n", " continue\n", " filepaths.append(episode_filepath)\n", " return filepaths\n", "\n", "english_fps_dev = _collect_filepaths(english_ids[-50:])\n", "print(len(english_fps_dev), \"english (dev) episodes\")\n", "tagalog_fps_dev = _collect_filepaths(tagalog_ids[-50:])\n", "print(len(tagalog_fps_dev), \"tagalog (dev) episodes\")\n", "english_fps_tr = _collect_filepaths(english_ids[:-50])\n", "print(len(english_fps_tr), \"english (train) episodes\")\n", "tagalog_fps_tr = _collect_filepaths(tagalog_ids[:-50])\n", "print(len(tagalog_fps_tr), \"tagalog (train) episodes\")\n", "# 180 english (dev) episodes\n", "# 184 tagalog (dev) episodes\n", "# 3497 english (train) episodes\n", "# 3073 tagalog (train) episodes" ] }, { "cell_type": "code", "execution_count": 106, "id": "cc68e8f0", "metadata": {}, "outputs": [], "source": [ "# shutil.rmtree(CUSTOM_DATA_DIR + \"train\", ignore_errors=True)\n", "# shutil.rmtree(CUSTOM_DATA_DIR + \"dev\", ignore_errors=True)" ] }, { "cell_type": "code", "execution_count": 107, "id": "7a548e67", "metadata": {}, "outputs": [], "source": [ "os.makedirs(CUSTOM_DATA_DIR + \"train\")\n", "os.makedirs(CUSTOM_DATA_DIR + \"dev\")" ] }, { "cell_type": "code", "execution_count": 110, "id": "3e03576f", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████| 1/1 [02:18<00:00, 138.92s/it]\n", "100%|███████████████████████████████████████████████████████████████████████| 18/18 [40:21<00:00, 134.53s/it]\n", "100%|███████████████████████████████████████████████████████████████████████| 16/16 [33:41<00:00, 126.34s/it]\n" ] } ], "source": [ "EPISODE_OFFSET_S = 1 * 60\n", "EPISODE_MAX_DURATION_S = 30 * 60\n", "MIN_SEGMENT_DURATION_S = 0.5\n", "MAX_SEGMENT_DURATION_S = 5.0\n", "\n", "random.seed(6006)\n", "\n", "def _get_random_segment_durations(\n", " audio_duration_s,\n", " min_segment_duration_s=MIN_SEGMENT_DURATION_S, \n", " max_segment_duration_s=MAX_SEGMENT_DURATION_S,\n", "):\n", " segment_durations_s = []\n", " total_duration_s = 0\n", " while True:\n", " segment_duration_s = (\n", " min_segment_duration_s + random.random() * (max_segment_duration_s - min_segment_duration_s)\n", " )\n", " total_duration_s += segment_duration_s\n", " if total_duration_s > audio_duration_s:\n", " break\n", " segment_durations_s.append((total_duration_s-segment_duration_s, total_duration_s))\n", " return segment_durations_s\n", " \n", "def _get_audio_slice_info(audio_duration_s, from_s=EPISODE_OFFSET_S, to_s=EPISODE_OFFSET_S+EPISODE_MAX_DURATION_S):\n", " if audio_duration_s <= from_s:\n", " return []\n", " valid_duration_s = np.min([audio_duration_s - from_s, to_s - from_s])\n", " segment_durations_s = _get_random_segment_durations(valid_duration_s)\n", " segments_meta = [\n", " {\n", " \"source_audio_offset_s\": from_s + rel_from_s,\n", " \"duration_s\": rel_to_s - rel_from_s,\n", " \"uid\": str(uuid.uuid4()),\n", " } \n", " for rel_from_s, rel_to_s in segment_durations_s\n", " ]\n", " return segments_meta\n", "\n", "def _write_segments(audio_filepath, set_type=\"train\", lang=\"en\"):\n", " episode_id = re.sub(r\"\\..*?$\", \"\", audio_filepath.split(\"/\")[-1])\n", " podcast_id = audio_filepath.split(\"/\")[-2]\n", " audio = Audio.from_file(audio_filepath, sample_rate=16_000).convert(16_000, 2, 1)\n", " segments_info = _get_audio_slice_info(audio.duration_s)\n", " segment_rel_dir = os.path.join(set_type, lang, podcast_id)\n", " segment_abs_dir = os.path.join(CUSTOM_DATA_DIR, segment_rel_dir)\n", " os.makedirs(segment_abs_dir, exist_ok=True)\n", " for segment_info in segments_info:\n", " filename = segment_info[\"uid\"] + \".wav\"\n", " segment_fp = os.path.join(segment_abs_dir, filename)\n", " from_s = segment_info[\"source_audio_offset_s\"]\n", " to_s = segment_info[\"source_audio_offset_s\"] + segment_info[\"duration_s\"]\n", " audio.get_slice(from_s, to_s).to_wav(segment_fp)\n", " segment_info[\"uri\"] = os.path.join(segment_rel_dir, filename)\n", " segment_info[\"episode_id\"] = episode_id\n", " segment_info[\"podcast_id\"] = podcast_id\n", " return segments_info\n", " \n", "def _mp_write_segments(audio_filepaths, set_type=\"train\", lang=\"en\", chunksize=200):\n", " func = funcy.partial(_write_segments, set_type=set_type, lang=lang)\n", " p = multiprocessing.Pool(20)\n", " out = []\n", " n_chunks = len(audio_filepaths) // chunksize + 1\n", " for chunk_filepaths in tqdm.tqdm(funcy.chunks(chunksize, audio_filepaths), total=n_chunks):\n", " out.extend(p.map(func, chunk_filepaths, chunksize=2))\n", " segments_meta = list(funcy.flatten(out))\n", " p.close()\n", " p.join()\n", " return segments_meta\n", "\n", "# ~1.2h total for 7k episodes with 20 cores, \n", "english_segments_meta_dev = _mp_write_segments(english_fps_dev, set_type=\"dev\", lang=\"en\")\n", "tagalog_segments_meta_dev = _mp_write_segments(tagalog_fps_dev, set_type=\"dev\", lang=\"tl\")\n", "english_segments_meta_tr = _mp_write_segments(english_fps_tr, set_type=\"train\", lang=\"en\")\n", "tagalog_segments_meta_tr = _mp_write_segments(tagalog_fps_tr, set_type=\"train\", lang=\"tl\")" ] }, { "cell_type": "code", "execution_count": 111, "id": "de411226", "metadata": {}, "outputs": [], "source": [ "raw_segments_meta = {\n", " \"english_segments_meta_tr\": english_segments_meta_tr,\n", " \"tagalog_segments_meta_tr\": tagalog_segments_meta_tr,\n", " \"english_segments_meta_dev\": english_segments_meta_dev,\n", " \"tagalog_segments_meta_dev\": tagalog_segments_meta_dev,\n", "}\n", "with open(CUSTOM_DATA_DIR + \"meta/02_raw_segments_meta.json\", \"w\") as f:\n", " json.dump(raw_segments_meta, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "a901bf38", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a0fb549a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "dba41f5f", "metadata": {}, "source": [ "### Run Asr" ] }, { "cell_type": "code", "execution_count": 4, "id": "6f257aac", "metadata": {}, "outputs": [], "source": [ "import random\n", "import pathlib\n", "import funcy\n", "from suno_utils.tasks.asr import transcribe\n", "from suno_utils.utils.notebook import Audio\n", "from scipy.special import softmax\n", "\n", "CUSTOM_DATA_DIR = \"/mnt/data-ssd-1/data/custom/lang_id-en_tl/\"" ] }, { "cell_type": "code", "execution_count": 5, "id": "78cf6f7f", "metadata": {}, "outputs": [], "source": [ "with open(CUSTOM_DATA_DIR + \"meta/02_raw_segments_meta.json\") as f:\n", " raw_segments_meta = json.load(f)\n", "english_segments_meta_dev = raw_segments_meta[\"english_segments_meta_dev\"]\n", "tagalog_segments_meta_dev = raw_segments_meta[\"tagalog_segments_meta_dev\"]\n", "english_segments_meta_tr = raw_segments_meta[\"english_segments_meta_tr\"]\n", "tagalog_segments_meta_tr = raw_segments_meta[\"tagalog_segments_meta_tr\"]" ] }, { "cell_type": "code", "execution_count": 6, "id": "8bbeb318", "metadata": {}, "outputs": [], "source": [ "def _get_en_p_from_logits(logits):\n", " # values > 0.1 and < 0.8 seem like foreign language\n", " if len(logits) == 0:\n", " return 0.0\n", " logits = softmax(logits, axis=1)\n", " s = np.where(logits.argmax(axis=1) != 128, logits.max(axis=1), np.nan)\n", " s = s[np.logical_not(np.isnan(s))]\n", " if len(s) == 0:\n", " return 0.0\n", " return s.mean()\n", "\n", "def _add_en_p(segments_meta, chunk_size=100_000):\n", " en_probs = []\n", " n_chunks = len(segments_meta) // chunk_size + 1\n", " for n, metas_chunk in enumerate(funcy.chunks(chunk_size, segments_meta)):\n", " print(f\"processing chunk {n+1}/{n_chunks}...\")\n", " fp_chunk = [os.path.join(CUSTOM_DATA_DIR, e[\"uri\"]) for e in metas_chunk]\n", " _, logits_list = transcribe(fp_chunk, mode=\"logits\", batch_size=64, n_workers=0)\n", " en_probs_chunk = [_get_en_p_from_logits(logits) for logits in logits_list]\n", " en_probs.extend(en_probs_chunk)\n", " for en_p, segment_meta in zip(en_probs, segments_meta):\n", " segment_meta[\"en_p\"] = float(en_p) # json serializable" ] }, { "cell_type": "code", "execution_count": null, "id": "73e35652", "metadata": {}, "outputs": [], "source": [ "# ~2h for at 1 gpu with n_workers=0\n", "_add_en_p(english_segments_meta_dev)\n", "_add_en_p(tagalog_segments_meta_dev)\n", "_add_en_p(english_segments_meta_tr)\n", "_add_en_p(tagalog_segments_meta_tr)" ] }, { "cell_type": "code", "execution_count": 16, "id": "e362c813", "metadata": {}, "outputs": [], "source": [ "raw_post_asr_segments_meta = {\n", " \"english_segments_meta_tr\": english_segments_meta_tr,\n", " \"tagalog_segments_meta_tr\": tagalog_segments_meta_tr,\n", " \"english_segments_meta_dev\": english_segments_meta_dev,\n", " \"tagalog_segments_meta_dev\": tagalog_segments_meta_dev,\n", "}\n", "with open(CUSTOM_DATA_DIR + \"meta/03_raw_post_asr_segments_meta.json\", \"w\") as f:\n", " json.dump(raw_post_asr_segments_meta, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "04eee574", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "47643129", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5b11910b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "eef57f94", "metadata": {}, "source": [ "## Use asr to filter" ] }, { "cell_type": "code", "execution_count": 18, "id": "13c51db9", "metadata": {}, "outputs": [], "source": [ "import random\n", "import pathlib\n", "import funcy\n", "from suno_utils.tasks.asr import transcribe\n", "from suno_utils.utils.notebook import Audio\n", "from scipy.special import softmax\n", "\n", "CUSTOM_DATA_DIR = \"/mnt/data-ssd-1/data/custom/lang_id-en_tl/\"" ] }, { "cell_type": "code", "execution_count": 20, "id": "79e8fc69", "metadata": {}, "outputs": [], "source": [ "with open(CUSTOM_DATA_DIR + \"meta/03_raw_post_asr_segments_meta.json\") as f:\n", " raw_post_asr_segments_meta = json.load(f)" ] }, { "cell_type": "code", "execution_count": 44, "id": "f526486b", "metadata": {}, "outputs": [], "source": [ "import shutil\n", "shutil.rmtree(os.path.join(CUSTOM_DATA_DIR, \"train_clean\"), ignore_errors=True)\n", "shutil.rmtree(os.path.join(CUSTOM_DATA_DIR, \"dev_clean\"), ignore_errors=True)" ] }, { "cell_type": "code", "execution_count": 45, "id": "64c94c26", "metadata": {}, "outputs": [], "source": [ "os.makedirs(os.path.join(CUSTOM_DATA_DIR, \"train_clean/en\"), exist_ok=True)\n", "os.makedirs(os.path.join(CUSTOM_DATA_DIR, \"train_clean/tl\"), exist_ok=True)\n", "os.makedirs(os.path.join(CUSTOM_DATA_DIR, \"train_clean/00\"), exist_ok=True)\n", "os.makedirs(os.path.join(CUSTOM_DATA_DIR, \"dev_clean/en\"), exist_ok=True)\n", "os.makedirs(os.path.join(CUSTOM_DATA_DIR, \"dev_clean/tl\"), exist_ok=True)\n", "os.makedirs(os.path.join(CUSTOM_DATA_DIR, \"dev_clean/00\"), exist_ok=True)" ] }, { "cell_type": "code", "execution_count": 46, "id": "c3cc2e78", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████| 1740563/1740563 [01:23<00:00, 20787.82it/s]\n", "100%|███████████████████████████████████████████████████████████| 1554308/1554308 [01:27<00:00, 17792.46it/s]\n", "100%|███████████████████████████████████████████████████████████████| 80932/80932 [00:04<00:00, 19569.86it/s]\n", "100%|███████████████████████████████████████████████████████████████| 92985/92985 [00:04<00:00, 19605.71it/s]\n" ] } ], "source": [ "def _process_segments(segments_meta, set_type=\"train\", lang=\"en\"):\n", " manifest = []\n", " for e in tqdm.tqdm(segments_meta):\n", " from_fp = os.path.join(CUSTOM_DATA_DIR, e[\"uri\"])\n", " if e[\"en_p\"] < 0.01:\n", " # label as silence\n", " label = \"00\"\n", " elif (\n", " (lang == \"en\" and e[\"en_p\"] >= 0.85) or # only if high confidence asr\n", " (lang == \"tl\" and e[\"en_p\"] > 0.1 and e[\"en_p\"] < 0.8) # only if low confidence asr but not silence\n", " ):\n", " # keep as en\n", " label = lang\n", " else:\n", " # discard\n", " continue\n", " to_fp = os.path.join(CUSTOM_DATA_DIR, f\"{set_type}_clean\", label, e[\"uid\"] + \".wav\")\n", " os.symlink(from_fp, to_fp)\n", " manifest.append({\n", " \"audio_filepath\": to_fp,\n", " \"duration\": e[\"duration_s\"],\n", " \"offset\": 0,\n", " \"label\": label,\n", " })\n", " return manifest\n", "\n", "nemo_manifest_train = (\n", " _process_segments(raw_post_asr_segments_meta[\"english_segments_meta_tr\"], set_type=\"train\", lang=\"en\") + \n", " _process_segments(raw_post_asr_segments_meta[\"tagalog_segments_meta_tr\"], set_type=\"train\", lang=\"tl\")\n", ")\n", "\n", "nemo_manifest_dev = (\n", " _process_segments(raw_post_asr_segments_meta[\"english_segments_meta_dev\"], set_type=\"dev\", lang=\"en\") + \n", " _process_segments(raw_post_asr_segments_meta[\"tagalog_segments_meta_dev\"], set_type=\"dev\", lang=\"tl\")\n", ") " ] }, { "cell_type": "code", "execution_count": 49, "id": "a19e4cce", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "62.1% of english retained in train\n", "65.7% of tagalog retained in train\n", "64.4% of english retained in dev\n", "67.9% of tagalog retained in dev\n", "4.9% of train is silence\n", "4.8% of dev is silence\n" ] } ], "source": [ "n_before = len(raw_post_asr_segments_meta[\"english_segments_meta_tr\"])\n", "n_after = np.sum([1 for e in nemo_manifest_train if e[\"label\"] == \"en\"])\n", "print(\"{}% of english retained in train\".format(round(n_after / n_before * 100, 1)))\n", "\n", "n_before = len(raw_post_asr_segments_meta[\"tagalog_segments_meta_tr\"])\n", "n_after = np.sum([1 for e in nemo_manifest_train if e[\"label\"] == \"tl\"])\n", "print(\"{}% of tagalog retained in train\".format(round(n_after / n_before * 100, 1)))\n", "\n", "n_before = len(raw_post_asr_segments_meta[\"english_segments_meta_dev\"])\n", "n_after = np.sum([1 for e in nemo_manifest_dev if e[\"label\"] == \"en\"])\n", "print(\"{}% of english retained in dev\".format(round(n_after / n_before * 100, 1)))\n", "\n", "n_before = len(raw_post_asr_segments_meta[\"tagalog_segments_meta_dev\"])\n", "n_after = np.sum([1 for e in nemo_manifest_dev if e[\"label\"] == \"tl\"])\n", "print(\"{}% of tagalog retained in dev\".format(round(n_after / n_before * 100, 1)))\n", "\n", "n_silence = np.sum([1 for e in nemo_manifest_train if e[\"label\"] == \"00\"])\n", "n_train = len(nemo_manifest_train)\n", "print(\"{}% of train is silence\".format(round(n_silence / n_train * 100, 1)))\n", "\n", "n_silence = np.sum([1 for e in nemo_manifest_dev if e[\"label\"] == \"00\"])\n", "n_dev = len(nemo_manifest_dev)\n", "print(\"{}% of dev is silence\".format(round(n_silence / n_dev * 100, 1)))" ] }, { "cell_type": "code", "execution_count": 50, "id": "05adfd05", "metadata": {}, "outputs": [], "source": [ "import random\n", "random.seed(6006)\n", "random.shuffle(nemo_manifest_train)\n", "random.shuffle(nemo_manifest_dev)" ] }, { "cell_type": "code", "execution_count": 70, "id": "69367f5c", "metadata": {}, "outputs": [], "source": [ "# en_p_map = {}\n", "# for _, v in raw_post_asr_segments_meta.items():\n", "# for vv in v:\n", "# en_p_map[vv[\"uid\"]] = vv[\"en_p\"]" ] }, { "cell_type": "code", "execution_count": 69, "id": "011ac285", "metadata": {}, "outputs": [], "source": [ "## listen to a bunch\n", "# for e in nemo_manifest_train[15:30]:\n", "# uid = e[\"audio_filepath\"].split(\"/\")[-1].split(\".\")[0]\n", "# print(e[\"label\"], \"-\", round(en_p_map[uid], 5))\n", "# Audio.from_file(e[\"audio_filepath\"]).play()\n", "# print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": null, "id": "5fb0916d", "metadata": {}, "outputs": [], "source": [ "# TODO: found one tl at en_p=0.701. maybe we need to lower the threshold?" ] }, { "cell_type": "code", "execution_count": 52, "id": "be3e141e", "metadata": {}, "outputs": [], "source": [ "with open(CUSTOM_DATA_DIR + \"meta/nemo_manifest_train.jsonl\", \"w\") as f:\n", " f.write(\"\\n\".join([json.dumps(e) for e in nemo_manifest_train]))\n", "\n", "with open(CUSTOM_DATA_DIR + \"meta/nemo_manifest_dev.jsonl\", \"w\") as f:\n", " f.write(\"\\n\".join([json.dumps(e) for e in nemo_manifest_dev]))" ] }, { "cell_type": "code", "execution_count": null, "id": "6b1323ed", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "863b0a8a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c50e6cc0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0c367c30", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "fe08f95f", "metadata": {}, "source": [ "### Make tar shards" ] }, { "cell_type": "code", "execution_count": null, "id": "a62bde86", "metadata": {}, "outputs": [], "source": [ "# python /home/georg/notebooks/tasks/foreign_language/speechbrain/custom_params/create_wds_shards.py \\\n", "# /mnt/data-ssd-1/data/custom/lang_id-en_tl/train_clean/ \\\n", "# /mnt/data-ssd-1/data/custom/lang_id-en_tl/shards/train_clean \\\n", "# --min-duration=0.5\n", "\n", "# python /home/georg/notebooks/tasks/foreign_language/speechbrain/custom_params/create_wds_shards.py \\\n", "# /mnt/data-ssd-1/data/custom/lang_id-en_tl/dev_clean/ \\\n", "# /mnt/data-ssd-1/data/custom/lang_id-en_tl/shards/dev_clean \\\n", "# --min-duration=0.5" ] }, { "cell_type": "code", "execution_count": 75, "id": "ff9b65dd", "metadata": {}, "outputs": [], "source": [ "import json \n", "\n", "CUSTOM_DATA_DIR = \"/mnt/data-ssd-1/data/custom/lang_id-en_tl/\"\n", "\n", "with open(CUSTOM_DATA_DIR + \"meta/nemo_manifest_train.jsonl\") as f:\n", " nemo_manifest_train = [json.loads(e) for e in f.read().strip().split(\"\\n\")]\n", "\n", "with open(CUSTOM_DATA_DIR + \"meta/nemo_manifest_dev.jsonl\") as f:\n", " nemo_manifest_dev = [json.loads(e) for e in f.read().strip().split(\"\\n\")]" ] }, { "cell_type": "code", "execution_count": null, "id": "655879a3", "metadata": {}, "outputs": [], "source": [ "import shutil\n", "shutil.rmtree(os.path.join(CUSTOM_DATA_DIR, \"shards\"), ignore_errors=True)" ] }, { "cell_type": "code", "execution_count": 93, "id": "ebb8163a", "metadata": {}, "outputs": [], "source": [ "import json\n", "import pathlib\n", "import argparse\n", "import random\n", "import re\n", "from collections import defaultdict\n", "\n", "import torch\n", "import torchaudio\n", "import webdataset as wds\n", "\n", "\n", "def _load_audio(audio_file_path: pathlib.Path) -> torch.Tensor:\n", " t, sr = torchaudio.load(audio_file_path)\n", " if sr != 16000:\n", " raise ValueError(\"expected sampling rate of 16 kHz\")\n", " return t\n", "\n", "\n", "def _write_shards(\n", " audio_metas,\n", " shards_path,\n", " min_duration_s=0.5,\n", " seed=12345,\n", " samples_per_shard=5000,\n", "):\n", " if isinstance(shards_path, str):\n", " shards_path = pathlib.Path(shards_path)\n", " # make sure output folder exist\n", " shards_path.mkdir(parents=True, exist_ok=True)\n", " # create tuples (unique_sample_id, language_id, path_to_audio_file, duration)\n", " data_tuples = []\n", " # track statistics on data\n", " all_language_ids = set()\n", " sample_keys_per_language = defaultdict(list)\n", " print(len(audio_metas), \"audio files\")\n", " for audio_meta in audio_metas:\n", " uid = audio_meta[\"audio_filepath\"].split(\"/\")[-1].split(\".\")[0]\n", " label = audio_meta[\"label\"]\n", " duration_s = audio_meta[\"duration\"]\n", " if duration_s >= min_duration_s:\n", " all_language_ids.add(label)\n", " sample_keys_per_language[label].append(uid)\n", " data_tuples.append((uid, label, audio_meta[\"audio_filepath\"], duration_s))\n", " print(len(data_tuples), \"processed files\")\n", " all_language_ids = sorted(all_language_ids)\n", "\n", " # write a meta.json file which contains statistics on the data\n", " # which will be written to shards\n", " meta_dict = {\n", " \"language_ids\": list(all_language_ids),\n", " \"sample_keys_per_language\": sample_keys_per_language,\n", " \"num_data_samples\": len(data_tuples),\n", " }\n", "\n", " with (shards_path / \"meta.json\").open(\"w\") as f:\n", " json.dump(meta_dict, f)\n", "\n", " # shuffle the tuples so that each shard has a large variety in languages\n", " random.seed(seed)\n", " random.shuffle(data_tuples)\n", "\n", " # write shards\n", " all_keys = set()\n", " shards_path.mkdir(exist_ok=True, parents=True)\n", " pattern = str(shards_path / \"shard\") + \"-%06d.tar\"\n", "\n", " with wds.ShardWriter(pattern, maxcount=samples_per_shard) as sink:\n", " for key, language_id, f, duration in data_tuples:\n", " # load the audio tensor\n", " tensor = _load_audio(f)\n", " # verify key is unique\n", " assert key not in all_keys\n", " all_keys.add(key)\n", " # create sample to write\n", " sample = {\n", " \"__key__\": key,\n", " \"audio.pth\": tensor,\n", " \"language_id\": language_id,\n", " }\n", " # write sample to sink\n", " sink.write(sample)" ] }, { "cell_type": "code", "execution_count": null, "id": "45bfde22", "metadata": {}, "outputs": [], "source": [ "dev_shards_path = \"/mnt/data-ssd-1/data/custom/lang_id-en_tl/shards/dev\"\n", "_write_shards(nemo_manifest_dev, dev_shards_path, min_duration_s=0.5)" ] }, { "cell_type": "code", "execution_count": 90, "id": "5048db97", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "max_shard_path: shard-000024.tar\n" ] } ], "source": [ "max_shard_path = sorted(\n", " [(e, int(e.split(\"-\")[-1].split(\".\")[0])) for e in os.listdir(dev_shards_path) if e.endswith(\".tar\")],\n", " key=lambda x: x[-1]\n", ")[-1][0]\n", "print(\"max_shard_path:\", max_shard_path)" ] }, { "cell_type": "code", "execution_count": null, "id": "87f84159", "metadata": {}, "outputs": [], "source": [ "train_shards_path = \"/mnt/data-ssd-1/data/custom/lang_id-en_tl/shards/train\"\n", "_write_shards(nemo_manifest_train, train_shards_path, min_duration_s=0.5)" ] }, { "cell_type": "code", "execution_count": 96, "id": "18b57c1e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "max_shard_path: shard-000441.tar\n" ] } ], "source": [ "max_shard_path = sorted(\n", " [(e, int(e.split(\"-\")[-1].split(\".\")[0])) for e in os.listdir(train_shards_path) if e.endswith(\".tar\")],\n", " key=lambda x: x[-1]\n", ")[-1][0]\n", "print(\"max_shard_path:\", max_shard_path)" ] }, { "cell_type": "code", "execution_count": null, "id": "e51720b5", "metadata": {}, "outputs": [], "source": [ "# update number of shards in:\n", "# /home/georg/notebooks/tasks/foreign_language/speechbrain/custom_params/train_ecapa.yaml" ] }, { "cell_type": "code", "execution_count": null, "id": "e7461b0a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "dcededdd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "95aaf770", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "327d1e27", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f2277446", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }