{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"id": "ee52ad57",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
"%pylab inline"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "e06d16da",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\""
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "7cdc9fee",
"metadata": {},
"outputs": [],
"source": [
"import time\n",
"import os\n",
"import tqdm\n",
"import shutil\n",
"import random\n",
"import funcy\n",
"import json\n",
"import numpy as np\n",
"import multiprocessing\n",
"import pandas as pd\n",
"from suno_utils.audio import Audio\n",
"from suno_utils.utils.display import suppress_logging\n",
"from IPython.utils.io import capture_output"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "8fd69200",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[NeMo I 2022-06-21 09:57:26 cloud:56] Found existing object /home/georg/.cache/torch/NeMo/NeMo_1.10.0rc0/stt_en_conformer_transducer_xlarge/0d13710522bd7bfd198ba54de79645fb/stt_en_conformer_transducer_xlarge.nemo.\n",
"[NeMo I 2022-06-21 09:57:26 cloud:62] Re-using file from: /home/georg/.cache/torch/NeMo/NeMo_1.10.0rc0/stt_en_conformer_transducer_xlarge/0d13710522bd7bfd198ba54de79645fb/stt_en_conformer_transducer_xlarge.nemo\n",
"[NeMo I 2022-06-21 09:57:26 common:789] Instantiating model from pre-trained checkpoint\n",
"[NeMo I 2022-06-21 09:57:28 mixins:166] Tokenizer SentencePieceTokenizer initialized with 1024 tokens\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[NeMo W 2022-06-21 09:57:29 modelPT:148] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.\n",
" Train config : \n",
" manifest_filepath:\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket1/tarred_audio_manifest.json\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket2/tarred_audio_manifest.json\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket3/tarred_audio_manifest.json\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket4/tarred_audio_manifest.json\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket5/tarred_audio_manifest.json\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket6/tarred_audio_manifest.json\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket7/tarred_audio_manifest.json\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket8/tarred_audio_manifest.json\n",
" sample_rate: 16000\n",
" batch_size: 1\n",
" shuffle: true\n",
" num_workers: 4\n",
" pin_memory: true\n",
" use_start_end_token: false\n",
" trim_silence: false\n",
" max_duration: 20\n",
" min_duration: 0\n",
" is_tarred: true\n",
" tarred_audio_filepaths:\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket1/audio__OP_0..8191_CL_.tar\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket2/audio__OP_0..8191_CL_.tar\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket3/audio__OP_0..8191_CL_.tar\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket4/audio__OP_0..8191_CL_.tar\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket5/audio__OP_0..8191_CL_.tar\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket6/audio__OP_0..8191_CL_.tar\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket7/audio__OP_0..8191_CL_.tar\n",
" - - /data/NeMo_ASR_SET/English/v3.0/train_bucketed/bucket8/audio__OP_0..8191_CL_.tar\n",
" shuffle_n: 2048\n",
" bucketing_strategy: synced_randomized\n",
" bucketing_batch_size:\n",
" - 32\n",
" - 32\n",
" - 16\n",
" - 16\n",
" - 16\n",
" - 16\n",
" - 8\n",
" - 8\n",
" \n",
"[NeMo W 2022-06-21 09:57:29 modelPT:155] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). \n",
" Validation config : \n",
" manifest_filepath:\n",
" - /data/librispeech_withsp2/manifests/librivox-dev-other.json\n",
" - /data/librispeech_withsp2/manifests/librivox-dev-clean.json\n",
" - /data/librispeech_withsp2/manifests/librivox-test-other.json\n",
" - /data/librispeech_withsp2/manifests/librivox-test-clean.json\n",
" sample_rate: 16000\n",
" batch_size: 4\n",
" shuffle: false\n",
" num_workers: 4\n",
" pin_memory: true\n",
" use_start_end_token: false\n",
" is_tarred: false\n",
" tarred_audio_filepaths: na\n",
" min_duration: 0\n",
" \n",
"[NeMo W 2022-06-21 09:57:29 modelPT:161] Please call the ModelPT.setup_test_data() or ModelPT.setup_multiple_test_data() method and provide a valid configuration file to setup the test data loader(s).\n",
" Test config : \n",
" manifest_filepath:\n",
" - /data/librispeech_withsp2/manifests/librivox-test-other.json\n",
" - /data/librispeech_withsp2/manifests/librivox-dev-clean.json\n",
" - /data/librispeech_withsp2/manifests/librivox-dev-other.json\n",
" - /data/librispeech_withsp2/manifests/librivox-test-clean.json\n",
" sample_rate: 16000\n",
" batch_size: 4\n",
" shuffle: false\n",
" num_workers: 4\n",
" pin_memory: true\n",
" use_start_end_token: false\n",
" is_tarred: false\n",
" tarred_audio_filepaths: na\n",
" \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[NeMo I 2022-06-21 09:57:29 features:200] PADDING: 0\n",
"[NeMo I 2022-06-21 09:57:32 rnnt_models:203] Using RNNT Loss : warprnnt_numba\n",
" Loss warprnnt_numba_kwargs: {'fastemit_lambda': 0.0, 'clamp': -1.0}\n",
"[NeMo I 2022-06-21 09:57:32 audio_preprocessing:491] Numba CUDA SpecAugment kernel is being used\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[NeMo W 2022-06-21 09:57:32 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: Torchmetrics v0.9 introduced a new argument class property called `full_state_update` that has\n",
" not been set for this class (RNNTWER). The property determines if `update` by\n",
" default needs access to the full metric state. If this is not the case, significant speedups can be\n",
" achieved and we recommend setting this to `False`.\n",
" We provide an checking function\n",
" `from torchmetrics.utilities import check_forward_no_full_state`\n",
" that can be used to check if the `full_state_update=True` (old and potential slower behaviour,\n",
" default for now) or if `full_state_update=False` can be used safely.\n",
" \n",
" warnings.warn(*args, **kwargs)\n",
" \n",
"[NeMo W 2022-06-21 09:57:32 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: Torchmetrics v0.9 introduced a new argument class property called `full_state_update` that has\n",
" not been set for this class (RNNTBPEWER). The property determines if `update` by\n",
" default needs access to the full metric state. If this is not the case, significant speedups can be\n",
" achieved and we recommend setting this to `False`.\n",
" We provide an checking function\n",
" `from torchmetrics.utilities import check_forward_no_full_state`\n",
" that can be used to check if the `full_state_update=True` (old and potential slower behaviour,\n",
" default for now) or if `full_state_update=False` can be used safely.\n",
" \n",
" warnings.warn(*args, **kwargs)\n",
" \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[NeMo I 2022-06-21 09:57:37 save_restore_connector:243] Model EncDecRNNTBPEModel was successfully restored from /home/georg/.cache/torch/NeMo/NeMo_1.10.0rc0/stt_en_conformer_transducer_xlarge/0d13710522bd7bfd198ba54de79645fb/stt_en_conformer_transducer_xlarge.nemo.\n"
]
}
],
"source": [
"import nemo.collections.asr as nemo_asr\n",
"asr_model = nemo_asr.models.EncDecRNNTBPEModel.from_pretrained(model_name=\"stt_en_conformer_transducer_xlarge\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "3fb86a6d",
"metadata": {},
"outputs": [],
"source": [
"with capture_output():\n",
" asr_model.transcribe([\"tmp/conversation_2.mp3\"])[0][0]"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "71eb065e",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"a = Audio.from_file(\"tmp/conversation_2.mp3\")\n",
"a.play()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "6020112d",
"metadata": {},
"outputs": [],
"source": [
"with open(\"/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/03_segment_meta_post_timestamp.json\") as f:\n",
" d = json.load(f)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "9df97e1d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"16 found\n"
]
}
],
"source": [
"candidates = []\n",
"for e in d:\n",
" a = e[\"transcript_normalized\"][\"text\"].split()\n",
" b = e[\"transcript_asr\"][\"transcript\"].split()\n",
" if len(a) <= 10:\n",
" continue\n",
" if len(a) / len(b) <= 1.5:\n",
" continue\n",
" candidates.append(e)\n",
"random.seed(6006)\n",
"random.shuffle(candidates)\n",
"print(len(candidates), \"found\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1cf736bb",
"metadata": {},
"outputs": [],
"source": [
"for n, e in enumerate(candidates[:5]):\n",
" print(n, \"-\", e[\"uri\"])\n",
" Audio.from_file(e[\"uri\"]).play()\n",
" print(\"TRUE:\", e[\"transcript_normalized\"][\"text\"])\n",
" print()\n",
" print(\"O_PR:\", e[\"transcript_asr\"][\"transcript\"])\n",
" print()\n",
" with capture_output():\n",
" pred = asr_model.transcribe([e[\"uri\"]])[0][0]\n",
" print(\"PRED:\", pred)\n",
" print(\"-\"*10)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "9e3a57e2",
"metadata": {},
"outputs": [],
"source": [
"l = [\n",
" \"13 - /mnt/data-ssd-1/data/gdata-mate/pipeline/segments/4a10dcb7-31bc-4451-bc66-d47b3cc3a690.wav\",\n",
" \"14 - /mnt/data-ssd-1/data/gdata-mate/pipeline/segments/595a9840-5de0-472a-aa8a-6c968411750f.wav\",\n",
" \"15 - /mnt/data-ssd-1/data/gdata-mate/pipeline/segments/45fe7d33-00c0-4d96-b3d9-866a3fe57678.wav\",\n",
" \"5 - /mnt/data-ssd-1/data/gdata-mate/pipeline/segments/91128e2c-4a3c-47c4-8db9-9e6a0d5c9b5c.wav\",\n",
" \"2 - /mnt/data-ssd-1/data/gdata-mate/pipeline/segments/cd23ee31-a617-4311-8b56-b2a5184d0770.wav\",\n",
" \"0 - /mnt/data-ssd-1/data/gdata-mate/pipeline/segments/4c9e7afc-eca6-41d4-a699-d6d1fe38be5e.wav\",\n",
" \"6 - /mnt/data-ssd-1/data/gdata-mate/pipeline/segments/13fc3f02-f864-4551-91a4-480109b4ae62.wav\",\n",
" \"11 - /mnt/data-ssd-1/data/gdata-mate/pipeline/segments/5101d492-11fa-463c-ad01-0b112a654104.wav\",\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": 47,
"id": "fd9d560b",
"metadata": {},
"outputs": [],
"source": [
"for e in l:\n",
" fp = e.split(\" - \")[-1]\n",
" shutil.copy(fp, \"nemo_suno_aus_samples/\" + fp.split(\"/\")[-1])"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "d4ca46fa",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"13fc3f02-f864-4551-91a4-480109b4ae62.wav\r\n",
"45fe7d33-00c0-4d96-b3d9-866a3fe57678.wav\r\n",
"4a10dcb7-31bc-4451-bc66-d47b3cc3a690.wav\r\n",
"4c9e7afc-eca6-41d4-a699-d6d1fe38be5e.wav\r\n",
"5101d492-11fa-463c-ad01-0b112a654104.wav\r\n",
"595a9840-5de0-472a-aa8a-6c968411750f.wav\r\n",
"91128e2c-4a3c-47c4-8db9-9e6a0d5c9b5c.wav\r\n",
"cd23ee31-a617-4311-8b56-b2a5184d0770.wav\r\n",
"transcripts.json\r\n"
]
}
],
"source": [
"!ls nemo_suno_aus_samples"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "d315e06e",
"metadata": {},
"outputs": [],
"source": [
"transcripts = {}\n",
"for e in l:\n",
" idx = int(e.split(\" - \")[0])\n",
" filepath = e.split(\" - \")[-1]\n",
" uid = filepath.split(\"/\")[-1].split(\".\")[0]\n",
" transcripts[uid] = {\n",
" \"text\": candidates[idx]['transcript'][\"text\"],\n",
" \"text_norm\": candidates[idx]['transcript_normalized'][\"text\"],\n",
" }\n",
"with open(\"nemo_suno_aus_samples/transcripts.json\", \"w\") as f:\n",
" json.dump(transcripts, f)"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "f9e21abb",
"metadata": {},
"outputs": [],
"source": [
"!tar -czf nemo_suno_aus_samples.tar.gz nemo_suno_aus_samples"
]
},
{
"cell_type": "code",
"execution_count": 31,
"id": "e1913ce4",
"metadata": {},
"outputs": [],
"source": [
"refs = []\n",
"hyps = []\n",
"for k, v in transcripts.items():\n",
" refs.append(v[\"text_norm\"])\n",
" with capture_output():\n",
" out = asr_model.transcribe([f\"nemo_suno_aus_samples/{k}.wav\"])[0][0]\n",
" hyps.append(out)"
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "16e6fb79",
"metadata": {},
"outputs": [],
"source": [
"from suno_utils.utils.metrics import get_wer_bulk"
]
},
{
"cell_type": "code",
"execution_count": 33,
"id": "3d8976fe",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.6073717948717948"
]
},
"execution_count": 33,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"get_wer_bulk(refs, hyps)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6fd32f4a",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "72df162d",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "ee7f897f",
"metadata": {},
"source": [
"## Playground"
]
},
{
"cell_type": "code",
"execution_count": 52,
"id": "6c8da9fc",
"metadata": {},
"outputs": [],
"source": [
"# import nemo.collections.asr as nemo_asr\n",
"# from IPython.utils.io import capture_output\n",
"# with capture_output():\n",
"# asr_model = nemo_asr.models.EncDecRNNTBPEModel.from_pretrained(\n",
"# model_name=\"stt_en_conformer_transducer_xlarge\"\n",
"# )"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "277131ec",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "15c5220b",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "5a93e24f",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "d3f9673a",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "5dfdf605",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "72858ce1",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "bc1d4a8f",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}