{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "b5af1c6b",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
"%pylab inline"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "c30f7ff7",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\""
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "7bb94758",
"metadata": {},
"outputs": [],
"source": [
"DATA_DIR = \"/home/georg/notebooks/tasks/diarization/data/\""
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "2a8311ab",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[NeMo I 2022-06-21 17:08:35 speaker_utils:82] Number of files to diarize: 1\n"
]
}
],
"source": [
"from nemo.collections.asr.parts.utils.diarization_utils import ASR_DIAR_OFFLINE\n",
"import json\n",
"from omegaconf import OmegaConf\n",
"\n",
"config_path = os.path.join(DATA_DIR, 'offline_diarization_with_asr.yaml')\n",
"cfg = OmegaConf.load(config_path)\n",
"\n",
"meta = {\n",
" 'audio_filepath': os.path.join(DATA_DIR, 'an4_diarize_test.wav'), \n",
" 'offset': 0, \n",
" 'duration':None, \n",
" 'label': 'infer', \n",
" 'text': '-', \n",
" 'num_speakers': None, \n",
" 'rttm_filepath': None, \n",
" 'uem_filepath' : None\n",
"}\n",
"with open(os.path.join(DATA_DIR, 'input_manifest.json'), 'w') as fp:\n",
" fp.write(json.dumps(meta) + '\\n')\n",
"cfg.diarizer.manifest_filepath = os.path.join(DATA_DIR,'input_manifest.json')\n",
"\n",
"pretrained_speaker_model = 'titanet_large'\n",
"cfg.diarizer.manifest_filepath = cfg.diarizer.manifest_filepath\n",
"cfg.diarizer.out_dir = DATA_DIR # Directory to store intermediate files and prediction outputs\n",
"cfg.diarizer.speaker_embeddings.model_path = pretrained_speaker_model\n",
"cfg.diarizer.speaker_embeddings.parameters.window_length_in_sec = 1.5\n",
"cfg.diarizer.speaker_embeddings.parameters.shift_length_in_sec = 0.75\n",
"cfg.diarizer.clustering.parameters.oracle_num_speakers = False\n",
"\n",
"# Using VAD generated from ASR timestamps\n",
"cfg.diarizer.asr.model_path = 'QuartzNet15x5Base-En'\n",
"cfg.diarizer.oracle_vad = False # ----> Not using oracle VAD \n",
"cfg.diarizer.asr.parameters.asr_based_vad = True\n",
"cfg.diarizer.asr.parameters.threshold = 100 # ASR based VAD threshold: If 100, all silences under 1 sec are ignored.\n",
"cfg.diarizer.asr.parameters.decoder_delay_in_sec = 0.2 # Decoder delay is compensated for 0.2 sec\n",
"\n",
"asr_diar_offline = ASR_DIAR_OFFLINE(**cfg.diarizer)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "2b09cef8",
"metadata": {},
"outputs": [],
"source": [
"# ValueError: Only one of diarizer.oracle_vad, vad.model_path or vad.external_vad_manifest must be passed"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "654b6d33",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[NeMo I 2022-06-21 17:08:39 clustering_diarizer:153] Loading pretrained titanet_large model from NGC\n",
"[NeMo I 2022-06-21 17:08:39 cloud:56] Found existing object /home/georg/.cache/torch/NeMo/NeMo_1.10.0rc0/titanet-l/492c0ab8416139171dc18c21879a9e45/titanet-l.nemo.\n",
"[NeMo I 2022-06-21 17:08:39 cloud:62] Re-using file from: /home/georg/.cache/torch/NeMo/NeMo_1.10.0rc0/titanet-l/492c0ab8416139171dc18c21879a9e45/titanet-l.nemo\n",
"[NeMo I 2022-06-21 17:08:39 common:789] Instantiating model from pre-trained checkpoint\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[NeMo W 2022-06-21 17:08:40 modelPT:148] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.\n",
" Train config : \n",
" manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/train.json\n",
" sample_rate: 16000\n",
" labels: null\n",
" batch_size: 64\n",
" shuffle: true\n",
" time_length: 3\n",
" is_tarred: false\n",
" tarred_audio_filepaths: null\n",
" tarred_shard_strategy: scatter\n",
" augmentor:\n",
" noise:\n",
" manifest_path: /manifests/noise/rir_noise_manifest.json\n",
" prob: 0.5\n",
" min_snr_db: 0\n",
" max_snr_db: 15\n",
" speed:\n",
" prob: 0.5\n",
" sr: 16000\n",
" resample_type: kaiser_fast\n",
" min_speed_rate: 0.95\n",
" max_speed_rate: 1.05\n",
" num_workers: 15\n",
" pin_memory: true\n",
" \n",
"[NeMo W 2022-06-21 17:08:40 modelPT:155] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). \n",
" Validation config : \n",
" manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/dev.json\n",
" sample_rate: 16000\n",
" labels: null\n",
" batch_size: 128\n",
" shuffle: false\n",
" time_length: 3\n",
" num_workers: 15\n",
" pin_memory: true\n",
" \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[NeMo I 2022-06-21 17:08:40 features:200] PADDING: 16\n",
"[NeMo I 2022-06-21 17:08:40 label_models:100] loss is Angular Softmax\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[NeMo W 2022-06-21 17:08:40 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: Torchmetrics v0.9 introduced a new argument class property called `full_state_update` that has\n",
" not been set for this class (TopKClassificationAccuracy). The property determines if `update` by\n",
" default needs access to the full metric state. If this is not the case, significant speedups can be\n",
" achieved and we recommend setting this to `False`.\n",
" We provide an checking function\n",
" `from torchmetrics.utilities import check_forward_no_full_state`\n",
" that can be used to check if the `full_state_update=True` (old and potential slower behaviour,\n",
" default for now) or if `full_state_update=False` can be used safely.\n",
" \n",
" warnings.warn(*args, **kwargs)\n",
" \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[NeMo I 2022-06-21 17:08:40 save_restore_connector:243] Model EncDecSpeakerLabelModel was successfully restored from /home/georg/.cache/torch/NeMo/NeMo_1.10.0rc0/titanet-l/492c0ab8416139171dc18c21879a9e45/titanet-l.nemo.\n",
"[NeMo I 2022-06-21 17:08:40 speaker_utils:82] Number of files to diarize: 1\n",
"[NeMo I 2022-06-21 17:08:40 clustering_diarizer:271] Subsegmentation for embedding extraction: scale0, /home/georg/notebooks/tasks/diarization/data/speaker_outputs/subsegments_scale0.json\n",
"[NeMo I 2022-06-21 17:08:40 clustering_diarizer:325] Extracting embeddings for Diarization\n",
"[NeMo I 2022-06-21 17:08:40 collections:289] Filtered duration for loading collection is 0.000000.\n",
"[NeMo I 2022-06-21 17:08:40 collections:293] # 5 files loaded accounting to # 1 labels\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 0%| | 0/1 [00:00"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"diarization"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "53930fd9",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from suno_utils.audio import Audio\n",
"Audio.from_file(os.path.join(DATA_DIR, 'an4_diarize_test.wav')).play()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "6430442d",
"metadata": {},
"outputs": [],
"source": [
"from suno_utils.tasks.diarization import diarize"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "9541ad2e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[(0.33, 2.79, 0), (3.21, 5.1, 0)]"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"diarize('data/an4_diarize_test.wav')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "dc33d2d6",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "222377b5",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "718d1bf3",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "54f787db",
"metadata": {},
"source": [
"## Playground"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "98d94b80",
"metadata": {},
"outputs": [],
"source": [
"# https://github.com/NVIDIA/NeMo/blob/main/tutorials/speaker_tasks/Speaker_Diarization_Inference.ipynb\n",
"# https://github.com/NVIDIA/NeMo/blob/main/tutorials/speaker_tasks/ASR_with_SpeakerDiarization.ipynb"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "56b98d19",
"metadata": {},
"outputs": [],
"source": [
"import nemo.collections.asr as nemo_asr\n",
"import numpy as np\n",
"from IPython.display import Audio, display\n",
"import librosa\n",
"import os\n",
"import wget\n",
"import matplotlib.pyplot as plt\n",
"\n",
"import nemo\n",
"import glob\n",
"\n",
"import pprint\n",
"pp = pprint.PrettyPrinter(indent=4)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "fe70e6e7",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Input audio file list: \n",
" ['/home/georg/notebooks/tasks/diarization/data/an4_diarize_test.wav']\n"
]
},
{
"data": {
"text/html": [
"\n",
" \n",
" "
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"ROOT = os.getcwd()\n",
"data_dir = os.path.join(ROOT,'data')\n",
"os.makedirs(data_dir, exist_ok=True)\n",
"\n",
"an4_audio_url = \"https://nemo-public.s3.us-east-2.amazonaws.com/an4_diarize_test.wav\"\n",
"if not os.path.exists(os.path.join(data_dir,'an4_diarize_test.wav')):\n",
" AUDIO_FILENAME = wget.download(an4_audio_url, data_dir)\n",
"else:\n",
" AUDIO_FILENAME = os.path.join(data_dir,'an4_diarize_test.wav')\n",
"\n",
"audio_file_list = glob.glob(f\"{data_dir}/*.wav\")\n",
"print(\"Input audio file list: \\n\", audio_file_list)\n",
"\n",
"signal, sample_rate = librosa.load(AUDIO_FILENAME, sr=None)\n",
"display(Audio(signal,rate=sample_rate))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a9b68573",
"metadata": {},
"outputs": [],
"source": [
"from omegaconf import OmegaConf\n",
"import shutil\n",
"\n",
"config_path = os.path.join(data_dir,'offline_diarization_with_asr.yaml')\n",
"cfg = OmegaConf.load(config_path)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "01282a86",
"metadata": {},
"outputs": [],
"source": [
"# Create a manifest file for input with below format. \n",
"# {\"audio_filepath\": \"/path/to/audio_file\", \"offset\": 0, \"duration\": null, \"label\": \"infer\", \"text\": \"-\", \n",
"# \"num_speakers\": null, \"rttm_filepath\": \"/path/to/rttm/file\", \"uem_filepath\"=\"/path/to/uem/filepath\"}\n",
"import json\n",
"meta = {\n",
" 'audio_filepath': AUDIO_FILENAME, \n",
" 'offset': 0, \n",
" 'duration':None, \n",
" 'label': 'infer', \n",
" 'text': '-', \n",
" 'num_speakers': 2, \n",
" 'rttm_filepath': None, \n",
" 'uem_filepath' : None\n",
"}\n",
"with open(os.path.join(data_dir,'input_manifest.json'),'w') as fp:\n",
" json.dump(meta,fp)\n",
" fp.write('\\n')\n",
"\n",
"cfg.diarizer.manifest_filepath = os.path.join(data_dir,'input_manifest.json')"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "1c5ae3da",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[NeMo I 2022-06-21 13:36:47 speaker_utils:82] Number of files to diarize: 1\n",
"[NeMo I 2022-06-21 13:36:47 cloud:66] Downloading from: https://api.ngc.nvidia.com/v2/models/nvidia/nemospeechmodels/versions/1.0.0a5/files/QuartzNet15x5Base-En.nemo to /home/georg/.cache/torch/NeMo/NeMo_1.10.0rc0/QuartzNet15x5Base-En/2b066be39e9294d7100fb176ec817722/QuartzNet15x5Base-En.nemo\n",
"100% [........................................................................] 71087043 / 71087043[NeMo I 2022-06-21 13:37:47 common:789] Instantiating model from pre-trained checkpoint\n",
"[NeMo I 2022-06-21 13:37:47 features:200] PADDING: 16\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[NeMo W 2022-06-21 13:37:48 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: Torchmetrics v0.9 introduced a new argument class property called `full_state_update` that has\n",
" not been set for this class (WER). The property determines if `update` by\n",
" default needs access to the full metric state. If this is not the case, significant speedups can be\n",
" achieved and we recommend setting this to `False`.\n",
" We provide an checking function\n",
" `from torchmetrics.utilities import check_forward_no_full_state`\n",
" that can be used to check if the `full_state_update=True` (old and potential slower behaviour,\n",
" default for now) or if `full_state_update=False` can be used safely.\n",
" \n",
" warnings.warn(*args, **kwargs)\n",
" \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[NeMo I 2022-06-21 13:37:48 save_restore_connector:243] Model EncDecCTCModel was successfully restored from /home/georg/.cache/torch/NeMo/NeMo_1.10.0rc0/QuartzNet15x5Base-En/2b066be39e9294d7100fb176ec817722/QuartzNet15x5Base-En.nemo.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[NeMo W 2022-06-21 13:37:48 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: Torchmetrics v0.9 introduced a new argument class property called `full_state_update` that has\n",
" not been set for this class (WER_TS). The property determines if `update` by\n",
" default needs access to the full metric state. If this is not the case, significant speedups can be\n",
" achieved and we recommend setting this to `False`.\n",
" We provide an checking function\n",
" `from torchmetrics.utilities import check_forward_no_full_state`\n",
" that can be used to check if the `full_state_update=True` (old and potential slower behaviour,\n",
" default for now) or if `full_state_update=False` can be used safely.\n",
" \n",
" warnings.warn(*args, **kwargs)\n",
" \n",
"[NeMo W 2022-06-21 13:37:48 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/autocast_mode.py:162: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
" warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
" \n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "73d06645071b4c3aa8c9ac8335d18dcc",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Transcribing: 0%| | 0/1 [00:00 speaker_1 ',\n",
" 'SPEAKER an4_diarize_test 1 0.980 0.920 speaker_1 ',\n",
" 'SPEAKER an4_diarize_test 1 1.980 1.100 speaker_1 ',\n",
" 'SPEAKER an4_diarize_test 1 3.180 0.980 speaker_0 ',\n",
" 'SPEAKER an4_diarize_test 1 4.300 0.720 speaker_0 ']\n"
]
}
],
"source": [
"def read_file(path_to_file):\n",
" with open(path_to_file) as f:\n",
" contents = f.read().splitlines()\n",
" return contents\n",
"\n",
"predicted_speaker_label_rttm_path = f\"{data_dir}/pred_rttms/an4_diarize_test.rttm\"\n",
"pred_rttm = read_file(predicted_speaker_label_rttm_path)\n",
"\n",
"pp.pprint(pred_rttm)\n",
"\n",
"from nemo.collections.asr.parts.utils.speaker_utils import rttm_to_labels\n",
"pred_labels = rttm_to_labels(predicted_speaker_label_rttm_path)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "19396a45",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[NeMo I 2022-06-21 13:38:16 diarization_utils:506] Creating results for Session: an4_diarize_test n_spk: 2 \n",
"[NeMo I 2022-06-21 13:38:16 diarization_utils:443] Diarization with ASR output files are saved in: /home/georg/notebooks/tasks/diarization/data/pred_rttms\n"
]
},
{
"data": {
"text/plain": [
"{'an4_diarize_test': OrderedDict([('status', 'Success'),\n",
" ('session_id', 'an4_diarize_test'),\n",
" ('transcription',\n",
" 'eleven twenty seven fifty seven october twenty fourth nineteen seventy'),\n",
" ('speaker_count', 2),\n",
" ('words',\n",
" [{'word': 'eleven',\n",
" 'start_time': 0.4,\n",
" 'end_time': 0.84,\n",
" 'speaker_label': 'speaker_1'},\n",
" {'word': 'twenty',\n",
" 'start_time': 0.98,\n",
" 'end_time': 1.34,\n",
" 'speaker_label': 'speaker_1'},\n",
" {'word': 'seven',\n",
" 'start_time': 1.38,\n",
" 'end_time': 1.9,\n",
" 'speaker_label': 'speaker_1'},\n",
" {'word': 'fifty',\n",
" 'start_time': 1.98,\n",
" 'end_time': 2.32,\n",
" 'speaker_label': 'speaker_1'},\n",
" {'word': 'seven',\n",
" 'start_time': 2.36,\n",
" 'end_time': 3.08,\n",
" 'speaker_label': 'speaker_1'},\n",
" {'word': 'october',\n",
" 'start_time': 3.18,\n",
" 'end_time': 3.58,\n",
" 'speaker_label': 'speaker_0'},\n",
" {'word': 'twenty',\n",
" 'start_time': 3.62,\n",
" 'end_time': 3.88,\n",
" 'speaker_label': 'speaker_0'},\n",
" {'word': 'fourth',\n",
" 'start_time': 3.92,\n",
" 'end_time': 4.16,\n",
" 'speaker_label': 'speaker_0'},\n",
" {'word': 'nineteen',\n",
" 'start_time': 4.3,\n",
" 'end_time': 4.64,\n",
" 'speaker_label': 'speaker_0'},\n",
" {'word': 'seventy',\n",
" 'start_time': 4.66,\n",
" 'end_time': 5.02,\n",
" 'speaker_label': 'speaker_0'}]),\n",
" ('sentences',\n",
" [{'sentence': 'eleven twenty seven fifty seven',\n",
" 'start_time': 0.4,\n",
" 'end_time': 3.08,\n",
" 'speaker_label': 'speaker_1'},\n",
" {'sentence': 'october twenty fourth nineteen seventy',\n",
" 'start_time': 3.18,\n",
" 'end_time': 5.02,\n",
" 'speaker_label': 'speaker_0'}])])}"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"asr_diar_offline.get_transcript_with_speaker_labels(diar_hyp, word_hyp, word_ts_hyp)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "03642a56",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[ '[00:00.40 - 00:03.08] speaker_1: eleven twenty seven fifty seven',\n",
" '[00:03.18 - 00:05.02] speaker_0: october twenty fourth nineteen seventy']\n"
]
}
],
"source": [
"transcription_path_to_file = f\"{data_dir}/pred_rttms/an4_diarize_test.txt\"\n",
"transcript = read_file(transcription_path_to_file)\n",
"pp.pprint(transcript)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f1fafec3",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "81aaeb98",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "776ec623",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}