{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "6ca248c0",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
"%pylab inline"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "c2b15e10",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\""
]
},
{
"cell_type": "code",
"execution_count": 37,
"id": "448165e6",
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"from suno_utils.audio import Audio\n",
"with open(\"initial_sample/hinglish_sample_20_annotations.json\") as f:\n",
" d = json.load(f)[\"027b55b2-1d60-4fde-b6c9-8c975b5eb5e4\"]\n",
"assert(len(d) == 1)\n",
"transcript = d[0][\"text\"]\n",
"transcript_norm = transcript.lower().replace(\".\", \"\").replace(\"?\", \"\").replace(\",\", \"\")\n",
"audio_filepath = \"initial_sample/annotations/027b55b2-1d60-4fde-b6c9-8c975b5eb5e4.mp3\"\n",
"audio = Audio.from_file(audio_filepath, sample_rate=16_000)"
]
},
{
"cell_type": "code",
"execution_count": 160,
"id": "93a88689",
"metadata": {},
"outputs": [],
"source": [
"from suno_utils.utils.display import suppress_logging\n",
"import nemo.collections.asr as nemo_asr\n",
"from IPython.utils.io import capture_output\n",
"from scipy.special import softmax\n",
"from pyctcdecode import build_ctcdecoder\n",
"import numpy as np\n",
"\n",
"with suppress_logging():\n",
" en_asr_model = nemo_asr.models.EncDecCTCModelBPE.from_pretrained(\"stt_en_conformer_ctc_large\")\n",
" hi_asr_model = nemo_asr.models.EncDecCTCModelBPE.from_pretrained(\"stt_hi_conformer_ctc_medium\")\n",
"# [e.pretrained_model_name for e in nemo_asr.models.EncDecCTCModelBPE.list_available_models()]\n",
" en_decoder = build_ctcdecoder(en_asr_model.cfg[\"decoder\"][\"vocabulary\"] + [\"\"])\n",
" hi_decoder = build_ctcdecoder(hi_asr_model.cfg[\"decoder\"][\"vocabulary\"] + [\"\"])\n",
"\n",
"def pred_en(filepath, tokens=False):\n",
" with suppress_logging():\n",
" with capture_output():\n",
" out = en_asr_model.transcribe([filepath], logprobs=tokens)[0]\n",
" if tokens:\n",
" probs = softmax(out, axis=1)\n",
" tokens = en_decoder.decode_beams(out, beam_width=2)[0][2]\n",
" out = []\n",
" for token, (start_f, end_f) in tokens:\n",
" conf = np.mean(probs[start_f:end_f].max(axis=1))\n",
" out.append(((start_f, end_f), conf, token))\n",
" return out\n",
"\n",
"def pred_hi(filepath, tokens=False):\n",
" with suppress_logging():\n",
" with capture_output():\n",
" out = hi_asr_model.transcribe([filepath], logprobs=tokens)[0]\n",
" if tokens:\n",
" probs = softmax(out, axis=1)\n",
" tokens = hi_decoder.decode_beams(out, beam_width=2)[0][2]\n",
" out = []\n",
" for token, (start_f, end_f) in tokens:\n",
" conf = np.mean(probs[start_f:end_f].max(axis=1))\n",
" out.append(((start_f, end_f), conf, token))\n",
" return out\n",
"\n",
"def _resolve_tokens(tokens_en, tokens_hi):\n",
" sorted_tokens = sorted(tokens_en + tokens_hi, key=lambda x: x[1], reverse=True)\n",
" blocked_idx = set()\n",
" resolved_tokens = []\n",
" for (start_f, end_f), conf, token in sorted_tokens:\n",
" idx_set = set(range(start_f, end_f))\n",
" if len(blocked_idx & idx_set) == 0:\n",
" resolved_tokens.append(((start_f, end_f), conf, token))\n",
" blocked_idx |= idx_set\n",
" sorted_resolved_tokens = sorted(resolved_tokens, key=lambda x: x[0][0])\n",
" return sorted_resolved_tokens\n",
"\n",
"def pred_joint(filepath, min_confidence=0.8, punt_tag=\"...\"):\n",
" tokens_en = pred_en(filepath, tokens=True)\n",
"\n",
" tokens_hi = pred_hi(filepath, tokens=True)\n",
" merged_tokens = _resolve_tokens(tokens_en, tokens_hi)\n",
" text = \" \".join([t if conf >= min_confidence else punt_tag for _, conf, t in merged_tokens])\n",
" text = re.sub(r\"(\\s*\" + re.escape(punt_tag) + r\"\\s*)+\", \" \" + punt_tag + \" \", text).strip()\n",
" return text"
]
},
{
"cell_type": "code",
"execution_count": 164,
"id": "9d04497e",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"audio.play()"
]
},
{
"cell_type": "code",
"execution_count": 41,
"id": "0a08a404",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\"you know sari gandighi johi the maki spirituality bahar nkhalaga or usa hoga kya hamari under faith or trust johi wo banil lagega because human skill is nothing more important than hope it's like we live on hope itself because hamaga rako alam laagakivi soting ithought we have that hope kihamsub utinge\""
]
},
"execution_count": 41,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pred_en(audio_filepath)"
]
},
{
"cell_type": "code",
"execution_count": 165,
"id": "0042e781",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'सारी गंदगी जो है दिमाग की स्पेशवालिटी बाहर निकालेगा और उससे होगा क्या हमारे अंदर फेट और ट्रस्ट जो है वो बनने लगेगा बिकज ह्यूमन के लिए नथिंग मोर इम्पोर्टेन्टि हो सुबहठे'"
]
},
"execution_count": 165,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pred_hi(audio_filepath) # stops after `more important` and then garbage word"
]
},
{
"cell_type": "code",
"execution_count": 163,
"id": "82ed5150",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\"you know सारी गंदगी जो है दिमाग की spirituality बाहर निकालेगा और उससे होगा क्या हमारे अंदर faith और trust जो है वो बनने लगेगा because humans के लिए there's nothing more important than hope it's like we live on hope itself because हम अगर रात को alarm लगा के भी सोते है तो we have that hope की हम सुबह उठेंगे।\""
]
},
"execution_count": 163,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"transcript_norm"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d25ea867",
"metadata": {},
"outputs": [],
"source": [
"# `।` is Full Stop `.`"
]
},
{
"cell_type": "code",
"execution_count": 166,
"id": "876bf3d7",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\"You know सारी गंदगी जो है दिमाग की spirituality बाहर निकालेगा और उससे होगा क्या? हमारे अंदर faith और trust जो है वो बनने लगेगा because humans के लिए there's nothing more important than hope. It's like we live on hope itself because हम अगर रात को alarm लगा के भी सोते है तो we have that hope की हम सुबह उठेंगे।.\""
]
},
"execution_count": 166,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"transcript"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a3377332",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 168,
"id": "5b506014",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\"... सारी गंदगी जो है दिमाग की spirituality बाहर निकालेगा और or उससे होगा क्या हमारे अंदर faith और or trust जो है वो बनने लगेगा because human ... लिए ... nothing more important than ... hope it's like we live on hope itself ... we have that hope ...\""
]
},
"execution_count": 168,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pred_joint(audio_filepath, min_confidence=0.93)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d105e6eb",
"metadata": {},
"outputs": [],
"source": [
"# faith और trust -- faith and (or) trust"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fd1972b5",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "3f44d30d",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "401d85a0",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "ee3a61ca",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "c9cbf0fe",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "75aeb01a",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "43d495d3",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}