{ "cells": [ { "cell_type": "markdown", "id": "a521e4e9", "metadata": {}, "source": [ "## Test Norm" ] }, { "cell_type": "code", "execution_count": 1, "id": "70339fcf", "metadata": {}, "outputs": [], "source": [ "# TODO: update\n", "# TODO: make faster\n", "# TODO: add Tokens stuff" ] }, { "cell_type": "code", "execution_count": 5, "id": "3e65fcbd", "metadata": {}, "outputs": [], "source": [ "# !rm -rf /tmp/prod_nemo_fst/*" ] }, { "cell_type": "code", "execution_count": 1, "id": "fb8defe0", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-08-18 08:39:13 optimizers:77] Could not import distributed_fused_adam optimizer from Apex\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[NeMo I 2022-08-18 08:39:14 post_processing:50] Post processing graph was restored from /tmp/prod_nemo_fst_georg/en_tn_post_processing.far.\n", "[NeMo I 2022-08-18 08:39:21 tokenize_and_classify_with_audio:99] ClassifyFst.fst was restored from /tmp/prod_nemo_fst_georg/_cased_en_tn_False_deterministicnemo_custom_norm.tsv.far.\n", "[NeMo I 2022-08-18 08:39:21 verbalize_final:52] VerbalizeFinalFst graph was restored from /tmp/prod_nemo_fst_georg/en_tn_False_deterministic_verbalizer.far.\n" ] } ], "source": [ "import time\n", "from suno_utils.utils.text_normalizer import normalize, normalize_mp, CUSTOM_NORM_RULES\n", "from nemo_text_processing.text_normalization.normalize_with_audio import (\n", " NormalizerWithAudio,\n", ")\n", "from nemo_text_processing.text_normalization.normalize import Normalizer\n", "from suno_utils.audio import Tokens\n", "\n", "text = \"Yesterday, I didn't like the first IPhone 7 presentation given my dream-like state.\"\n", "asr_text = \"yesterday i didn't like the first iphone seven presentation given my dream like state\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "fd51a248", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "78186" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import json\n", "d = \"/mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/pipeline/test_mfa_2022_08_11/\"\n", "with open(d + \"artifacts/02_segment_meta_post_norm_and_asr.json\") as f:\n", " manifest = json.load(f)\n", "len(manifest)" ] }, { "cell_type": "code", "execution_count": 48, "id": "fd35fbc5", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "from suno_utils.utils.text_normalizer import (\n", " _change_to_asr_format\n", "# funcy, NORMALIZER, _pre_normalize, suppress_prints, _change_to_asr_format, _normalize_asr_prediction,\n", "# get_cer, _post_process_punct\n", ")\n", "\n", "l = []\n", "for m in manifest[:100]:\n", " written = Tokens.from_dict(m[\"transcript\"][\"tokens\"]).plaintext\n", " audio_normed = Tokens.from_dict(m[\"transcript_normalized\"][\"tokens\"]).text\n", "# normed = normalizer.normalize(\n", "# written, \n", "# verbose=False, \n", "# punct_pre_process=True,\n", "# punct_post_process=True,\n", "# )\n", " l.append({\n", " \"text\": written,\n", "# \"written\": _change_to_asr_format(written),\n", "# \"normed_raw\": normed,\n", "# \"normed\": _change_to_asr_format(normed),\n", " \"text_norm\": audio_normed,\n", " \"asr\": m[\"transcript_asr\"][\"transcript\"],\n", " })\n", "df = pd.DataFrame(l)" ] }, { "cell_type": "code", "execution_count": 5, "id": "66bf397e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[NeMo I 2022-08-18 08:46:54 post_processing:50] Post processing graph was restored from /tmp/test_speed/en_tn_post_processing.far.\n", "[NeMo I 2022-08-18 08:47:06 tokenize_and_classify_with_audio:99] ClassifyFst.fst was restored from /tmp/test_speed/_cased_en_tn_False_deterministic.far.\n", "[NeMo I 2022-08-18 08:47:06 verbalize_final:52] VerbalizeFinalFst graph was restored from /tmp/test_speed/en_tn_False_deterministic_verbalizer.far.\n" ] } ], "source": [ "normalizer_a = NormalizerWithAudio(\n", " lang=\"en\",\n", " input_case=\"cased\",\n", " overwrite_cache=False,\n", " cache_dir=\"/tmp/test_speed\",\n", ")" ] }, { "cell_type": "code", "execution_count": 3, "id": "a2b7f3c1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[NeMo I 2022-08-18 08:39:40 tokenize_and_classify:87] Creating ClassifyFst grammars.\n", "Created /tmp/test2_speed/en_tn_True_deterministic_cased__tokenize.far\n", "[NeMo I 2022-08-18 08:40:12 tokenize_and_classify:215] ClassifyFst grammars are saved to /tmp/test2_speed/en_tn_True_deterministic_cased__tokenize.far.\n", "Created /tmp/test2_speed/en_tn_True_deterministic_verbalizer.far\n", "[NeMo I 2022-08-18 08:40:13 verbalize_final:76] VerbalizeFinalFst grammars are saved to /tmp/test2_speed/en_tn_True_deterministic_verbalizer.far.\n" ] } ], "source": [ "normalizer = Normalizer(\n", " lang=\"en\",\n", " input_case=\"cased\",\n", " overwrite_cache=False,\n", " cache_dir=\"/tmp/test2_speed\",\n", " post_process=False,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "04b18207", "metadata": {}, "outputs": [], "source": [ "# first do custom norm\n", "# remove trailing periods and comma\n", "# check if multiple norm options remaining\n", "# only then do norm\n", "\n", "# OR\n", "\n", "# first do custom norm\n", "# remove trailing periods and comma\n", "# lowercase all capi`talized words\n", "# check if determ norm is same as original\n", "# only then do norm\n", "\n", "# OR\n", "\n", "# first do custom norm\n", "# remove trailing periods and comma\n", "# lowercase all capitalized words\n", "# then only norm numbers and special symbols bounded by normal word\n", "\n", "# ALSO\n", "\n", "# just deterministic norm if cer too high" ] }, { "cell_type": "code", "execution_count": 53, "id": "0a12ba8e", "metadata": {}, "outputs": [], "source": [ "import re\n", "from suno_utils.utils.text import normalize_whitespace\n", "\n", "def _lower_repl_helper(m):\n", " return \" \" + m.group(1).lower() + \" \"\n", "\n", "def new_norm(s):\n", " # remove trailing periods and comma\n", " s = re.sub(r\"[\\.\\,\\?](\\s|$)\", \" \", s)\n", " # lowercase all capitalized words\n", " s = re.sub(r\"(?:^|(?<=\\s))([A-Z][a-z]*)(?:$|(?=\\s))\", _lower_repl_helper, s)\n", " s = normalize_whitespace(s)\n", " # check if determ norm is same as original\n", " # only then do norm\n", " return s\n", "\n", "# TODO: fail on roman numeral `I`" ] }, { "cell_type": "code", "execution_count": 54, "id": "2955af05", "metadata": {}, "outputs": [], "source": [ "# check if normed is same as written (and verify audio normed isn't improving)" ] }, { "cell_type": "code", "execution_count": 71, "id": "c034aced", "metadata": {}, "outputs": [], "source": [ "skip_data = []\n", "for _, row in df.iterrows():\n", " new_text_norm = new_norm(row[\"text\"])\n", " tmp_text_norm = normalizer.normalize(\n", " new_text_norm, \n", " verbose=False, \n", " punct_pre_process=False,\n", " punct_post_process=False,\n", " )\n", " if new_text_norm == tmp_text_norm:\n", " if _change_to_asr_format(new_text_norm) == row[\"text_norm\"]:\n", " skip_data.append(True)\n", " else:\n", " skip_data.append(((row[\"text\"], _change_to_asr_format(new_text_norm), row[\"text_norm\"])))" ] }, { "cell_type": "code", "execution_count": 72, "id": "9233d54c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "93/100 skipped.\n", "7 skip mistakes\n" ] } ], "source": [ "print(\"{}/{} skipped.\".format(len(skip_data), 100))\n", "print(\"{} skip mistakes\".format(len([e for e in skip_data if e is not True])))" ] }, { "cell_type": "code", "execution_count": 75, "id": "b2ac025a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'okay uh are you already connected to the VPN'" ] }, "execution_count": 75, "metadata": {}, "output_type": "execute_result" } ], "source": [ "text = 'Okay. Uh, are you already connected to the VPN?'\n", "new_text_norm = new_norm(text)\n", "new_text_norm" ] }, { "cell_type": "code", "execution_count": 82, "id": "9ba13aa1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "okay uh are you already connected to the VPN\n" ] }, { "data": { "text/plain": [ "{'okay uh are you already connected to the V P N',\n", " 'okay uh are you already connected to the VPN'}" ] }, "execution_count": 82, "metadata": {}, "output_type": "execute_result" } ], "source": [ "normalizer_a.normalize(\n", " 'okay uh are you already connected to the VPN', \n", " verbose=False, \n", " n_tagged=100,\n", ")" ] }, { "cell_type": "code", "execution_count": 73, "id": "f9a1108c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " ('Okay. Uh, are you already connected to the VPN?',\n", " 'okay uh are you already connected to the vpn',\n", " 'okay uh are you already connected to the v p n'),\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " ('So, upon logging in, so it asked for the VPN portal for Verizon, right?',\n", " 'so upon logging in so it asked for the vpn portal for verizon right',\n", " 'so upon logging in so it asked for the v p n portal for verizon right'),\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " ('L Uh, just go to, uh, were you able to log in also, like, uh, using your WinID?',\n", " 'l uh just go to uh were you able to log in also like uh using your winid',\n", " 'fifty uh just go to uh were you able to log in also like uh using your winid'),\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " (\"Okay. It says that you're already connected with the VPN. And you were able to log in to the PC using your WinID, with\",\n", " \"okay it says that you're already connected with the vpn and you were able to log in to the pc using your winid with\",\n", " \"okay it says that you're already connected with the v p n and you were able to log in to the p c using your winid with\"),\n", " True,\n", " True,\n", " True,\n", " True,\n", " (\"Okay. Uh, are you ready to take it down? Okay. It's SR, it's sierra, uh, S for sierra, R for Romeo.\",\n", " \"okay uh are you ready to take it down okay it's sr it's sierra uh s for sierra r for romeo\",\n", " \"okay uh are you ready to take it down okay it's s r it's sierra uh s for sierra r for romeo\"),\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " ('Then try to, uh, connect again, the VPN, sign in again.',\n", " 'then try to uh connect again the vpn sign in again',\n", " 'then try to uh connect again the v p n sign in again'),\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " True,\n", " (\"Okay, um. Ma'am, go to sign in without your agent ID in your Avaya.\",\n", " \"okay um ma'am go to sign in without your agent id in your avaya\",\n", " \"okay um ma'am go to sign in without your agent i d in your avaya\")]" ] }, "execution_count": 73, "metadata": {}, "output_type": "execute_result" } ], "source": [ "skip_data" ] }, { "cell_type": "code", "execution_count": null, "id": "67f9cab9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "71a1fa8f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8fcc3b59", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "3a704e45", "metadata": {}, "source": [ "### Add split_text_into_sentences before brute force splitting" ] }, { "cell_type": "code", "execution_count": null, "id": "87fdb6e3", "metadata": {}, "outputs": [], "source": [ " # TODO: use normalizer.split_text_into_sentences\n", " if len(text_list) == 1 and len(text_list[0]) == 0:\n", " return [\"\"]\n", " if custom_norm_rules is None:\n", " custom_norm_rules = []\n", " # sub-split in case a string is too long\n", " chunk_map = []\n", " safe_text_list = []\n", " n = 0\n", " for e in text_list:\n", " new_chunks = [\n", " \" \".join(ee) for ee in list(funcy.chunks(n_max_tokens, e.split()))\n", " ]\n", " safe_text_list.extend(new_chunks)\n", " chunk_map.append((n, n + len(new_chunks)))\n", " n += len(new_chunks)" ] }, { "cell_type": "code", "execution_count": null, "id": "9b539195", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "870483fb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "cbb8c87f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "aba45d6f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b379a2d8", "metadata": {}, "source": [ "## Test MFA" ] }, { "cell_type": "code", "execution_count": null, "id": "4e4e0789", "metadata": {}, "outputs": [], "source": [ "import time\n", "from suno_utils.web.mfa import align_text, align_tokens\n", "from suno_utils.audio import Tokens\n", "\n", "with open(\"sample_audio/russia_norm.txt\") as f:\n", " transcript = f.read()" ] }, { "cell_type": "code", "execution_count": 10, "id": "d845b8cc", "metadata": {}, "outputs": [], "source": [ "N = 1000\n", "conda_sh_filepath = \"/home/georg/anaconda3/etc/profile.d/conda.sh\"\n", "env_name = \"mfa\"\n", "num_cores = 1\n", "# model_name = \"english\"\n", "model_name = \"english_mfa\"" ] }, { "cell_type": "code", "execution_count": 11, "id": "ccdf928e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "181.9 seconds runtime.\n" ] } ], "source": [ "t0 = time.time()\n", "out = align_tokens(\n", " [\"sample_audio/russia.wav\"]*N,\n", " [Tokens.from_text(transcript)]*N,\n", " conda_sh_filepath,\n", " env_name,\n", " num_cores=num_cores,\n", " dictionary_name=model_name,\n", " acoustic_model_name=model_name,\n", ")\n", "t1 = time.time()\n", "print(round(t1 - t0, 1), \"seconds runtime.\")" ] }, { "cell_type": "code", "execution_count": null, "id": "106df2d3", "metadata": {}, "outputs": [], "source": [ "# N= 1, n_cores= 1: 19.0 seconds runtime.\n", "# N= 10, n_cores= 1: 23.6 seconds runtime.\n", "# N= 50, n_cores= 1: 26.6 seconds runtime.\n", "# N= 200, n_cores= 1: 36.6 seconds runtime.\n", "# N= 200, n_cores= 5: 27.2 seconds runtime.\n", "# N=1000, n_cores= 1: 271.1 seconds runtime.\n", "# N=1000, n_cores=10: 55.3 seconds runtime.\n", "# N=1000, n_cores=40: 40.5 seconds runtime.\n", "\n", "# mfa\n", "# N= 10, n_cores= 1: 25.0 seconds runtime.\n", "# N= 100, n_cores= 1: 40.0 seconds runtime.\n", "# N=1000, n_cores= 1: 182.0 seconds runtime.\n", "# N=1000, n_cores=10: 46.0 seconds runtime.\n", "# N=1000, n_cores=40: 37.6 seconds runtime.\n", "# N=4000, n_cores=40: 66.8 seconds runtime.\n", "\n", "# -> estimate for 78186: 22 mins" ] }, { "cell_type": "code", "execution_count": null, "id": "ce9489d1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a417241a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 37, "id": "c320ddcf", "metadata": {}, "outputs": [], "source": [ "d = \"/mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/pipeline/test_mfa_2022_08_11/\"" ] }, { "cell_type": "code", "execution_count": 35, "id": "efb7054d", "metadata": {}, "outputs": [], "source": [ "# os.remove(os.path.join(d, \"artifacts/nvidia_dataset_viewer.jsonl\"))" ] }, { "cell_type": "code", "execution_count": 57, "id": "81b2a485", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 8.6G\r\n", "drwxrwxr-x 3 georg georg 4.0K Aug 17 07:46 .\r\n", "drwxrwxr-x 4 georg georg 4.0K Aug 16 22:42 ..\r\n", "-rw-rw-r-- 1 georg georg 118M Aug 16 22:42 01_segment_meta.json\r\n", "-rw-rw-r-- 1 georg georg 196M Aug 16 22:42 02_segment_meta_post_norm_and_asr.json\r\n", "-rw-rw-r-- 1 georg georg 270M Aug 16 23:22 03_segment_meta_post_timestamp.json\r\n", "-rw-rw-r-- 1 georg georg 162M Aug 17 07:46 04_segment_meta_distribution.jsonl\r\n", "-rw-rw-r-- 1 georg georg 321M Aug 17 07:37 data_sample.tar.gz\r\n", "-rw-rw-r-- 1 georg georg 7.5G Aug 17 07:46 data.tar.gz\r\n", "-rw-rw-r-- 1 georg georg 20M Aug 17 07:32 nvidia_dataset_viewer.jsonl\r\n", "drwxrwxr-x 3 georg georg 4.0K Aug 17 07:36 web-sample-data\r\n" ] } ], "source": [ "!ls -lah /mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/pipeline/test_mfa_2022_08_11/artifacts" ] }, { "cell_type": "code", "execution_count": 45, "id": "be659b36", "metadata": {}, "outputs": [], "source": [ "# cd /home/georg/notebooks\n", "# python run_modified_pipeline.py" ] }, { "cell_type": "code", "execution_count": null, "id": "6699e1f1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7a467d90", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "84dbf5d4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 48, "id": "f05b796d", "metadata": {}, "outputs": [], "source": [ "with open(\"/mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/pipeline/test_mfa_2022_08_11/artifacts/02_segment_meta_post_norm_and_asr.json\") as f:\n", " d = json.load(f)" ] }, { "cell_type": "code", "execution_count": null, "id": "2560d774", "metadata": {}, "outputs": [], "source": [ "# new pipeline\n", "# simplify s3_sample_data_prefix and dataset_name\n", "# add --generate-tar\n", "# add --generate-sample_data\n", "# modify web sample data to just sample data\n", "\n", "# what about sample_data_manifest? gentle_alignment_dir?\n", "# make inputs a config?" ] }, { "cell_type": "code", "execution_count": null, "id": "e72330bd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d7a494d2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 23, "id": "2d6e6fe5", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.text_normalizer import (\n", " funcy, NORMALIZER, _pre_normalize, suppress_prints, _change_to_asr_format, _normalize_asr_prediction,\n", " get_cer, _post_process_punct\n", ")\n", "\n", "text_list = [text]\n", "asr_prediction = asr_text\n", "n_max_hyp = 200\n", "n_max_tokens = 200\n", "custom_norm_rules = CUSTOM_NORM_RULES" ] }, { "cell_type": "code", "execution_count": 28, "id": "9ea0edb5", "metadata": {}, "outputs": [], "source": [ "if custom_norm_rules is None:\n", " custom_norm_rules = []\n", "# sub-split in case a string is too long\n", "chunk_map = []\n", "safe_text_list = []\n", "n = 0\n", "for e in text_list:\n", " new_chunks = [\n", " \" \".join(ee) for ee in list(funcy.chunks(n_max_tokens, e.split()))\n", " ]\n", " safe_text_list.extend(new_chunks)\n", " chunk_map.append((n, n + len(new_chunks)))\n", " n += len(new_chunks)" ] }, { "cell_type": "code", "execution_count": 36, "id": "2808c3e9", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "custom_normalized_texts" ] }, { "cell_type": "code", "execution_count": null, "id": "091c4ff8", "metadata": {}, "outputs": [], "source": [ " NORMALIZER.normalize(\n", " text=_pre_normalize(custom_normalized_text),\n", " verbose=False,\n", " n_tagged=n_max_hyp if n_max_hyp is not None else -1,\n", " punct_post_process=False, # requires nemo NLP & Moses de-tokenization\n", " )" ] }, { "cell_type": "code", "execution_count": 41, "id": "07fb1297", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "((re.compile(r'(?<=[a-z])\\.(?=[a-z])', re.UNICODE), ' dot '),\n", " (re.compile(r'\\\"(.+?)\\\"', re.UNICODE), ' quote \\\\1'),\n", " (re.compile(r'\\\"(.+?)\\\"', re.UNICODE), ' quote unquote \\\\1'))" ] }, "execution_count": 41, "metadata": {}, "output_type": "execute_result" } ], "source": [ "CUSTOM_NORM_RULES" ] }, { "cell_type": "code", "execution_count": 43, "id": "eb50994d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[\"Yesterday, I didn't like the first IPhone 7 presentation given my dream-like state.\"]" ] }, "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ "safe_text_list" ] }, { "cell_type": "code", "execution_count": 50, "id": "88c8c099", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2.3 seconds\n" ] } ], "source": [ "# do normalization\n", "options = [(\"\", [])]\n", "top_rewrites = []\n", "text = safe_text_list[0]\n", "\n", "\n", "# custom_normalized_texts = []\n", "# # add more custom norm options\n", "# for cpt_ptn, repl_ptn in custom_norm_rules:\n", "# if cpt_ptn.search(text):\n", "# custom_normalized_texts.append(cpt_ptn.sub(repl_ptn, text))\n", "# if n_max_hyp is not None:\n", "# custom_normalized_texts = custom_normalized_texts[:n_max_hyp]\n", "# custom_normalized_texts = list(set(custom_normalized_texts))\n", "# print(custom_normalized_texts)\n", "# normalized_texts = []\n", "# for custom_normalized_text in custom_normalized_texts:\n", "# with suppress_prints():\n", "# normalized_texts.extend(\n", "# NORMALIZER.normalize(\n", "# text=_pre_normalize(custom_normalized_text),\n", "# verbose=False,\n", "# n_tagged=n_max_hyp if n_max_hyp is not None else -1,\n", "# punct_post_process=False, # requires nemo NLP & Moses de-tokenization\n", "# )\n", "# )\n", "\n", "pre_normalized_text = _pre_normalize(text)\n", "t0 = time.time()\n", "with suppress_prints():\n", " normalized_texts = NORMALIZER.normalize(\n", " text=pre_normalized_text,\n", " verbose=False,\n", " n_tagged=n_max_hyp if n_max_hyp is not None else -1,\n", " punct_post_process=False, # very slow if numbers explode\n", " )\n", " top_rewrite = NORMALIZER.normalize(\n", " text=pre_normalized_text, verbose=False, n_tagged=1, punct_post_process=False,\n", " )\n", "t1 = time.time()\n", "assert len(top_rewrite) == 1\n", "top_rewrites.append(_change_to_asr_format(list(top_rewrite)[0]))\n", "new_options = []\n", "seen_txt = set()\n", "for ot, ol in options:\n", " for nt in normalized_texts:\n", " # maybe fix with known normalization dictionaries in symbols\n", " nnt = _change_to_asr_format(nt)\n", " sep = \" \" if len(ot) > 0 else \"\"\n", " s = ot + sep + nnt\n", " if s not in seen_txt:\n", " seen_txt.add(s)\n", " else:\n", " continue\n", " idx_l = ol + [len(nnt.split())]\n", " new_options.append((s, idx_l))\n", "options = new_options\n", "print(round(t1 - t0, 1), \"seconds\")" ] }, { "cell_type": "code", "execution_count": 52, "id": "44658672", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\"Yesterday, I didn't like the first IPhone 7 presentation given my dream-like state.\"" ] }, "execution_count": 52, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pre_normalized_text" ] }, { "cell_type": "code", "execution_count": 65, "id": "b645eb30", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Yesterday, I didn't like the first IPhone 7 presentation given my dream-like state.\n", "1.2 seconds\n" ] } ], "source": [ "t0 = time.time()\n", "normalized_texts = NORMALIZER.normalize(\n", " text=\"Yesterday, I didn't like the first IPhone 7 presentation given my dream-like state.\",\n", " verbose=False,\n", " n_tagged=1, # 200\n", " punct_post_process=False, # very slow if numbers explode\n", ")\n", "t1 = time.time()\n", "print(round(t1 - t0, 1), \"seconds\")" ] }, { "cell_type": "code", "execution_count": 60, "id": "4adb4302", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'one two kg', 'twelve kg', 'twelve kilograms'}" ] }, "execution_count": 60, "metadata": {}, "output_type": "execute_result" } ], "source": [ "normalized_texts" ] }, { "cell_type": "code", "execution_count": null, "id": "fe6920b5", "metadata": {}, "outputs": [], "source": [ " def normalize_list(\n", " self,\n", " texts: List[str],\n", " verbose: bool = False,\n", " punct_pre_process: bool = False,\n", " punct_post_process: bool = False,\n", " batch_size: int = 1,\n", " n_jobs: int = 1,\n", " ):" ] }, { "cell_type": "code", "execution_count": null, "id": "13f97e52", "metadata": {}, "outputs": [], "source": [ "split_text_into_sentences" ] }, { "cell_type": "code", "execution_count": 74, "id": "63f471d0", "metadata": {}, "outputs": [ { "data": { "image/svg+xml": [], "text/plain": [ "" ] }, "execution_count": 74, "metadata": {}, "output_type": "execute_result" } ], "source": [ "NORMALIZER.find_verbalizer(\"Yesterday, I didn't\")" ] }, { "cell_type": "code", "execution_count": null, "id": "f58358c1", "metadata": {}, "outputs": [], "source": [ "text = \"Yesterday, I didn't like the first IPhone 7 presentation given my dream-like state.\"" ] }, { "cell_type": "code", "execution_count": null, "id": "ff3adf25", "metadata": {}, "outputs": [], "source": [ "NormalizerWithAudio(\n", " lang=\"en\",\n", " input_case=\"cased\",\n", " overwrite_cache=False,\n", " cache_dir=USER_SPECIFIC_TEMP_DIR,\n", " whitelist=CUSTOM_NORM_PATH,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "e2feffc2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "2086ed63", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 70, "id": "0c49e1cf", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 0/1 [01:07\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m out = NORMALIZER.normalize_list(\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mtexts\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Yesterday, I didn't like the first IPhone 7 presentation given my dream-like state.\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;31m# 200\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mpunct_pre_process\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/code/NeMo/nemo_text_processing/text_normalization/normalize.py\u001b[0m in \u001b[0;36mnormalize_list\u001b[0;34m(self, texts, verbose, punct_pre_process, punct_post_process, batch_size, n_jobs)\u001b[0m\n\u001b[1;32m 158\u001b[0m )\n\u001b[1;32m 159\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 160\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 161\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 162\u001b[0m \u001b[0mnormalized_texts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitertools\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mnormalized_texts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/code/NeMo/nemo_text_processing/text_normalization/normalize.py\u001b[0m in \u001b[0;36mnormalize_list\u001b[0;34m(self, texts, verbose, punct_pre_process, punct_post_process, batch_size, n_jobs)\u001b[0m\n\u001b[1;32m 153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 154\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 155\u001b[0;31m normalized_texts = Parallel(n_jobs=n_jobs)(\n\u001b[0m\u001b[1;32m 156\u001b[0m \u001b[0mdelayed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__process_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtexts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpunct_pre_process\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpunct_post_process\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 157\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtexts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m 1041\u001b[0m \u001b[0;31m# remaining jobs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1042\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1043\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1044\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_iterator\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1045\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[0;34m(self, iterator)\u001b[0m\n\u001b[1;32m 859\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 860\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 861\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 862\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 863\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m 777\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 778\u001b[0m \u001b[0mjob_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 779\u001b[0;31m \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 780\u001b[0m \u001b[0;31m# A job can complete so quickly than its callback is\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 781\u001b[0m \u001b[0;31m# called before we get here, causing self._jobs to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[0;34m(self, func, callback)\u001b[0m\n\u001b[1;32m 206\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 207\u001b[0m \u001b[0;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 208\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 209\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 210\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m 570\u001b[0m \u001b[0;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 571\u001b[0m \u001b[0;31m# arguments in memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 572\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 573\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 574\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 260\u001b[0m \u001b[0;31m# change the default number of processes to -1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 261\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 262\u001b[0;31m return [func(*args, **kwargs)\n\u001b[0m\u001b[1;32m 263\u001b[0m for func, args, kwargs in self.items]\n\u001b[1;32m 264\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 260\u001b[0m \u001b[0;31m# change the default number of processes to -1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 261\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 262\u001b[0;31m return [func(*args, **kwargs)\n\u001b[0m\u001b[1;32m 263\u001b[0m for func, args, kwargs in self.items]\n\u001b[1;32m 264\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/code/NeMo/nemo_text_processing/text_normalization/normalize.py\u001b[0m in \u001b[0;36m__process_batch\u001b[0;34m(self, batch, verbose, punct_pre_process, punct_post_process)\u001b[0m\n\u001b[1;32m 172\u001b[0m \u001b[0mpunct_post_process\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mwhether\u001b[0m \u001b[0mto\u001b[0m \u001b[0mdo\u001b[0m \u001b[0mpunctuation\u001b[0m \u001b[0mpost\u001b[0m \u001b[0mprocessing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 173\u001b[0m \"\"\"\n\u001b[0;32m--> 174\u001b[0;31m normalized_lines = [\n\u001b[0m\u001b[1;32m 175\u001b[0m self.normalize(\n\u001b[1;32m 176\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpunct_pre_process\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpunct_pre_process\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpunct_post_process\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpunct_post_process\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/code/NeMo/nemo_text_processing/text_normalization/normalize.py\u001b[0m in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 173\u001b[0m \"\"\"\n\u001b[1;32m 174\u001b[0m normalized_lines = [\n\u001b[0;32m--> 175\u001b[0;31m self.normalize(\n\u001b[0m\u001b[1;32m 176\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpunct_pre_process\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpunct_pre_process\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpunct_post_process\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpunct_post_process\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 177\u001b[0m )\n", "\u001b[0;31mTypeError\u001b[0m: normalize() got an unexpected keyword argument 'punct_pre_process'" ] } ], "source": [ "out = NORMALIZER.normalize_list(\n", " texts=[\"Yesterday, I didn't like the first IPhone 7 presentation given my dream-like state.\"]*5,\n", " verbose=False,\n", " n_jobs=1, # 200a\n", " punct_pre_process = False,\n", " punct_post_process = False,\n", "# punct_post_process=True, # very slow if numbers explode\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "03f3c819", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 54, "id": "55cf6753", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "200" ] }, "execution_count": 54, "metadata": {}, "output_type": "execute_result" } ], "source": [ "n_max_hyp" ] }, { "cell_type": "code", "execution_count": 45, "id": "2a685001", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[(\"yesterday i didn ' t like the first iphone seven presentation given my dream like state\",\n", " [16]),\n", " (\"yesterday i didn't like the first iphone seven presentation given my dream like state\",\n", " [14])]" ] }, "execution_count": 45, "metadata": {}, "output_type": "execute_result" } ], "source": [ "options" ] }, { "cell_type": "code", "execution_count": 29, "id": "27b4e1a3", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\"yesterday i didn't like the first iphone seven presentation given my dream like state\"" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "normalized_text, _ = NORMALIZER.select_best_match(\n", " normalized_texts=[o for o, _ in options],\n", " input_text=options[0][0], # this is default return if pred is \"\"\n", " pred_text=_normalize_asr_prediction(asr_prediction),\n", " verbose=False,\n", " remove_punct=True,\n", ")\n", "cer = get_cer(normalized_text, _normalize_asr_prediction(asr_prediction))\n", "if cer < 0.5:\n", " correct_idx_map = [idx_map for o, idx_map in options if o == normalized_text]\n", " assert len(correct_idx_map) == 1\n", " correct_idx_map = correct_idx_map[0]\n", " norm_list = []\n", " offs = 0\n", " assert len(correct_idx_map) == len(safe_text_list)\n", " for n, raw_text in zip(correct_idx_map, safe_text_list):\n", " s = \" \".join(normalized_text.split()[offs : offs + n])\n", " norm_list.append(s)\n", " offs += n\n", " assert offs == len(normalized_text.split())\n", "else:\n", " norm_list = top_rewrites\n", "# detokenize\n", "# TODO: could swap in moses here:\n", "# s = NORMALIZER.processor.detokenize([s])\n", "# s = post_process_punct(input=raw_text, normalized_text=s)\n", "fixed_norm_list = [_post_process_punct(s) for s in norm_list]\n", "# assemble back original list incase of sub splits\n", "orig_norm_list = []\n", "for start_idx, end_idx in chunk_map:\n", " orig_norm_list.append(\" \".join(fixed_norm_list[start_idx:end_idx]))\n", "orig_norm_list[0]" ] }, { "cell_type": "code", "execution_count": null, "id": "fbb0964a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3d19a211", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8a1302bb", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }