{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "5531ebee", "metadata": {}, "outputs": [], "source": [ "# https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo/models/stt_en_fastconformer_ctc_large" ] }, { "cell_type": "code", "execution_count": 2, "id": "1ee51aff", "metadata": {}, "outputs": [], "source": [ "# import sys\n", "# sys.path.insert(0, \"/home/georg/code/hoot/\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "853b13c2", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"6\"" ] }, { "cell_type": "code", "execution_count": 4, "id": "6763016f", "metadata": {}, "outputs": [], "source": [ "import torch\n", "\n", "from hoot.model import Hoot, Tokenizer" ] }, { "cell_type": "code", "execution_count": 5, "id": "b64dd744", "metadata": {}, "outputs": [], "source": [ "model = Hoot.from_nemo(\n", " \"/mnt/data/georg/models/stt_en_fastconformer_ctc_large/model_weights.ckpt\",\n", ")\n", "model.eval()\n", "model.to(\"cuda\");\n", "\n", "tokenizer = Tokenizer(\n", " \"/mnt/data/georg/models/stt_en_fastconformer_ctc_large/0124d42f914e45e98c214bd5afd17f55_tokenizer.model\"\n", ")" ] }, { "cell_type": "code", "execution_count": 6, "id": "15e0b73d", "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/plain": [ "\"yeah good morning this is george um today we'll be talking about uh covervet and uh the impact that it's had on our all of our recent lives and remote meetings in general\"" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "logits = model.infer(\"../gpt/samples/covid.wav\")\n", "tokenizer.decode_logits(logits)" ] }, { "cell_type": "code", "execution_count": 7, "id": "09a92167", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\"remember those girls up ba they and they didn't even put up a fi they they didn't even make the sound i phone no way to but i never really stand the lady oh hello i got my ang now like a been away every rule i had tos the ris that i'm taking i never going to show you i surrounded by ba i can see you everything i need i the my youre the only one i and i'm addicted i sw to pull me back to the ground like a' been awayke every rule i had to the rik that i'm taking i'm never going to show you i can i won' very welled ba i can see everything i need maybe i can tell you i\"" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "logits = model.infer(\"../gpt/samples/halo_hq.wav\")\n", "tokenizer.decode_logits(logits)" ] }, { "cell_type": "code", "execution_count": 7, "id": "2b84f3b6", "metadata": {}, "outputs": [], "source": [ "# # check single-sample finetuned\n", "# d = torch.load(\"/mnt/data/georg/checkpoints/2023-05-26_10-27-34/last_ckpt.pt\", map_location=\"cpu\")[\"model\"]\n", "# model.load_state_dict(d);" ] }, { "cell_type": "code", "execution_count": 8, "id": "9ce0c8fc", "metadata": {}, "outputs": [], "source": [ "# logits = model.infer(\"../gpt/samples/halo_hq.wav\")\n", "# tokenizer.decode_logits(logits)" ] }, { "cell_type": "code", "execution_count": null, "id": "4a6fc758", "metadata": {}, "outputs": [], "source": [ "# check custom\n", "model = Hoot.from_checkpoint(\n", " \"/mnt/data/georg/checkpoints/2023-05-26_16-55-38/last_ckpt.pt\",\n", ")\n", "model.eval()\n", "model.to(\"cuda\");" ] }, { "cell_type": "code", "execution_count": 13, "id": "c4a1db8f", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\"remember those walls i built well baby they tumbling down and they didn't even put up a fight they didn't even make a sound i found a way to let you in but i never really had a doubt standing in the light of your halo i got my angel now it's like i've been awaken every rule i had you breaking it's the risk that i'm taking i ain't never gonna shut you out everywhere i'm looking now i'm surrounded by your embrace baby i can see your halo you know you're my saviving grace you're everything i need and more it's written all over your face baby i can feel your halo pray it won't ha to your halo halo halo to your halo halo halo your halo halo haloy see your halo halo halo hit me like a raylide burning through my dark at night you're the only one that i want and got addicted to your light i swore i'd never fall again but this don't even feel like falling gravity can't beget to pull me back to the ground again it's like i've been awakein every rule i had you breaking the risk that i'm taking i'm never gonna shut you out everywhere i'm looking now i'm surrounded by your embrace baby i can feel your halo pray it won't ha to your halo halo halo your halo halo halo he your halo halo haloy see your halo halo halo halo halo everywhere i'm looking now i'm surrounded by your embrace baby i can see your halo you know you're my saviving grace you're everything i need and more it's written all over your face baby i can feel your halo pray it won't to your halo halo halo to your halo ha halo your halo halo haloy see your halo halo halo\"" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "logits = model.infer(\"../gpt/samples/halo_hq.wav\")\n", "tokenizer.decode_logits(logits)" ] }, { "cell_type": "code", "execution_count": null, "id": "1e690092", "metadata": {}, "outputs": [], "source": [ "logits = model.infer(\"../gpt/samples/covid.wav\")\n", "tokenizer.decode_logits(logits)" ] }, { "cell_type": "code", "execution_count": null, "id": "c4fa33c4", "metadata": { "ExecuteTime": { "start_time": "2023-11-06T19:45:11.271Z" } }, "outputs": [], "source": [ "out = encode_filepaths([\"/home/tony/319cfea5-30ea-43b7-9654-6c82a5245046_0.mp3\"])" ] }, { "cell_type": "code", "execution_count": null, "id": "a1629b88", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "f15e477e", "metadata": {}, "source": [ "## Use glock" ] }, { "cell_type": "code", "execution_count": 1, "id": "71cd2c86", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:18.266073Z", "start_time": "2023-11-06T19:46:17.989087Z" } }, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", "from suno_utils.audio import Audio" ] }, { "cell_type": "code", "execution_count": 2, "id": "09a75d69", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:19.829947Z", "start_time": "2023-11-06T19:46:18.267533Z" } }, "outputs": [], "source": [ "from suno_utils.tasks.hoot import preload_models, encode_filepaths, encode, Tokenizer" ] }, { "cell_type": "code", "execution_count": 3, "id": "ef3225f2", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:19.834319Z", "start_time": "2023-11-06T19:46:19.832036Z" } }, "outputs": [], "source": [ "# model_dir = \"s3://suno-data/datasets/bundles/v1/models/hoot_v0\"\n", "# _ = preload_models(\n", "# checkpoint_filepath=os.path.join(model_dir, \"2023-05-26_16-55-38_last_ckpt.pt\"),\n", "# tokenizer_filepath=os.path.join(model_dir, \"tokenizer.model\"),\n", "# )" ] }, { "cell_type": "code", "execution_count": 4, "id": "501df841", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:19.888085Z", "start_time": "2023-11-06T19:46:19.836662Z" } }, "outputs": [], "source": [ "# !ls -t \"/home/tony/Data/checkpoints/hoot/2023-11-05_21-30-28\"" ] }, { "cell_type": "code", "execution_count": 5, "id": "5858503b", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:22.554202Z", "start_time": "2023-11-06T19:46:19.889670Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['']\n" ] } ], "source": [ "_ = preload_models(\n", " checkpoint_filepath=os.path.join(\"/home/tony/Data/Hoot/default_model.pt\"),\n", " # tokenizer_filepath=\"/home/tony/Work/tony/hoot/tokenizers/en3/tokenizer_spe_bpe_v4096/tokenizer.model\",\n", " tokenizer_filepath=\"/home/tony/Data/Hoot/tokenizer_v1024_default.model\"\n", ")" ] }, { "cell_type": "markdown", "id": "c2c51ee5", "metadata": {}, "source": [ "{\"audio_filepath\": \"/home/tony/Data/Hoot/exp/ivnWzCbx3hw.wav\", \"duration\": 144, \"text\": \"hey hey shout shout shout for happiness there s no joy unless you shout for happiness all day through oh sing sing let your voices ring bid your troubles adieu take the joy that is meant for you don t sit in a corner like jacky horner get where the bright lights are just look up high search the sky find your luck star shout shout shout for happiness there s no joy if your blue shout for happiness all day through well well well who do i see right in front of me but max goldie in the flesh hey gosh but you re looking well and full of spirits too no i didn t say whiskey come on max tell me what makes you so full of happiness been to see the cup final oh i can see right in your eye you re just dying for a drink ah well come on lets go\"}\n", "\n", "{\"audio_filepath\": \"/home/tony/Data/Hoot/exp/6e1XdHd_RjE.wav\", \"duration\": 187, \"text\": \"verse rapture rapture falling apart fracture fracture break me apart taste it taste it are we okay wasted wasted on an ego trip what a fucking shame drop dead just like the rest cause i was never heaven sent go tell your family and go tell your friends you ll drop dead just like the rest chorus so please won t you forget me cause i m leaving i m leaving get these memories out of my head verse confess for all that you ve done suppress and tell me that i m the one what a waste of such a beautiful face god damn can t you just get over it fuck this you re just like the rest and you were never heaven sent you made me like this drop dead just like the rest just like the rest confess confess fucking say it go ahead and watch me fall breakdown confess cause now you re the enemy suppress this won t be the end of me mindless the numb rotting the broken severed tongues enough remaining unspoken cause this is the plague and now you re infected your face is such a beautiful waste chorus so please won t you forget me cause i m leaving i m leaving get these memories out of my head confess suppress confess suppress to the face of punishment inside and out you ve dropped dead just like the rest\"}" ] }, { "cell_type": "code", "execution_count": 6, "id": "a89ce7eb", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:22.561267Z", "start_time": "2023-11-06T19:46:22.556447Z" } }, "outputs": [], "source": [ "t = Tokenizer(\"/home/tony/Data/Hoot/tokenizer_v1024_default.model\")" ] }, { "cell_type": "code", "execution_count": 7, "id": "36ddb452", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:22.616094Z", "start_time": "2023-11-06T19:46:22.563198Z" } }, "outputs": [], "source": [ "x = \"hey hey shout shout shout for happiness there s no joy unless you shout for happiness all day through oh sing sing let your voices ring bid your troubles adieu take the joy that is meant for you don t sit in a corner like jacky horner get where the bright lights are just look up high search the sky find your luck star shout shout shout for happiness there s no joy if your blue shout for happiness all day through well well well who do i see right in front of me but max goldie in the flesh hey gosh but you re looking well and full of spirits too no i didn t say whiskey come on max tell me what makes you so full of happiness been to see the cup final oh i can see right in your eye you re just dying for a drink ah well come on lets go\"" ] }, { "cell_type": "code", "execution_count": 8, "id": "29da9aaf", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:22.673039Z", "start_time": "2023-11-06T19:46:22.617701Z" } }, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "x_e = t._tokenizer.decode(t._tokenizer.encode(x))\n", "x_e == x" ] }, { "cell_type": "code", "execution_count": 9, "id": "fc8bd44f", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:23.180688Z", "start_time": "2023-11-06T19:46:22.674863Z" } }, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "test_audio = Audio.from_file(\"/home/tony/Data/Hoot/exp/ivnWzCbx3hw.wav\")\n", "test_audio.play()" ] }, { "cell_type": "code", "execution_count": 10, "id": "d77bbda0", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:23.183912Z", "start_time": "2023-11-06T19:46:23.181794Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "16000 1 2 144.02175\n" ] } ], "source": [ "print(test_audio.sample_rate, test_audio.n_channels, test_audio.byte_width, test_audio.duration_s)" ] }, { "cell_type": "code", "execution_count": 11, "id": "ba975cdb", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:23.419593Z", "start_time": "2023-11-06T19:46:23.186732Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[\"sho shout shout forss there's joy unless you shout fors all day through sing s letoing bigubles the ist for you can seeinker\"]\n" ] } ], "source": [ "out = encode_filepaths([\"/home/tony/Data/Hoot/exp/ivnWzCbx3hw.wav\"])" ] }, { "cell_type": "code", "execution_count": 12, "id": "cffd916e", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:23.580201Z", "start_time": "2023-11-06T19:46:23.422619Z" } }, "outputs": [], "source": [ "out = encode_filepaths([\"/home/tony/Data/Hoot/exp/ivnWzCbx3hw.wav\"], return_logits=True)" ] }, { "cell_type": "code", "execution_count": 13, "id": "daa19f74", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:23.583833Z", "start_time": "2023-11-06T19:46:23.581488Z" } }, "outputs": [ { "data": { "text/plain": [ "(1805, 1025)" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out[0].shape" ] }, { "cell_type": "code", "execution_count": 14, "id": "fa26cfe5", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:23.635848Z", "start_time": "2023-11-06T19:46:23.585904Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Counter({1024: 1757, 30: 6, 1: 5, 309: 4, 239: 3, 22: 3, 16: 2, 14: 2, 50: 1, 8: 1, 526: 1, 20: 1, 168: 1, 571: 1, 75: 1, 302: 1, 349: 1, 323: 1, 44: 1, 610: 1, 58: 1, 52: 1, 54: 1, 2: 1, 19: 1, 4: 1, 77: 1, 156: 1, 57: 1, 90: 1, 21: 1})\n" ] } ], "source": [ "from collections import Counter\n", "c = Counter(out[0].argmax(axis=1))\n", "print(c)" ] }, { "cell_type": "code", "execution_count": 16, "id": "a4d43c81", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:23.902771Z", "start_time": "2023-11-06T19:46:23.690563Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[\"rapture afterter falling apart frature fracture break me apart taste and taste are we okay wasted wasted on ego trap whatal fucking shame dr just like the rest i was never so go tell your family and go tell your friends you'll drop in just like the rest so please won't youget me'cause i'm leaving i'm leaving these memories out of head confess for all that you've done suress and tell me that i'm the one on a waste of such a beautiful face goddmn can you just get over it fuck this you're just like the rest and you are never this you made me like this dr in just like the rest just like the confess confess fucking s head and me through conf cause now this will be the end of me my paomb running the b a timeni remaining unspoken this is the b and now yourfected your face is such a beutiful word so please wont you forget me im bleeding bleeding\"]\n" ] } ], "source": [ "out = encode_filepaths([\"/home/tony/Data/Hoot/exp/6e1XdHd_RjE.wav\"])" ] }, { "cell_type": "code", "execution_count": 17, "id": "fe342e95", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:24.047911Z", "start_time": "2023-11-06T19:46:23.903995Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[\"in my kitchen there's a problem i can't tame the sink kissed up gets driving me insane every dish piled up i can't wash them away the sticky situation ruins my day oh kitchen sink won't you set me free i despise this clog its my enemy won't someone save me from this misery oh kitchen sink release my agony\"]\n" ] } ], "source": [ "out = encode_filepaths([\"/home/tony/Work/tony/audios/kitchen_sink_temperature_coarse_0.85_1.wav\"])" ] }, { "cell_type": "code", "execution_count": 18, "id": "119c9854", "metadata": { "ExecuteTime": { "end_time": "2023-11-06T19:46:36.926782Z", "start_time": "2023-11-06T19:46:36.676144Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[\"i seen the things that people wouldn't believe in i we stings blitter in the dark live out in mosergate wall my faate all those moments will be lost in time light this erase like ts erase and all lo moments will be lost in time like is erased\"]\n" ] } ], "source": [ "out = encode_filepaths([\"/home/tony/319cfea5-30ea-43b7-9654-6c82a5245046_0.mp3\"])" ] }, { "cell_type": "code", "execution_count": null, "id": "05eade7d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "172d2f5c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 7, "id": "dfa954fb", "metadata": { "ExecuteTime": { "end_time": "2023-11-01T17:53:33.559835Z", "start_time": "2023-11-01T17:53:33.504923Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 2.7 s, sys: 72.4 ms, total: 2.77 s\n", "Wall time: 52.6 ms\n" ] } ], "source": [ "%%time\n", "# ~2k realtime on a5000\n", "_ = encode_filepaths([\"../gpt/samples/halo_hq.wav\"]*100)" ] }, { "cell_type": "code", "execution_count": 10, "id": "09c97656", "metadata": { "ExecuteTime": { "end_time": "2023-11-01T17:43:56.625272Z", "start_time": "2023-11-01T17:43:56.623572Z" } }, "outputs": [], "source": [ "from suno_utils.audio import Audio" ] }, { "cell_type": "code", "execution_count": 18, "id": "5da208d4", "metadata": { "ExecuteTime": { "end_time": "2023-11-04T17:10:51.751788Z", "start_time": "2023-11-04T17:10:51.723882Z" } }, "outputs": [], "source": [ "test_audio = Audio.from_file(\"/home/tony/Data/Hoot/exp/JvqqNi82glQ.wav\")" ] }, { "cell_type": "code", "execution_count": 19, "id": "3c959a51", "metadata": { "ExecuteTime": { "end_time": "2023-11-04T17:10:52.661529Z", "start_time": "2023-11-04T17:10:52.096186Z" } }, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "test_audio.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "9a7141ca", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1f812d79", "metadata": {}, "outputs": [], "source": [ "modal run test_modal.py \\\n", " --embed-type 'hoot' \\\n", " --data-type 'youtube_music' \\\n", " --output-name 'hoot_logits'" ] }, { "cell_type": "code", "execution_count": null, "id": "c768fc04", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 29, "id": "3ff295e0", "metadata": {}, "outputs": [], "source": [ "relax_len_check = False\n", "\n", "metas_chunk = [{\"filepath\": \"blabal\", \"duration_s\": 50}]\n", "\n", "filepaths = [\n", " m.get(\"audio_filepath\", m.get(\"s3_filepath\", m.get(\"filepath\"))) if (\n", " relax_len_check or (\n", " m[\"duration_s\"] >= 10 and \n", " m[\"duration_s\"] <= 9999\n", " )\n", " ) else None\n", " for m in metas_chunk\n", "]" ] }, { "cell_type": "code", "execution_count": 30, "id": "62eb5407", "metadata": {}, "outputs": [], "source": [ "allow_continue = False\n", "assert(not allow_continue)" ] }, { "cell_type": "code", "execution_count": null, "id": "6b7d4d6a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 18, "id": "9319fe35", "metadata": { "ExecuteTime": { "end_time": "2023-11-05T02:37:41.256151Z", "start_time": "2023-11-05T02:37:41.253149Z" } }, "outputs": [], "source": [ "test_text = \"\"\"\n", "3 Direc▁tions In The ▁Exam\\n\\nAngelyn:\\n学校考试\\n准备好吗\\n老师正在发考试卷\\n老师讲解\\n这考试卷\\n讲完后就开始做考试卷\\nFerderick:\\nOh No惨了我没有为考试复习\\n我唯有去Copy他人的考卷\\n是谁敢Copy他人的考卷\\n老师就会拿起藤鞭\\nMomo:\\n哈！藤鞭来了\\nFerderick:\\n等下你就吃藤鞭不用吃方便面\\nChar Kueh Tiaw 加辣 Boom Boom\\nMomo:\\n藤鞭来了了了了了了了了了\\n\\nMomo:\\nNow\\n打我到打你\\n打我到打你\\n打我到打你\\n藤鞭来了\\nNow\\n打我到打你\\n打我到打你\\n打我到打你\\n藤鞭来了\\n\\nFerderick:\\nYa Guys Need To 复习For The 考试\\n拿着课本书包去学校诶\\nCome On Come On 复习\\n努力到死 Need To 复习\\n不临时抱着佛脚（佛脚）\\n去抄别人的考卷（考卷）\\n一定要复习不管怎样都一定要考A考A\\n\\n\\nAngelyn:\\n我惨了要拿考卷给爸爸看\\n可是怕他发现我没有考A\\n算了我唯有坦白地跟他讲\\n爸爸又会拿起藤鞭\\n\\nMomo:\\n哈！藤鞭来了\\nFerderick:\\n等下你就吃藤鞭不用吃方便面\\n炒干面加辣 Boom Boom\\nMomo:\\n藤鞭来了了了了了了了了了\\n\\nMomo:\\nNow\\n打我到打你\\n打我到打你\\n打我到打你\\n藤鞭来了\\nNow\\n打我到打你\\n打我到打你\\n打我到打你\\n藤鞭来了\\n\\n\\nFerderick:\\n我怕藤鞭所以我就要复习\\n\\nAngelyn:\\n不管怎样我都一定要复习\\n\\nFerderick:\\n所以考试我就会考到了A\\n\\nAll:\\n不要吃藤鞭就要复习\\nYeah\\n\\nFerderick:\\nBring Out The 复习本\\n\\nAll:\\nYeah\\n3 DIRECTIONS\\n复复复复复习\\n复复复复复复习\\n\\nMomo:\\n藤鞭来了\\n\\nAngelyn And Ferderick:\\n现在赶紧复习\\n考试就考到A\\n\\n复复复复复习\\n复复复复复复习\\n\\n现在赶紧复习\\n不然就会吃藤鞭\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": 25, "id": "22c57762", "metadata": { "ExecuteTime": { "end_time": "2023-11-05T02:39:27.243964Z", "start_time": "2023-11-05T02:39:27.235964Z" } }, "outputs": [], "source": [ "with open(\"test_save.json\", \"w\") as fp:\n", " json.dump(test_text, fp, ensure_ascii=True)\n", " fp.write('\\n')" ] }, { "cell_type": "code", "execution_count": 26, "id": "ebea6663", "metadata": { "ExecuteTime": { "end_time": "2023-11-05T02:39:27.415539Z", "start_time": "2023-11-05T02:39:27.413345Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "3 Direc▁tions In The ▁Exam\n", "\n", "Angelyn:\n", "学校考试\n", "准备好吗\n", "老师正在发考试卷\n", "老师讲解\n", "这考试卷\n", "讲完后就开始做考试卷\n", "Ferderick:\n", "Oh No惨了我没有为考试复习\n", "我唯有去Copy他人的考卷\n", "是谁敢Copy他人的考卷\n", "老师就会拿起藤鞭\n", "Momo:\n", "哈！藤鞭来了\n", "Ferderick:\n", "等下你就吃藤鞭不用吃方便面\n", "Char Kueh Tiaw 加辣 Boom Boom\n", "Momo:\n", "藤鞭来了了了了了了了了了\n", "\n", "Momo:\n", "Now\n", "打我到打你\n", "打我到打你\n", "打我到打你\n", "藤鞭来了\n", "Now\n", "打我到打你\n", "打我到打你\n", "打我到打你\n", "藤鞭来了\n", "\n", "Ferderick:\n", "Ya Guys Need To 复习For The 考试\n", "拿着课本书包去学校诶\n", "Come On Come On 复习\n", "努力到死 Need To 复习\n", "不临时抱着佛脚（佛脚）\n", "去抄别人的考卷（考卷）\n", "一定要复习不管怎样都一定要考A考A\n", "\n", "\n", "Angelyn:\n", "我惨了要拿考卷给爸爸看\n", "可是怕他发现我没有考A\n", "算了我唯有坦白地跟他讲\n", "爸爸又会拿起藤鞭\n", "\n", "Momo:\n", "哈！藤鞭来了\n", "Ferderick:\n", "等下你就吃藤鞭不用吃方便面\n", "炒干面加辣 Boom Boom\n", "Momo:\n", "藤鞭来了了了了了了了了了\n", "\n", "Momo:\n", "Now\n", "打我到打你\n", "打我到打你\n", "打我到打你\n", "藤鞭来了\n", "Now\n", "打我到打你\n", "打我到打你\n", "打我到打你\n", "藤鞭来了\n", "\n", "\n", "Ferderick:\n", "我怕藤鞭所以我就要复习\n", "\n", "Angelyn:\n", "不管怎样我都一定要复习\n", "\n", "Ferderick:\n", "所以考试我就会考到了A\n", "\n", "All:\n", "不要吃藤鞭就要复习\n", "Yeah\n", "\n", "Ferderick:\n", "Bring Out The 复习本\n", "\n", "All:\n", "Yeah\n", "3 DIRECTIONS\n", "复复复复复习\n", "复复复复复复习\n", "\n", "Momo:\n", "藤鞭来了\n", "\n", "Angelyn And Ferderick:\n", "现在赶紧复习\n", "考试就考到A\n", "\n", "复复复复复习\n", "复复复复复复习\n", "\n", "现在赶紧复习\n", "不然就会吃藤鞭\n", "\n" ] } ], "source": [ "with open(\"test_save.json\", \"r\") as fp:\n", " for l in fp:\n", " print(json.loads(l))" ] }, { "cell_type": "code", "execution_count": null, "id": "928f7b01", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "11dbcf5a", "metadata": {}, "source": [ "### Train" ] }, { "cell_type": "code", "execution_count": null, "id": "1c672f3d", "metadata": {}, "outputs": [], "source": [ "NCCL_ASYNC_ERROR_HANDLING=1 CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun \\\n", " --nnodes=1 --nproc_per_node=4 --node_rank=0 \\\n", " --rdzv_backend=c10d --rdzv-id=123 --max-restarts=3 --rdzv-endpoint=\"localhost:1234\" \\\n", " train.py \\\n", " --out_dir=\"/mnt/data/georg/checkpoints\" \\\n", " --data_dir=\"/mnt/data/georg/data/hoot_v1\" \\\n", " --preload_checkpoint=\"/mnt/data/georg/models/stt_en_fastconformer_ctc_large/model_weights.ckpt\" \\\n", " --max_duration_s=240 \\\n", " --gradient_accumulation_steps=1 \\\n", " --max_iters=20000 \\\n", " --log_interval=10 \\\n", " --eval_interval=500 \\\n", " --eval_iters=100 \\\n", " --batch_size=1 \\\n", " --learning_rate=3e-4 \\\n", " --min_lr=1e-5 \\\n", " --custom_seed_offset=1234 \\\n", " --wandb_log=True \\\n", " --wandb_project=\"hoot-v1\" \\\n", " --wandb_run_name=\"ft_fast\"\n", "\n", "# --preload_checkpoint=\"/mnt/data/georg/models/stt_en_fastconformer_ctc_large/model_weights.ckpt\" \\\n", "\n", "# 6s per step\n", "# TODO: figure out better scheduling/batching\n", "# using gradient accum for ddp with different sized inputs (350-1000 runtime per sample)\n", "# TODO: check slowdown through encodec" ] }, { "cell_type": "code", "execution_count": null, "id": "a180d35b", "metadata": {}, "outputs": [], "source": [ "# test run\n", "CUDA_VISIBLE_DEVICES=0 python train.py \\\n", " --out_dir=\"/mnt/data/georg/checkpoints\" \\\n", " --data_dir=\"/mnt/data/georg/data/hoot_v1\" \\\n", " --debug_val_only=True \\\n", " --max_duration_s=240 \\\n", " --gradient_accumulation_steps=6 \\\n", " --eval_iters=5 \\\n", " --batch_size=1\n", "\n", "NCCL_ASYNC_ERROR_HANDLING=1 CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun \\\n", " --nnodes=1 --nproc_per_node=4 --node_rank=0 \\\n", " --rdzv_backend=c10d --rdzv-id=123 --max-restarts=1 --rdzv-endpoint=\"localhost:1234\" \\\n", " train.py \\\n", " --out_dir=\"/mnt/data/georg/checkpoints\" \\\n", " --data_dir=\"/mnt/data/georg/data/hoot_v1\" \\\n", " --max_duration_s=240 \\\n", " --debug_val_only=True \\\n", " --eval_iters=5 \\\n", " --gradient_accumulation_steps=6 \\\n", " --log_interval=10 \\\n", " --batch_size=1 \\\n", " --custom_seed_offset=1234" ] }, { "cell_type": "code", "execution_count": null, "id": "b87c7ab6", "metadata": {}, "outputs": [], "source": [ "optim:\n", " name: adamw\n", " lr: 0.001\n", " betas:\n", " - 0.9\n", " - 0.98\n", " weight_decay: 0.001\n", " sched:\n", " name: CosineAnnealing\n", " warmup_steps: 15000\n", " warmup_ratio: null\n", " min_lr: 0.0001" ] }, { "cell_type": "code", "execution_count": null, "id": "8a36afa3", "metadata": {}, "outputs": [], "source": [ "weight_decay=1e-3 \\" ] }, { "cell_type": "code", "execution_count": 22, "id": "10fec19b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "sample_rate: 16000\r\n", "log_prediction: true\r\n", "ctc_reduction: mean_volume\r\n", "skip_nan_grad: false\r\n", "train_ds:\r\n", " manifest_filepath: null\r\n", " sample_rate: 16000\r\n", " batch_size: 1\r\n", " shuffle: true\r\n", " num_workers: 8\r\n", " pin_memory: true\r\n", " use_start_end_token: false\r\n", " trim_silence: false\r\n", " max_duration: 20\r\n", " min_duration: 0.1\r\n", " is_tarred: false\r\n", " tarred_audio_filepaths: null\r\n", " shuffle_n: 2048\r\n", " bucketing_strategy: fully_randomized\r\n", " bucketing_batch_size: null\r\n", "validation_ds:\r\n", " manifest_filepath: null\r\n", " sample_rate: 16000\r\n", " batch_size: 32\r\n", " shuffle: false\r\n", " num_workers: 8\r\n", " pin_memory: true\r\n", " use_start_end_token: false\r\n", " max_duration: 20\r\n", "test_ds:\r\n", " manifest_filepath: null\r\n", " sample_rate: 16000\r\n", " batch_size: 16\r\n", " shuffle: false\r\n", " num_workers: 8\r\n", " pin_memory: true\r\n", " use_start_end_token: false\r\n", "tokenizer:\r\n", " dir: /tokenizers/NeMo_ASR_SET/English/asr_set_3.0/tokenizer_spe_unigram_v1024\r\n", " type: bpe\r\n", " model_path: nemo:0124d42f914e45e98c214bd5afd17f55_tokenizer.model\r\n", " vocab_path: nemo:129be3e4b71e449e86261ee42b6849fa_vocab.txt\r\n", " spe_tokenizer_vocab: nemo:7561592dab144ebaaade5d1244a9ffb0_tokenizer.vocab\r\n", "preprocessor:\r\n", " _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor\r\n", " sample_rate: 16000\r\n", " normalize: per_feature\r\n", " window_size: 0.025\r\n", " window_stride: 0.01\r\n", " window: hann\r\n", " features: 80\r\n", " n_fft: 512\r\n", " log: true\r\n", " frame_splicing: 1\r\n", " dither: 1.0e-05\r\n", " pad_to: 0\r\n", " pad_value: 0.0\r\n", "spec_augment:\r\n", " _target_: nemo.collections.asr.modules.SpectrogramAugmentation\r\n", " freq_masks: 2\r\n", " time_masks: 10\r\n", " freq_width: 27\r\n", " time_width: 0.05\r\n", "encoder:\r\n", " _target_: nemo.collections.asr.modules.ConformerEncoder\r\n", " feat_in: 80\r\n", " feat_out: -1\r\n", " n_layers: 18\r\n", " d_model: 512\r\n", " subsampling: dw_striding\r\n", " subsampling_factor: 8\r\n", " subsampling_conv_channels: 256\r\n", " causal_downsampling: false\r\n", " ff_expansion_factor: 4\r\n", " self_attention_model: rel_pos\r\n", " n_heads: 8\r\n", " att_context_size:\r\n", " - -1\r\n", " - -1\r\n", " att_context_style: regular\r\n", " xscaling: true\r\n", " untie_biases: true\r\n", " pos_emb_max_len: 5000\r\n", " conv_kernel_size: 9\r\n", " conv_norm_type: batch_norm\r\n", " conv_context_size: null\r\n", " dropout: 0.1\r\n", " dropout_pre_encoder: 0.1\r\n", " dropout_emb: 0.0\r\n", " dropout_att: 0.1\r\n", "decoder:\r\n", " _target_: nemo.collections.asr.modules.ConvASRDecoder\r\n", " feat_in: 512\r\n", " num_classes: 1024\r\n", " vocabulary:\r\n", " - \r\n", " - s\r\n", " - ▁the\r\n", " - ▁a\r\n", " - t\r\n", " - ▁to\r\n", " - ▁and\r\n", " - ▁that\r\n", " - ''''\r\n", " - ▁of\r\n", " - ▁i\r\n", " - ▁in\r\n", " - d\r\n", " - ed\r\n", " - ing\r\n", " - ▁it\r\n", " - ▁you\r\n", " - 'n'\r\n", " - e\r\n", " - ▁is\r\n", " - 'y'\r\n", " - er\r\n", " - ▁for\r\n", " - m\r\n", " - ▁was\r\n", " - ▁we\r\n", " - re\r\n", " - ly\r\n", " - r\r\n", " - ▁he\r\n", " - ▁s\r\n", " - ▁be\r\n", " - c\r\n", " - ▁this\r\n", " - ▁so\r\n", " - ▁on\r\n", " - ▁have\r\n", " - g\r\n", " - a\r\n", " - ▁not\r\n", " - or\r\n", " - p\r\n", " - ▁they\r\n", " - ▁with\r\n", " - o\r\n", " - al\r\n", " - ▁but\r\n", " - ▁c\r\n", " - ▁are\r\n", " - ▁re\r\n", " - ▁there\r\n", " - ar\r\n", " - b\r\n", " - l\r\n", " - le\r\n", " - ▁know\r\n", " - ▁as\r\n", " - in\r\n", " - u\r\n", " - ▁what\r\n", " - ▁do\r\n", " - ▁or\r\n", " - ▁\r\n", " - ▁at\r\n", " - ation\r\n", " - w\r\n", " - ll\r\n", " - f\r\n", " - ▁one\r\n", " - ▁e\r\n", " - ▁f\r\n", " - ri\r\n", " - ▁if\r\n", " - ▁b\r\n", " - ent\r\n", " - ▁all\r\n", " - ▁de\r\n", " - ▁can\r\n", " - it\r\n", " - ▁would\r\n", " - ▁me\r\n", " - il\r\n", " - ▁like\r\n", " - ▁an\r\n", " - ce\r\n", " - ▁no\r\n", " - ▁t\r\n", " - ment\r\n", " - i\r\n", " - ve\r\n", " - k\r\n", " - ▁from\r\n", " - ▁just\r\n", " - ▁by\r\n", " - ▁his\r\n", " - ▁had\r\n", " - ion\r\n", " - ▁think\r\n", " - es\r\n", " - ch\r\n", " - 'on'\r\n", " - ▁about\r\n", " - th\r\n", " - ▁my\r\n", " - ck\r\n", " - en\r\n", " - ▁were\r\n", " - ▁your\r\n", " - ur\r\n", " - ▁will\r\n", " - ic\r\n", " - ▁go\r\n", " - ▁con\r\n", " - ▁she\r\n", " - li\r\n", " - an\r\n", " - ▁out\r\n", " - ter\r\n", " - ▁because\r\n", " - se\r\n", " - la\r\n", " - ▁when\r\n", " - ▁st\r\n", " - ▁has\r\n", " - ▁don\r\n", " - ▁which\r\n", " - ir\r\n", " - ra\r\n", " - ▁p\r\n", " - ▁two\r\n", " - ▁right\r\n", " - ▁court\r\n", " - ▁some\r\n", " - ▁pa\r\n", " - ▁pro\r\n", " - lo\r\n", " - ate\r\n", " - ▁time\r\n", " - ▁uh\r\n", " - te\r\n", " - ▁up\r\n", " - ▁our\r\n", " - ▁g\r\n", " - ▁case\r\n", " - ▁w\r\n", " - ▁her\r\n", " - ▁well\r\n", " - ro\r\n", " - ▁their\r\n", " - ▁who\r\n", " - ▁been\r\n", " - ▁yeah\r\n", " - ▁said\r\n", " - ers\r\n", " - ▁mo\r\n", " - ▁other\r\n", " - ▁see\r\n", " - ive\r\n", " - tion\r\n", " - el\r\n", " - ▁more\r\n", " - ▁se\r\n", " - vi\r\n", " - ▁any\r\n", " - ▁get\r\n", " - ▁also\r\n", " - ▁fa\r\n", " - ▁here\r\n", " - ▁un\r\n", " - ▁now\r\n", " - ▁co\r\n", " - ▁people\r\n", " - age\r\n", " - ist\r\n", " - ul\r\n", " - ▁how\r\n", " - ▁going\r\n", " - h\r\n", " - ▁very\r\n", " - ▁then\r\n", " - ▁could\r\n", " - de\r\n", " - ▁ma\r\n", " - ver\r\n", " - ▁mean\r\n", " - ▁su\r\n", " - id\r\n", " - ng\r\n", " - ▁lo\r\n", " - ▁sa\r\n", " - ▁la\r\n", " - ▁sp\r\n", " - ▁us\r\n", " - ol\r\n", " - ▁pre\r\n", " - ance\r\n", " - ▁want\r\n", " - ▁work\r\n", " - ld\r\n", " - ▁did\r\n", " - ge\r\n", " - ▁state\r\n", " - ▁po\r\n", " - ▁dis\r\n", " - ity\r\n", " - able\r\n", " - ▁say\r\n", " - ▁where\r\n", " - et\r\n", " - ▁ra\r\n", " - us\r\n", " - ▁li\r\n", " - ry\r\n", " - ction\r\n", " - ▁um\r\n", " - nd\r\n", " - ne\r\n", " - ▁even\r\n", " - ▁him\r\n", " - ▁ba\r\n", " - ad\r\n", " - ▁these\r\n", " - ▁part\r\n", " - ▁really\r\n", " - lu\r\n", " - ▁them\r\n", " - ▁those\r\n", " - ▁ho\r\n", " - v\r\n", " - ▁bo\r\n", " - ▁ex\r\n", " - ect\r\n", " - ▁over\r\n", " - ▁make\r\n", " - ru\r\n", " - ▁ro\r\n", " - is\r\n", " - ▁fi\r\n", " - ▁m\r\n", " - ut\r\n", " - ke\r\n", " - ence\r\n", " - ▁bu\r\n", " - ▁ch\r\n", " - ▁comp\r\n", " - me\r\n", " - ted\r\n", " - ▁en\r\n", " - ▁fact\r\n", " - z\r\n", " - ▁into\r\n", " - ▁need\r\n", " - ure\r\n", " - ▁way\r\n", " - ies\r\n", " - end\r\n", " - ▁mi\r\n", " - ▁take\r\n", " - un\r\n", " - ▁only\r\n", " - ▁should\r\n", " - as\r\n", " - ▁look\r\n", " - ▁three\r\n", " - one\r\n", " - ▁v\r\n", " - co\r\n", " - ▁com\r\n", " - ▁man\r\n", " - ta\r\n", " - ▁good\r\n", " - ated\r\n", " - ▁back\r\n", " - ▁first\r\n", " - ▁mr\r\n", " - ▁point\r\n", " - x\r\n", " - ▁new\r\n", " - ant\r\n", " - ▁vi\r\n", " - ▁dr\r\n", " - ff\r\n", " - ▁k\r\n", " - ci\r\n", " - ▁app\r\n", " - um\r\n", " - ▁n\r\n", " - ▁exp\r\n", " - ▁under\r\n", " - ▁after\r\n", " - po\r\n", " - ight\r\n", " - ▁d\r\n", " - est\r\n", " - mp\r\n", " - ▁okay\r\n", " - ine\r\n", " - ti\r\n", " - mo\r\n", " - qu\r\n", " - ▁da\r\n", " - ▁come\r\n", " - ▁day\r\n", " - ▁oh\r\n", " - ▁issue\r\n", " - at\r\n", " - ▁question\r\n", " - ▁than\r\n", " - ▁pri\r\n", " - ho\r\n", " - ten\r\n", " - ▁ca\r\n", " - ▁off\r\n", " - ain\r\n", " - ▁di\r\n", " - ▁o\r\n", " - ▁much\r\n", " - ▁five\r\n", " - ow\r\n", " - ▁ha\r\n", " - ▁thousand\r\n", " - ▁six\r\n", " - ▁give\r\n", " - ▁let\r\n", " - ▁le\r\n", " - ary\r\n", " - ous\r\n", " - sh\r\n", " - ▁yes\r\n", " - ▁four\r\n", " - ▁before\r\n", " - ▁call\r\n", " - ▁ga\r\n", " - ▁ne\r\n", " - ▁law\r\n", " - ma\r\n", " - ▁tra\r\n", " - ish\r\n", " - ical\r\n", " - ▁something\r\n", " - iv\r\n", " - ▁why\r\n", " - ▁may\r\n", " - ▁talk\r\n", " - ▁actually\r\n", " - ig\r\n", " - ▁act\r\n", " - ▁got\r\n", " - ▁car\r\n", " - ▁through\r\n", " - ful\r\n", " - ▁made\r\n", " - ▁hundred\r\n", " - om\r\n", " - man\r\n", " - ia\r\n", " - ▁didn\r\n", " - ual\r\n", " - ▁kind\r\n", " - ▁sta\r\n", " - ting\r\n", " - ▁things\r\n", " - ell\r\n", " - ▁down\r\n", " - im\r\n", " - ▁years\r\n", " - ▁lot\r\n", " - ▁ri\r\n", " - pe\r\n", " - ard\r\n", " - he\r\n", " - ▁gra\r\n", " - ▁year\r\n", " - ▁every\r\n", " - ▁eight\r\n", " - ▁reason\r\n", " - ▁seven\r\n", " - ca\r\n", " - ise\r\n", " - ▁twenty\r\n", " - ▁many\r\n", " - ▁per\r\n", " - op\r\n", " - ▁little\r\n", " - ▁mar\r\n", " - ▁same\r\n", " - ▁find\r\n", " - ▁different\r\n", " - ▁wa\r\n", " - pp\r\n", " - ▁nine\r\n", " - if\r\n", " - ▁does\r\n", " - ie\r\n", " - ach\r\n", " - cu\r\n", " - ▁never\r\n", " - ▁thing\r\n", " - ▁whether\r\n", " - ward\r\n", " - ▁again\r\n", " - ▁put\r\n", " - ▁claim\r\n", " - ▁op\r\n", " - ugh\r\n", " - ▁show\r\n", " - tic\r\n", " - ial\r\n", " - ▁still\r\n", " - ▁government\r\n", " - ok\r\n", " - ▁th\r\n", " - ▁long\r\n", " - ▁gu\r\n", " - ▁being\r\n", " - ▁plan\r\n", " - ▁own\r\n", " - am\r\n", " - ian\r\n", " - ▁evidence\r\n", " - ot\r\n", " - mb\r\n", " - ▁most\r\n", " - ▁ru\r\n", " - mit\r\n", " - ▁thank\r\n", " - ious\r\n", " - ▁person\r\n", " - em\r\n", " - ▁use\r\n", " - na\r\n", " - ▁sha\r\n", " - ▁r\r\n", " - ▁believe\r\n", " - mi\r\n", " - ▁school\r\n", " - ▁record\r\n", " - ition\r\n", " - ▁imp\r\n", " - ound\r\n", " - ▁place\r\n", " - ities\r\n", " - ▁happen\r\n", " - ▁ph\r\n", " - ▁great\r\n", " - ative\r\n", " - ha\r\n", " - 'no'\r\n", " - ▁cha\r\n", " - ite\r\n", " - ▁clear\r\n", " - ▁help\r\n", " - dic\r\n", " - ea\r\n", " - ▁high\r\n", " - ▁judge\r\n", " - ▁al\r\n", " - ▁last\r\n", " - di\r\n", " - qui\r\n", " - ▁sure\r\n", " - tra\r\n", " - go\r\n", " - ▁sh\r\n", " - ▁both\r\n", " - ▁read\r\n", " - ving\r\n", " - ▁district\r\n", " - ▁ref\r\n", " - ▁sub\r\n", " - ▁hi\r\n", " - ▁such\r\n", " - ▁ten\r\n", " - ▁start\r\n", " - ▁doesn\r\n", " - ph\r\n", " - ▁change\r\n", " - ay\r\n", " - ▁count\r\n", " - ▁ob\r\n", " - ▁found\r\n", " - ▁du\r\n", " - ▁ki\r\n", " - ▁pay\r\n", " - ▁order\r\n", " - ▁another\r\n", " - for\r\n", " - ▁hand\r\n", " - lic\r\n", " - ▁number\r\n", " - ▁too\r\n", " - ▁ta\r\n", " - ni\r\n", " - ens\r\n", " - ▁tell\r\n", " - ven\r\n", " - port\r\n", " - ▁fun\r\n", " - ▁h\r\n", " - ▁term\r\n", " - ▁end\r\n", " - rr\r\n", " - ▁must\r\n", " - ▁home\r\n", " - ▁hu\r\n", " - ▁wi\r\n", " - side\r\n", " - ▁certain\r\n", " - ▁name\r\n", " - ▁understand\r\n", " - ▁money\r\n", " - ize\r\n", " - ▁pu\r\n", " - ▁interest\r\n", " - ip\r\n", " - ▁came\r\n", " - ▁house\r\n", " - land\r\n", " - ▁men\r\n", " - ▁require\r\n", " - ▁says\r\n", " - min\r\n", " - ▁member\r\n", " - our\r\n", " - ating\r\n", " - ber\r\n", " - ▁jo\r\n", " - ▁might\r\n", " - vo\r\n", " - ▁support\r\n", " - ries\r\n", " - son\r\n", " - ▁problem\r\n", " - ling\r\n", " - ▁old\r\n", " - ▁pe\r\n", " - ▁second\r\n", " - ▁def\r\n", " - ness\r\n", " - ▁decision\r\n", " - row\r\n", " - ▁dollars\r\n", " - ▁mu\r\n", " - ▁conf\r\n", " - use\r\n", " - ▁ah\r\n", " - ▁va\r\n", " - ▁life\r\n", " - ▁inter\r\n", " - ▁play\r\n", " - va\r\n", " - ▁saying\r\n", " - ▁pi\r\n", " - ▁far\r\n", " - ▁went\r\n", " - ▁public\r\n", " - ▁war\r\n", " - ▁trans\r\n", " - ▁live\r\n", " - ▁ask\r\n", " - ▁real\r\n", " - rate\r\n", " - ▁important\r\n", " - ▁thought\r\n", " - ▁done\r\n", " - ▁between\r\n", " - ▁argument\r\n", " - ▁rep\r\n", " - ▁consider\r\n", " - ▁anything\r\n", " - ▁direct\r\n", " - less\r\n", " - ▁ja\r\n", " - ▁doing\r\n", " - ▁course\r\n", " - line\r\n", " - ▁month\r\n", " - ▁word\r\n", " - ▁care\r\n", " - ▁provide\r\n", " - ap\r\n", " - ▁honor\r\n", " - ba\r\n", " - ator\r\n", " - j\r\n", " - ▁report\r\n", " - ▁exc\r\n", " - ▁service\r\n", " - ▁tri\r\n", " - nk\r\n", " - ▁world\r\n", " - ▁without\r\n", " - ▁ju\r\n", " - ▁against\r\n", " - ible\r\n", " - ▁used\r\n", " - ▁always\r\n", " - ▁able\r\n", " - ship\r\n", " - ▁rule\r\n", " - ▁matter\r\n", " - ▁while\r\n", " - par\r\n", " - ▁maybe\r\n", " - ▁miss\r\n", " - ▁around\r\n", " - ▁u\r\n", " - ▁today\r\n", " - ▁friend\r\n", " - ▁ti\r\n", " - ▁big\r\n", " - bu\r\n", " - ▁next\r\n", " - ju\r\n", " - ▁thirty\r\n", " - ▁pass\r\n", " - ▁y\r\n", " - ▁business\r\n", " - ▁test\r\n", " - ▁fo\r\n", " - ▁nothing\r\n", " - ▁turn\r\n", " - ▁process\r\n", " - com\r\n", " - ▁el\r\n", " - ▁better\r\n", " - ▁commission\r\n", " - ▁week\r\n", " - ▁move\r\n", " - ▁trial\r\n", " - ▁open\r\n", " - ▁each\r\n", " - ▁specific\r\n", " - ▁feel\r\n", " - ▁cap\r\n", " - ency\r\n", " - ▁particular\r\n", " - ability\r\n", " - ▁system\r\n", " - gra\r\n", " - ▁present\r\n", " - ▁looking\r\n", " - duc\r\n", " - ▁fra\r\n", " - ▁correct\r\n", " - ▁allow\r\n", " - ▁side\r\n", " - ign\r\n", " - serv\r\n", " - ▁city\r\n", " - ▁agree\r\n", " - ▁having\r\n", " - ▁defendant\r\n", " - ▁keep\r\n", " - ▁follow\r\n", " - ▁invest\r\n", " - ▁add\r\n", " - ▁information\r\n", " - ▁whole\r\n", " - ▁general\r\n", " - hi\r\n", " - ▁away\r\n", " - ▁upon\r\n", " - ▁win\r\n", " - ▁based\r\n", " - ▁program\r\n", " - ▁develop\r\n", " - ▁country\r\n", " - ified\r\n", " - ▁sort\r\n", " - ▁board\r\n", " - ▁qua\r\n", " - ex\r\n", " - ▁family\r\n", " - ▁close\r\n", " - ▁dec\r\n", " - ▁though\r\n", " - ▁hard\r\n", " - ▁answer\r\n", " - ▁guess\r\n", " - ▁job\r\n", " - ▁trying\r\n", " - ▁je\r\n", " - ▁asked\r\n", " - ▁during\r\n", " - ▁fee\r\n", " - ▁love\r\n", " - ▁already\r\n", " - ▁address\r\n", " - ▁ever\r\n", " - ▁bill\r\n", " - gue\r\n", " - ▁cause\r\n", " - ▁gen\r\n", " - ▁free\r\n", " - ▁continue\r\n", " - ▁fifty\r\n", " - ▁contract\r\n", " - ▁european\r\n", " - ▁rest\r\n", " - ▁few\r\n", " - ▁water\r\n", " - ▁line\r\n", " - ▁position\r\n", " - ▁percent\r\n", " - ▁enough\r\n", " - ▁sea\r\n", " - ▁har\r\n", " - ▁america\r\n", " - ▁full\r\n", " - ▁since\r\n", " - ▁everything\r\n", " - ▁bring\r\n", " - ▁lead\r\n", " - ▁probably\r\n", " - ▁children\r\n", " - ▁tax\r\n", " - ▁face\r\n", " - ▁best\r\n", " - ▁concern\r\n", " - ▁sign\r\n", " - ▁mhm\r\n", " - ▁minute\r\n", " - ▁bit\r\n", " - ▁stand\r\n", " - ▁situation\r\n", " - ▁speak\r\n", " - ▁hope\r\n", " - ▁yet\r\n", " - ▁discuss\r\n", " - ▁standard\r\n", " - ▁brief\r\n", " - ▁statute\r\n", " - ▁officer\r\n", " - ▁ok\r\n", " - ▁respect\r\n", " - ▁together\r\n", " - ▁agreement\r\n", " - ▁room\r\n", " - ▁least\r\n", " - ▁market\r\n", " - ▁however\r\n", " - ▁getting\r\n", " - ▁else\r\n", " - ▁true\r\n", " - ▁counsel\r\n", " - ttle\r\n", " - ▁err\r\n", " - ▁effect\r\n", " - ▁example\r\n", " - ▁less\r\n", " - ▁cost\r\n", " - ▁left\r\n", " - ▁group\r\n", " - ▁level\r\n", " - ▁policy\r\n", " - ▁town\r\n", " - ▁small\r\n", " - ▁child\r\n", " - ▁president\r\n", " - ▁large\r\n", " - ▁later\r\n", " - ▁protect\r\n", " - ▁decide\r\n", " - way\r\n", " - ▁making\r\n", " - ▁regard\r\n", " - ▁book\r\n", " - ▁pretty\r\n", " - ▁author\r\n", " - ▁hour\r\n", " - ▁idea\r\n", " - ▁include\r\n", " - ▁exactly\r\n", " - ▁power\r\n", " - ▁limit\r\n", " - ▁watch\r\n", " - ▁night\r\n", " - ▁ya\r\n", " - ▁jury\r\n", " - ▁leave\r\n", " - ▁quite\r\n", " - ▁cover\r\n", " - ▁grow\r\n", " - ▁result\r\n", " - ▁singapore\r\n", " - ▁either\r\n", " - ▁company\r\n", " - ▁stop\r\n", " - ▁community\r\n", " - ▁million\r\n", " - ▁current\r\n", " - ▁federal\r\n", " - ▁united\r\n", " - ▁mind\r\n", " - ▁step\r\n", " - ▁class\r\n", " - ▁police\r\n", " - ▁remain\r\n", " - ▁health\r\n", " - ▁final\r\n", " - ▁represent\r\n", " - ▁light\r\n", " - ▁treat\r\n", " - ▁review\r\n", " - ▁testimony\r\n", " - ▁bank\r\n", " - ▁product\r\n", " - ▁appeal\r\n", " - ▁half\r\n", " - ▁cannot\r\n", " - ▁involve\r\n", " - ▁charge\r\n", " - ▁knew\r\n", " - ▁learn\r\n", " - ▁eye\r\n", " - ▁language\r\n", " - ▁individual\r\n", " - ▁receive\r\n", " - ▁further\r\n", " - ▁mister\r\n", " - ▁view\r\n", " - ▁council\r\n", " - ▁until\r\n", " - ▁human\r\n", " - ▁stuff\r\n", " - ▁student\r\n", " - ▁become\r\n", " - ▁walk\r\n", " - ▁control\r\n", " - ▁check\r\n", " - ▁seems\r\n", " - ▁strong\r\n", " - ▁period\r\n", " - ▁please\r\n", " - ▁food\r\n", " - ▁type\r\n", " - ▁complete\r\n", " - ▁purpose\r\n", " - ▁account\r\n", " - ▁project\r\n", " - ▁sense\r\n", " - ▁national\r\n", " - ▁appear\r\n", " - ▁simply\r\n", " - ▁return\r\n", " - ▁benefit\r\n", " - ▁employ\r\n", " - ▁create\r\n", " - ▁wrong\r\n", " - ▁whatever\r\n", " - ▁morning\r\n", " - ▁suggest\r\n", " - ▁condition\r\n", " - ▁amount\r\n", " - ▁object\r\n", " - ▁third\r\n", " - ▁remember\r\n", " - ▁himself\r\n", " - ▁property\r\n", " - ▁itself\r\n", " - ▁somebody\r\n", " - ▁suppose\r\n", " - ▁future\r\n", " - ▁short\r\n", " - ▁opinion\r\n", " - ▁several\r\n", " - ▁common\r\n", " - ▁taking\r\n", " - ▁notice\r\n", " - ▁office\r\n", " - ▁almost\r\n", " - ▁father\r\n", " - ▁document\r\n", " - ▁judgment\r\n", " - ▁experience\r\n", " - ▁uhhuh\r\n", " - ▁committee\r\n", " - ▁parent\r\n", " - ▁legal\r\n", " - ▁client\r\n", " - ▁major\r\n", " - ▁congress\r\n", " - ▁rather\r\n", " - ▁proceed\r\n", " - ▁possible\r\n", " - ▁plea\r\n", " - ▁subject\r\n", " - ▁along\r\n", " - ▁mother\r\n", " - ▁increase\r\n", " - ▁meeting\r\n", " - ▁girl\r\n", " - ▁couple\r\n", " - ▁north\r\n", " - ▁local\r\n", " - ▁sometimes\r\n", " - ▁brought\r\n", " - ▁fifteen\r\n", " - ▁organ\r\n", " - ▁difference\r\n", " - ▁entire\r\n", " - ▁security\r\n", " - ▁department\r\n", " - ▁south\r\n", " - ▁vote\r\n", " - ▁basis\r\n", " - ▁twelve\r\n", " - ▁conduct\r\n", " - ▁street\r\n", " - ▁social\r\n", " - ▁wonder\r\n", " - ▁supreme\r\n", " - ▁difficult\r\n", " - ▁travel\r\n", " - ▁opportunity\r\n", " - ▁defense\r\n", " - ▁plaintiff\r\n", " - ▁white\r\n", " - ▁focus\r\n", " - ▁therefore\r\n", " - ▁challenge\r\n", " - ▁value\r\n", " - ▁except\r\n", " - ▁among\r\n", " - ▁ground\r\n", " - ▁serve\r\n", " - ▁attorney\r\n", " - ▁education\r\n", " - ▁credit\r\n", " - ▁request\r\n", " - ▁women\r\n", " - ▁basically\r\n", " - ▁political\r\n", " - ▁determine\r\n", " - ▁access\r\n", " - ▁across\r\n", " - ▁sentence\r\n", " - ▁perform\r\n", " - ▁everybody\r\n", " - ▁black\r\n", " - ▁drug\r\n", " - ▁accept\r\n", " - ▁eleven\r\n", " - ▁establish\r\n", " - ▁design\r\n", " - ▁success\r\n", " - ▁including\r\n", " - ▁john\r\n", " - ▁employee\r\n", " - ▁nice\r\n", " - ▁crime\r\n", " - ibility\r\n", " - ▁circumstance\r\n", " - ▁occur\r\n", " - ▁citizen\r\n", " - ▁woman\r\n", " - ▁witness\r\n", " - ▁explain\r\n", " - ▁additional\r\n", " - ▁history\r\n", " - ▁front\r\n", " - ▁medical\r\n", " - ▁death\r\n", " - ification\r\n", " - ▁financial\r\n", " - ▁break\r\n", " - ▁connect\r\n", " - ▁perhaps\r\n", " - ▁sorry\r\n", " - ▁especially\r\n", " - ▁trust\r\n", " - ▁insurance\r\n", " - ▁economic\r\n", " - ▁effort\r\n", " - ▁circuit\r\n", " - ▁cross\r\n", " - ▁instead\r\n", " - ▁affect\r\n", " - ▁zero\r\n", " - ▁available\r\n", " - ▁team\r\n", " - ▁application\r\n", " - ▁separate\r\n", " - ▁absolutely\r\n", " - ▁christ\r\n", " - ization\r\n", " - ▁europe\r\n", " - ▁budget\r\n", " - ▁damage\r\n", " - ▁myself\r\n", " - ▁private\r\n", " - ▁figure\r\n", " - ▁provision\r\n", " - ▁collect\r\n", " - ▁similar\r\n", " - ▁regulation\r\n", " - ▁approach\r\n", " - ▁themselves\r\n", " - ▁construct\r\n", " - ▁petition\r\n", " - ▁material\r\n", " - ▁improve\r\n", " - ▁york\r\n", " - ▁parliament\r\n", " - ▁broad\r\n", " - ▁companies\r\n", " - ▁nature\r\n", " - ▁piece\r\n", " - ▁threat\r\n", " - ▁energy\r\n", " - ▁behind\r\n", " - ▁significant\r\n", " - ▁response\r\n", " - ▁definitely\r\n", " - ▁guide\r\n", " - ▁beyond\r\n", " - ▁amendment\r\n", " - ▁measure\r\n", " - ▁criminal\r\n", " - ▁original\r\n", " - ▁brother\r\n", " - ▁impact\r\n", " - q\r\n", "optim:\r\n", " name: adamw\r\n", " lr: 0.001\r\n", " betas:\r\n", " - 0.9\r\n", " - 0.98\r\n", " weight_decay: 0.001\r\n", " sched:\r\n", " name: CosineAnnealing\r\n", " warmup_steps: 15000\r\n", " warmup_ratio: null\r\n", " min_lr: 0.0001\r\n", "target: nemo.collections.asr.models.ctc_bpe_models.EncDecCTCModelBPE\r\n", "nemo_version: 1.13.0rc0\r\n", "decoding:\r\n", " strategy: greedy\r\n", " preserve_alignments: null\r\n", " compute_timestamps: null\r\n", " confidence_cfg:\r\n", " preserve_frame_confidence: false\r\n", " preserve_token_confidence: false\r\n", " preserve_word_confidence: false\r\n", " exclude_blank: true\r\n", " aggregation: min\r\n", " method_cfg:\r\n", " name: entropy\r\n", " entropy_type: tsallis\r\n", " temperature: 0.33\r\n", " entropy_norm: exp\r\n", " word_seperator: ' '\r\n", " ctc_timestamp_type: all\r\n", " batch_dim_index: 0\r\n", " greedy:\r\n", " preserve_alignments: false\r\n", " compute_timestamps: false\r\n", " preserve_frame_confidence: false\r\n", " confidence_method_cfg: null\r\n" ] } ], "source": [ "!cat /mnt/data/georg/models/stt_en_fastconformer_ctc_large/model_config.yaml" ] }, { "cell_type": "code", "execution_count": null, "id": "78a4f9dc", "metadata": {}, "outputs": [], "source": [ "train.py --out_dir=/mnt/data/georg/checkpoints \\\n", "--data_dir=/mnt/data/georg/data/hoot_v1 \\\n", "--max_duration_s=240 \\\n", "--gradient_accumulation_steps=1 \\\n", "--max_iters=50000 \\\n", "--warmup_iters=5000 \\\n", "--log_interval=10 \\\n", "--eval_interval=500 \\\n", "--eval_iters=100 \\\n", "--batch_size=1 \\\n", "--learning_rate=3e-4 \\\n", "--min_lr=1e-5 \\\n", "--custom_seed_offset=1234 \\\n", "--wandb_log=True \\\n", "--wandb_project=hoot \\\n", "--wandb_run_name=suno_base_fast\n" ] }, { "cell_type": "code", "execution_count": null, "id": "82258096", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "264a9a0b", "metadata": {}, "outputs": [], "source": [ "OMP_NUM_THREADS=6 python -u /home/tony/Work/glockenspiel/mert_training/src/fairseq/fairseq_cli/hydra_train.py \\\n", " --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n", " --config-name 'custom_95M_fsq' \\\n", " common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n", " common.wandb_project='mert_test' \\\n", " checkpoint.save_dir='/app/suno/checkpoints/mert_encode_fsq_mid_deeper' \\\n", " distributed_training.distributed_rank=0 \\\n", " distributed_training.distributed_world_size=8 \\\n", " distributed_training.nprocs_per_node=8 \\\n", " optimization.update_freq='[1]' \\\n", " distributed_training.distributed_init_method=\"tcp://127.0.0.2:39683\" \\\n", " task.data='/app/suno/data/mert_25hz_short/audio_speech_tsv' \\\n", " task.label_dir='/app/suno/data/mert_25hz_short/speech_label' \\\n", " task.labels='[\"codec_0\", \"codec_1\"]' \\\n", " dataset.num_workers=6 \\\n", " dataset.max_tokens=5000000 \\\n", " dataset.disable_validation=false \\\n", " model.label_rate=25" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }