{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "5531ebee", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:19:31.086683Z", "start_time": "2024-01-16T17:19:31.085341Z" } }, "outputs": [], "source": [ "# https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo/models/stt_en_fastconformer_ctc_large" ] }, { "cell_type": "code", "execution_count": null, "id": "853b13c2", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:19:31.171433Z", "start_time": "2024-01-16T17:19:31.087630Z" } }, "outputs": [], "source": [ "# import os\n", "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"6\"\n", "\n", "# import torch\n", "# from hoot.model import Hoot, Tokenizer" ] }, { "cell_type": "code", "execution_count": null, "id": "b64dd744", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:19:31.233044Z", "start_time": "2024-01-16T17:19:31.172322Z" } }, "outputs": [], "source": [ "# model = Hoot.from_nemo(\n", "# \"/mnt/data/georg/models/stt_en_fastconformer_ctc_large/model_weights.ckpt\",\n", "# )\n", "# model.eval()\n", "# model.to(\"cuda\");\n", "# tokenizer = Tokenizer(\n", "# \"/mnt/data/georg/models/stt_en_fastconformer_ctc_large/0124d42f914e45e98c214bd5afd17f55_tokenizer.model\"\n", "# )\n", "\n", "# logits = model.infer(\"../gpt/samples/covid.wav\")\n", "# tokenizer.decode_logits(logits)" ] }, { "cell_type": "code", "execution_count": null, "id": "4a6fc758", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:19:31.294866Z", "start_time": "2024-01-16T17:19:31.234639Z" } }, "outputs": [], "source": [ "# # check custom\n", "# model = Hoot.from_checkpoint(\n", "# \"/mnt/data/georg/checkpoints/2023-05-26_16-55-38/last_ckpt.pt\",\n", "# )\n", "# model.eval()\n", "# model.to(\"cuda\");\n", "\n", "# logits = model.infer(\"../gpt/samples/halo_hq.wav\")\n", "# tokenizer.decode_logits(logits)\n", "\n", "# logits = model.infer(\"../gpt/samples/covid.wav\")\n", "# tokenizer.decode_logits(logits)" ] }, { "cell_type": "markdown", "id": "3e76fc0c", "metadata": {}, "source": [ "## Test API" ] }, { "cell_type": "code", "execution_count": null, "id": "47a0112c", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:53.969833Z", "start_time": "2024-01-16T17:19:31.295733Z" } }, "outputs": [], "source": [ "import os\n", "from suno_utils.tasks.hoot import preload_models, encode_filepaths\n", "# v0 model, en \n", "# model_dir = \"s3://suno-data/datasets/bundles/v1/models/hoot_v0\"\n", "# _ = preload_models(\n", "# checkpoint_filepath=os.path.join(model_dir, \"2023-05-26_16-55-38_last_ckpt.pt\"),\n", "# tokenizer_filepath=os.path.join(model_dir, \"tokenizer.model\"),\n", "# )\n", "# v1 model, mulit-lingo\n", "# model_dir = \"s3://suno-data/checkpoints/hoot_v1\"\n", "# v3 model, mulit-lingo\n", "model_dir = \"s3://suno-data/checkpoints/hoot_v3\"\n", "_ = preload_models(\n", " checkpoint_filepath=os.path.join(model_dir, \"hoot_ckpt.pt\"),\n", " tokenizer_filepath=os.path.join(model_dir, \"tokenizer.model\"),\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "7b43fef5", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:55.983638Z", "start_time": "2024-01-16T17:20:53.971893Z" } }, "outputs": [], "source": [ "transcription = encode_filepaths([\"/home/tony/Work/tony/audios/test_adele.mp3\"])" ] }, { "cell_type": "code", "execution_count": null, "id": "84609414", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:55.987554Z", "start_time": "2024-01-16T17:20:55.985246Z" } }, "outputs": [], "source": [ "print(transcription)" ] }, { "cell_type": "markdown", "id": "f15e477e", "metadata": {}, "source": [ "## Use glock" ] }, { "cell_type": "code", "execution_count": null, "id": "71cd2c86", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:56.053233Z", "start_time": "2024-01-16T17:20:55.988645Z" } }, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"2\"" ] }, { "cell_type": "code", "execution_count": null, "id": "09a75d69", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:56.119588Z", "start_time": "2024-01-16T17:20:56.054294Z" } }, "outputs": [], "source": [ "from suno_utils.audio import Audio\n", "from collections import Counter\n", "from suno_utils.tasks.hoot import preload_models, encode_filepaths, encode, Tokenizer, load_model_list, ctc_align\n", "from suno_utils.utils.metrics import get_cer" ] }, { "cell_type": "code", "execution_count": null, "id": "ef3225f2", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:56.179821Z", "start_time": "2024-01-16T17:20:56.120625Z" } }, "outputs": [], "source": [ "# model_dir = \"s3://suno-data/datasets/bundles/v1/models/hoot_v0\"\n", "# _ = preload_models(\n", "# checkpoint_filepath=os.path.join(model_dir, \"2023-05-26_16-55-38_last_ckpt.pt\"),\n", "# tokenizer_filepath=os.path.join(model_dir, \"tokenizer.model\"),\n", "# )" ] }, { "cell_type": "code", "execution_count": null, "id": "501df841", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:56.239821Z", "start_time": "2024-01-16T17:20:56.180809Z" } }, "outputs": [], "source": [ "# !ls -t \"/home/tony/Data/checkpoints/hoot/2023-11-05_21-30-28\"" ] }, { "cell_type": "code", "execution_count": null, "id": "5858503b", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:56.297985Z", "start_time": "2024-01-16T17:20:56.240777Z" } }, "outputs": [], "source": [ "# # this is a good en model\n", "# _ = preload_models(\n", "# checkpoint_filepath=os.path.join(\"/home/tony/Data/checkpoints/hoot/2023-11-06_03-19-40\", \"last_ckpt.pt\"), \n", "# # tokenizer_filepath=\"/home/tony/Work/tony/hoot/tokenizers/en3/tokenizer_spe_bpe_v4096/tokenizer.model\",\n", "# tokenizer_filepath=\"/home/tony/Data/Hoot/tokenizer_v1024_default.model\"\n", "# )" ] }, { "cell_type": "code", "execution_count": null, "id": "fc25857c", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:56.381535Z", "start_time": "2024-01-16T17:20:56.300551Z" } }, "outputs": [], "source": [ "model_list = load_model_list(\n", " # checkpoint_filepath=os.path.join(\"/home/tony/Data/checkpoints/hoot/2023-11-09_16-40-39\", \"246k_ckpt.pt\"), \n", " # checkpoint_filepath=os.path.join(\"/home/tony/Data/checkpoints/hoot/2023-11-17_15-08-57\", \"28k_ckpt.pt\"), \n", " # checkpoint_filepath=os.path.join(\"/home/tony/Data/checkpoints/hoot/2023-11-19_18-38-15\", \"28k_ckpt.pt\"), # good multi-lingo large vocab\n", " # checkpoint_filepath=os.path.join(\"/home/tony/Data/checkpoints/hoot/2024-01-02_21-53-24/\", \"180k_ckpt.pt\"), \n", " # checkpoint_filepath=os.path.join(\"/home/tony/Data/checkpoints/hoot/2024-01-10_05-56-59/60k_ckpt.pt\"),\n", " checkpoint_filepath=os.path.join(\"/home/tony/Data/checkpoints/hoot/2024-01-12_22-51-29/170k_ckpt.pt\"),\n", " # checkpoint_filepath=os.path.join(\"/home/tony/Data/checkpoints/hoot/2023-11-22_20-58-21\", \"30k_ckpt.pt\"), \n", " # tokenizer_filepath=\"/home/tony/Work/tony/hoot/tokenizers/en3/tokenizer_spe_bpe_v4096/tokenizer.model\",\n", " # tokenizer_filepath=\"/home/tony/Work/tony/hoot/tokenizers/v3/tokenizer_spe_bpe_v10240/tokenizer.model\"\n", " tokenizer_filepath=\"/home/tony/Work/tony/hoot/tokenizers/full/tokenizer_spe_bpe_v20480/tokenizer.model\"\n", ") \n", "\n", "model = model_list[0][\"model\"]\n", "tokenizer = model_list[0][\"tokenizer\"]" ] }, { "cell_type": "markdown", "id": "c2c51ee5", "metadata": {}, "source": [ "337179:{\"audio_filepath\": \"/app/suno/data/hoot/ivnWzCbx3hw.wav\", \"duration\": 144, \"text\": \"hey hey shout shout shout for happiness there's no joy unless you shout for happiness all day through oh sing sing let your voices ring bid your troubles adieu take the joy that is meant for you don't sit in a corner like jacky horner get where the bright lights are just look up high search the sky find your luck star shout shout shout for happiness there's no joy if your blue shout for happiness all day through well well well who do i see right in front of me but max goldie in the flesh hey gosh but you're looking well and full of spirits too no i didn't say whiskey come on max tell me what makes you so full of happiness been to see the cup final oh i can see right in your eye you're just dying for a drink ah well come on lets go\"}\n", "\n", "29934:{\"audio_filepath\": \"/app/suno/data/hoot/6e1XdHd_RjE.wav\", \"duration\": 187, \"text\": \"verse rapture rapture falling apart fracture fracture break me apart taste it taste it are we okay wasted wasted on an ego trip what a fucking shame drop dead just like the rest cause i was never heaven sent go tell your family and go tell your friends you'll drop dead just like the rest chorus so please won't you forget me cause i'm leaving i'm leaving get these memories out of my head verse confess for all that you've done suppress and tell me that i'm the one what a waste of such a beautiful face god damn can't you just get over it fuck this you're just like the rest and you were never heaven sent you made me like this drop dead just like the rest just like the rest confess confess fucking say it go ahead and watch me fall breakdown confess cause now you're the enemy suppress this won't be the end of me mindless the numb rotting the broken severed tongues enough remaining unspoken cause this is the plague and now you're infected your face is such a beautiful waste chorus so please won't you forget me cause i'm leaving i'm leaving get these memories out of my head confess suppress confess suppress to the face of punishment inside and out you've dropped dead just like the rest\"}" ] }, { "cell_type": "code", "execution_count": null, "id": "fc8bd44f", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:57.075180Z", "start_time": "2024-01-16T17:20:56.383087Z" } }, "outputs": [], "source": [ "test_audio = Audio.from_file(\"/app/suno/data/hoot/6e1XdHd_RjE.wav\")\n", "test_audio.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "d77bbda0", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:57.078048Z", "start_time": "2024-01-16T17:20:57.076278Z" } }, "outputs": [], "source": [ "print(test_audio.sample_rate, test_audio.n_channels, test_audio.byte_width, test_audio.duration_s)" ] }, { "cell_type": "code", "execution_count": null, "id": "ba975cdb", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:57.934563Z", "start_time": "2024-01-16T17:20:57.079062Z" } }, "outputs": [], "source": [ "out = encode_filepaths([\"/app/suno/data/hoot/ivnWzCbx3hw.wav\"])\n", "print(out)" ] }, { "cell_type": "markdown", "id": "7fbf15d0", "metadata": {}, "source": [ "[\"sho shout shout forss there's joy unless you shout fors all day through sing s letoing bigubles the ist for you can seeinker\"]\n" ] }, { "cell_type": "code", "execution_count": null, "id": "b4e5c76e", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:58.116425Z", "start_time": "2024-01-16T17:20:57.935990Z" } }, "outputs": [], "source": [ "out = encode_filepaths([\"/app/suno/data/hoot/ivnWzCbx3hw.wav\"], return_logits=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "daa19f74", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:58.120113Z", "start_time": "2024-01-16T17:20:58.117651Z" } }, "outputs": [], "source": [ "out[0].shape" ] }, { "cell_type": "code", "execution_count": null, "id": "fa26cfe5", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:58.434444Z", "start_time": "2024-01-16T17:20:58.121178Z" } }, "outputs": [], "source": [ "t1 = tokenizer.decode_logits(out[0])" ] }, { "cell_type": "code", "execution_count": null, "id": "261efa49", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:58.441888Z", "start_time": "2024-01-16T17:20:58.435984Z" } }, "outputs": [], "source": [ "gt_text = \"\"\"hey hey shout shout shout for happiness there's no joy unless you shout for happiness all day through oh sing sing let your voices ring bid your troubles adieu take the joy that is meant for you don't sit in a corner like jacky horner get where the bright lights are just look up high search the sky find your luck star shout shout shout for happiness there's no joy if your blue shout for happiness all day through well well well who do i see right in front of me but max goldie in the flesh hey gosh but you're looking well and full of spirits too no i didn't say whiskey come on max tell me what makes you so full of happiness been to see the cup final oh i can see right in your eye you're just dying for a drink ah well come on lets go\"\"\"\n", "t2 = tokenizer.decode_logits(out[0], prior_text=gt_text)" ] }, { "cell_type": "code", "execution_count": null, "id": "c2ec6385", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:58.525016Z", "start_time": "2024-01-16T17:20:58.443405Z" } }, "outputs": [], "source": [ "get_cer(gt_text, t1), get_cer(gt_text, t2)" ] }, { "cell_type": "code", "execution_count": null, "id": "d0ceaa81", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:58.587047Z", "start_time": "2024-01-16T17:20:58.526000Z" } }, "outputs": [], "source": [ "# t.decode_logits(out[0][:, :-1])" ] }, { "cell_type": "code", "execution_count": null, "id": "a4d43c81", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:20:59.676606Z", "start_time": "2024-01-16T17:20:58.588461Z" } }, "outputs": [], "source": [ "out = encode_filepaths([\"/app/suno/data/hoot/6e1XdHd_RjE.wav\"])\n", "print(out)" ] }, { "cell_type": "markdown", "id": "d1ce8374", "metadata": { "ExecuteTime": { "end_time": "2023-11-05T21:45:28.663742Z", "start_time": "2023-11-05T21:45:28.661345Z" } }, "source": [ "[\"rapture afterter falling apart frature fracture break me apart taste and taste are we okay wasted wasted on ego trap whatal fucking shame dr just like the rest i was never so go tell your family and go tell your friends you'll drop in just like the rest so please won't youget me'cause i'm leaving i'm leaving these memories out of head confess for all that you've done suress and tell me that i'm the one on a waste of such a beautiful face goddmn can you just get over it fuck this you're just like the rest and you are never this you made me like this dr in just like the rest just like the confess confess fucking s head and me through conf cause now this will be the end of me my paomb running the b a timeni remaining unspoken this is the b and now yourfected your face is such a beutiful word so please wont you forget me im bleeding bleeding\"]\n", "\n", "verse rapture rapture falling apart fracture fracture break me apart taste it taste it are we okay wasted wasted on an ego trip what a fucking shame drop dead just like the rest cause i was never heaven sent go tell your family and go tell your friends you'll drop dead just like the rest chorus so please won't you forget me cause i'm leaving i'm leaving get these memories out of my head verse confess for all that you've done suppress and tell me that i'm the one what a waste of such a beautiful face god damn can't you just get over it fuck this you're just like the rest and you were never heaven sent you made me like this drop dead just like the rest just like the rest confess confess fucking say it go ahead and watch me fall breakdown confess cause now you're the enemy suppress this won't be the end of me mindless the numb rotting the broken severed tongues enough remaining unspoken cause this is the plague and now you're infected your face is such a beautiful waste chorus so please won't you forget me cause i'm leaving i'm leaving get these memories out of my head confess suppress confess suppress to the face of punishment inside and out you've dropped dead just like the rest\n" ] }, { "cell_type": "code", "execution_count": null, "id": "f8e21ec2", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:00.072411Z", "start_time": "2024-01-16T17:20:59.678033Z" } }, "outputs": [], "source": [ "out = encode_filepaths([\"/home/tony/Work/tony/audios/kitchen_sink_temperature_coarse_0.95_1.wav\"])\n", "print(out)" ] }, { "cell_type": "code", "execution_count": null, "id": "dfa954fb", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:00.076618Z", "start_time": "2024-01-16T17:21:00.074744Z" } }, "outputs": [], "source": [ "# %%time\n", "# ~2k realtime on a5000\n", "# _ = encode_filepaths([\"/home/tony/Data/Hoot/exp/JvqqNi82glQ.wav\"]*100)" ] }, { "cell_type": "code", "execution_count": null, "id": "411bac5c", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:00.872697Z", "start_time": "2024-01-16T17:21:00.077930Z" } }, "outputs": [], "source": [ "out = encode_filepaths([\"/home/tony/Work/tony/audios/agony_ai_song1.mp3\"])\n", "print(out)" ] }, { "cell_type": "code", "execution_count": null, "id": "0ed770d2", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:00.964638Z", "start_time": "2024-01-16T17:21:00.874970Z" } }, "outputs": [], "source": [ "test_audio = Audio.from_file(\"/home/tony/Work/tony/audios/good.mp3\")" ] }, { "cell_type": "code", "execution_count": null, "id": "3c959a51", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:01.011334Z", "start_time": "2024-01-16T17:21:00.966195Z" } }, "outputs": [], "source": [ "test_audio.play(compress=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "381329af", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:01.483807Z", "start_time": "2024-01-16T17:21:01.012752Z" } }, "outputs": [], "source": [ "out = encode_filepaths([\"/home/tony/Work/tony/audios/good.mp3\"])\n", "print(out)" ] }, { "cell_type": "code", "execution_count": null, "id": "c1e00c05", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:02.455758Z", "start_time": "2024-01-16T17:21:01.486502Z" } }, "outputs": [], "source": [ "out = encode_filepaths([\"/home/tony/Work/tony/audios/319cfea5-30ea-43b7-9654-6c82a5245046_0.mp3\"])\n", "print(out)" ] }, { "cell_type": "markdown", "id": "01e572bb", "metadata": { "ExecuteTime": { "end_time": "2024-01-11T16:56:38.211202Z", "start_time": "2024-01-11T16:56:38.208626Z" } }, "source": [ "\" I've seen things that people wouldn't believe I watched the things glitter in the dark Without ten lies against That was my fate And all those moments Will be lost in time Like tears erasing Like tears erasing Like tears erasing And all those moments Will be lost in time Like tears erasing\"" ] }, { "cell_type": "code", "execution_count": null, "id": "1f812d79", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:02.460479Z", "start_time": "2024-01-16T17:21:02.458487Z" } }, "outputs": [], "source": [ "# Modal run config\n", "# modal run test_modal.py \\\n", "# --embed-type 'hoot' \\\n", "# --data-type 'youtube_music' \\\n", "# --output-name 'hoot_logits'" ] }, { "cell_type": "markdown", "id": "11dbcf5a", "metadata": {}, "source": [ "# Train" ] }, { "cell_type": "code", "execution_count": null, "id": "87fa9bc6", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:02.831779Z", "start_time": "2024-01-16T17:21:02.461744Z" } }, "outputs": [], "source": [ "out = encode_filepaths([\"/home/tony/Work/tony/audios/319cfea5-30ea-43b7-9654-6c82a5245046_0.mp3\"], return_logits=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "d11e409c", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:02.835251Z", "start_time": "2024-01-16T17:21:02.833509Z" } }, "outputs": [], "source": [ "gt_text = \" I've seen things that people wouldn't believe I watched the things glitter in the dark Without ten lies against That was my fate And all those moments Will be lost in time Like tears erasing Like tears erasing Like tears erasing And all those moments Will be lost in time Like tears erasing\"" ] }, { "cell_type": "code", "execution_count": null, "id": "b666fb63", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:03.096822Z", "start_time": "2024-01-16T17:21:02.836285Z" } }, "outputs": [], "source": [ "tokenizer.decode_logits(out[0])" ] }, { "cell_type": "code", "execution_count": null, "id": "813dd82b", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:03.101582Z", "start_time": "2024-01-16T17:21:03.097835Z" } }, "outputs": [], "source": [ "tokenizer.decode_logits(out[0], prior_text=gt_text)" ] }, { "cell_type": "code", "execution_count": null, "id": "41816270", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:03.303839Z", "start_time": "2024-01-16T17:21:03.102562Z" } }, "outputs": [], "source": [ "out = encode_filepaths([\"/home/tony/Work/tony/audios/good.mp3\"], return_logits=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "94281369", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:03.361089Z", "start_time": "2024-01-16T17:21:03.305353Z" } }, "outputs": [], "source": [ "tokenizer.decode_logits(out[0])" ] }, { "cell_type": "code", "execution_count": null, "id": "a2c18c9a", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:03.372477Z", "start_time": "2024-01-16T17:21:03.362122Z" } }, "outputs": [], "source": [ "gt_text = \"一壶漂泊浪迹天涯难入喉 你走之后酒暖回忆思念瘦 水向东流时间怎么偷 花开就一次成熟我却错过 谁在用琵琶弹奏一曲东风破\"\"\"" ] }, { "cell_type": "code", "execution_count": null, "id": "b37bcc70", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:03.440118Z", "start_time": "2024-01-16T17:21:03.374189Z" } }, "outputs": [], "source": [ "tokenizer.decode(tokenizer.encode(gt_text).tolist())" ] }, { "cell_type": "code", "execution_count": null, "id": "3a7eb1c9", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:03.503551Z", "start_time": "2024-01-16T17:21:03.443416Z" } }, "outputs": [], "source": [ "\"\".join(tokenizer.decode(x) for x in tokenizer.encode(gt_text).tolist())" ] }, { "cell_type": "code", "execution_count": null, "id": "1bf0604e", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:03.567124Z", "start_time": "2024-01-16T17:21:03.504446Z" } }, "outputs": [], "source": [ "tokenizer.decode_logits(out[0], prior_text=gt_text)" ] }, { "cell_type": "code", "execution_count": null, "id": "9edd67f6", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:03.653423Z", "start_time": "2024-01-16T17:21:03.568133Z" } }, "outputs": [], "source": [ "gt_text = \"\"\"\n", "[Verse 1]\n", "一盏离愁孤灯伫立在窗口\n", "我在门后假装你人还没走\n", "旧地如重游月圆更寂寞\n", "夜半清醒的烛火不忍苛责我\n", "\n", "[Verse 2]\n", "一壶漂泊浪迹天涯难入喉\n", "你走之后酒暖回忆思念瘦\n", "水向东流时间怎么偷\n", "花开就一次成熟我却错过\n", "\n", "[Chorus]\n", "谁在用琵琶弹奏一曲东风破\n", "岁月在墙上剥落看见小时候\n", "犹记得那年我们都还很年幼\n", "而如今琴声幽幽我的等候你没听过\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": null, "id": "b6409dd3", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:03.726905Z", "start_time": "2024-01-16T17:21:03.655061Z" } }, "outputs": [], "source": [ "import numpy as np\n", "import torch\n", "import math\n", "torch.mean(torch.exp(torch.from_numpy(out[0])), axis=0), torch.std(torch.exp(torch.from_numpy(out[0])), axis=0)" ] }, { "cell_type": "code", "execution_count": null, "id": "d839bf85", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:03.913297Z", "start_time": "2024-01-16T17:21:03.727940Z" } }, "outputs": [], "source": [ "test_audio = Audio.from_file(\"/home/tony/Work/tony/audios/good.mp3\")" ] }, { "cell_type": "code", "execution_count": null, "id": "6db4a950", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:04.053249Z", "start_time": "2024-01-16T17:21:03.914736Z" } }, "outputs": [], "source": [ "out = encode_filepaths([\"/home/tony/Work/tony/audios/good.mp3\"], return_logits=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "1daf2b68", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:04.144538Z", "start_time": "2024-01-16T17:21:04.054350Z" } }, "outputs": [], "source": [ "for s in ctc_align(out[0], gt_text):\n", " print(s)\n", " if s['end_s'] - s['start_s'] > 1:\n", " test_audio.get_segment(from_s=math.floor(s['start_s']), to_s=math.ceil(s['end_s'])).play(compress=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "3e6844da", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:05.159359Z", "start_time": "2024-01-16T17:21:04.145676Z" } }, "outputs": [], "source": [ "test_audio = Audio.from_file(\"/app/suno/data/hoot/PdjbRvvJAzg.wav\")\n", "test_audio.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "039dd2ec", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:06.577963Z", "start_time": "2024-01-16T17:21:05.160598Z" } }, "outputs": [], "source": [ "out = encode_filepaths([\"/app/suno/data/hoot/PdjbRvvJAzg.wav\"])\n", "print(out)" ] }, { "cell_type": "code", "execution_count": null, "id": "c96a1ff8", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:07.068376Z", "start_time": "2024-01-16T17:21:06.579405Z" } }, "outputs": [], "source": [ "test_audio = Audio.from_file(\"../audios/test_korean.mp3\")\n", "test_audio.play(compress=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "db8ae0f4", "metadata": { "ExecuteTime": { "end_time": "2024-01-16T17:21:07.683917Z", "start_time": "2024-01-16T17:21:07.069690Z" } }, "outputs": [], "source": [ "out = encode_filepaths([\"../audios/test_korean.mp3\"], prior_texts=[\"This is the song that never ends\"])\n", "print(out)" ] }, { "cell_type": "code", "execution_count": null, "id": "b3e0e761", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fc1e1229", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "298319dd", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.13" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }