{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "7503fbeb", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:13.108291Z", "start_time": "2024-04-24T02:00:13.106831Z" }, "execution": { "iopub.execute_input": "2024-10-17T19:58:11.186292Z", "iopub.status.busy": "2024-10-17T19:58:11.185680Z", "iopub.status.idle": "2024-10-17T19:58:11.190593Z", "shell.execute_reply": "2024-10-17T19:58:11.190160Z", "shell.execute_reply.started": "2024-10-17T19:58:11.186273Z" } }, "outputs": [], "source": [ "import os\n", "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "c6fcdd37", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:14.496223Z", "start_time": "2024-04-24T02:00:13.109247Z" }, "execution": { "iopub.execute_input": "2024-10-17T19:58:12.111028Z", "iopub.status.busy": "2024-10-17T19:58:12.110882Z", "iopub.status.idle": "2024-10-17T19:58:14.121405Z", "shell.execute_reply": "2024-10-17T19:58:14.120759Z", "shell.execute_reply.started": "2024-10-17T19:58:12.111014Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2.4.1+cu121\n", "2.4.0+cu121\n", "cuda\n" ] } ], "source": [ "import torch\n", "import torchaudio\n", "import re\n", "import json\n", "import tqdm\n", "import numpy as np\n", "from Bio import Align\n", "from suno_utils.audio import Audio\n", "from suno_utils.tasks.hoot import (\n", " encode_filepaths,\n", " encode,\n", " clean_text,\n", " encode_and_align,\n", " _get_alignable_tokens,\n", " _assign_word_timings,\n", " get_aligned_lyrics,\n", " ctc_align,\n", " load_model_list,\n", " get_word_timing_from_audio_and_lyrics,\n", " _merge_into_lines,\n", ")\n", "from suno_utils.utils.s3 import read_from_s3\n", "import ctc_segmentation\n", "from suno_utils.utils.metrics import get_cer\n", "\n", "\n", "print(torch.__version__)\n", "print(torchaudio.__version__)\n", "\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "print(device)\n", "\n", "%reload_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 3, "id": "0ccd88e3", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:14.560456Z", "start_time": "2024-04-24T02:00:14.514064Z" }, "execution": { "iopub.execute_input": "2024-10-17T19:58:17.010533Z", "iopub.status.busy": "2024-10-17T19:58:17.010083Z", "iopub.status.idle": "2024-10-17T19:58:17.022448Z", "shell.execute_reply": "2024-10-17T19:58:17.021934Z", "shell.execute_reply.started": "2024-10-17T19:58:17.010516Z" } }, "outputs": [], "source": [ "def _reload_models_if_needed():\n", " model_list = load_model_list(\n", " # checkpoint_filepath=os.path.join(\"/home/tony/Data/checkpoints/hoot/2023-11-17_15-08-57\", \"28k_ckpt.pt\"),\n", " # checkpoint_filepath=os.path.join(\"/app/suno/models\", \"hoot_v3.pt\"),\n", " # checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-01-15_03-43-37/25k_ckpt.pt\",\n", " # tokenizer_filepath=\"/app/suno/models/hoot_v3_tokenizer.model\",\n", " checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-01-18_05-50-04/200k_ckpt.pt\",\n", " tokenizer_filepath=\"/app/suno/models/hoot_v3_tokenizer.model\",\n", " n_gpus=None,\n", " )\n", " model = model_list[0][\"model\"]\n", " tokenizer = model_list[0][\"tokenizer\"]\n", " return model_list, model, tokenizer" ] }, { "cell_type": "markdown", "id": "6cff3425", "metadata": { "ExecuteTime": { "end_time": "2023-12-18T17:47:51.681132Z", "start_time": "2023-12-18T17:47:51.662114Z" } }, "source": [ "# Test alignment" ] }, { "cell_type": "code", "execution_count": 4, "id": "d4ff6ed6", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:17.115000Z", "start_time": "2024-04-24T02:00:14.561957Z" }, "execution": { "iopub.execute_input": "2024-10-17T19:58:20.694141Z", "iopub.status.busy": "2024-10-17T19:58:20.693646Z", "iopub.status.idle": "2024-10-17T19:58:21.112545Z", "shell.execute_reply": "2024-10-17T19:58:21.111794Z", "shell.execute_reply.started": "2024-10-17T19:58:20.694125Z" } }, "outputs": [], "source": [ "model_list, model, tokenizer = _reload_models_if_needed()" ] }, { "cell_type": "code", "execution_count": 5, "id": "e884d407", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:18.124130Z", "start_time": "2024-04-24T02:00:17.116306Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "s3://suno-data-uploads/studio/uploads/1cee79db-a0ee-44e4-9646-d312cd620997.mp3\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "input_audio = Audio.from_file(\"../audios/walking_down.mp3\", n_channels=2)\n", "test_input_audio = input_audio # .get_segment(from_s=180, to_s=200.1)\n", "# test_input_audio.play(compress=False)\n", "\n", "gen_id = \"1cee79db-a0ee-44e4-9646-d312cd620997\" # chinese\n", "# gen_id = \"23f1997b-3ae1-4a0a-b93f-099a8ce7b213\" # sara\n", "\n", "mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.mp3\"\n", "print(mp3_filepath)\n", "test_input_audio = Audio.from_s3(mp3_filepath, n_channels=2)\n", "test_input_audio.play()" ] }, { "cell_type": "code", "execution_count": 6, "id": "47304ff7", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:18.138195Z", "start_time": "2024-04-24T02:00:18.125329Z" } }, "outputs": [], "source": [ "prior_texts = \"\"\"\n", "[Intro]\n", "Ah ah ha ha ha\n", "Ah ah ha ha ha\n", "Ah ah ha ha ha\n", "Ah ah ha ha ha\n", "Ah ah ha ha ha\n", "\n", "[Verse]\n", "Been here all along\n", "So, why can't you see?\n", "You belong with me\n", "Standing by and waiting at your backdoor\n", "All this time how could you not know, baby?\n", "You belong with me\n", "You belong with me \n", "\n", "[Chorus]\n", "You belong with me\n", "You belong with me\n", "You belong with me\n", "You belong with me\n", "\"\"\"\n", "\n", "prior_texts = \"\"\"\n", "[verse 1]\n", "滚滚长江东逝水\n", "浪花淘尽英雄\n", "\n", "[verse 2]\n", "是非成败转头空\n", "青山依旧在\n", "几度夕阳红\n", "\n", "[bridge]\n", "白发渔樵江渚上\n", "惯看秋月春风\n", "\n", "[chorus]\n", "一壶浊酒喜相逢\n", "古今多少事\n", "都付笑谈中\n", "\n", "[Refrain]\n", "一壶浊酒喜相逢\n", "古今多少事\n", "都付笑谈中\n", "\n", "[end]\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": 7, "id": "73aedfb5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "with prior --> 滚上花多是水 浪花头今一雄是非成白转头空 青山依酒在几都夕阳空 白花一今长一上看看秋月长风 一付多酒一上风 古今多少事都非笑看中 一付多酒一上风 古今多少事都非笑看中 一付多酒喜上风古尽多少事都付看中 一花酒酒夕相风古尽多少事都花笑看中\n", "without prior --> 不管嘲降多睡水 拉花逃紧意雄是非成擺轉頭空 清沙依舊在己妒吸阳空 白發一起降了傷管看秋越出 thir瘋 你呼座救西上風 孤ille多少事都不小看中 你呼座救西上風 孤今多少事都不小看重 你呼座脚洗上風孤经多少事都不秒看中 以后座脚心相風孤紧多少事都不小看中\n" ] } ], "source": [ "# get the hooted output -- with and without prior\n", "hoot_without_prior = encode(test_input_audio)\n", "hoot_with_prior = encode(test_input_audio, prior_texts=[prior_texts])\n", "print(\"with prior -->\", hoot_with_prior)\n", "print(\"without prior -->\", hoot_without_prior)" ] }, { "cell_type": "code", "execution_count": 8, "id": "2013de74", "metadata": {}, "outputs": [], "source": [ "model_list, model, tokenizer = _reload_models_if_needed()" ] }, { "cell_type": "code", "execution_count": 9, "id": "97bf81bd", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:19.950252Z", "start_time": "2024-04-24T02:00:19.858997Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "FAILED alignments -- alignment length 0. Fallback to hoot.\n", "Failed to align lyrics at idx 5. skipping.\n", "FAILED alignments -- alignment length 0. Fallback to hoot.\n", "Failed to align lyrics at idx 6. skipping.\n", "FAILED alignments -- alignment length 0. Fallback to hoot.\n", "Failed to align lyrics at idx 10. skipping.\n", "FAILED alignments -- alignment length 0. Fallback to hoot.\n", "Failed to align lyrics at idx 11. skipping.\n", "Merging lyrics:start_idx--> 0, end_idx--> 102\n", "Merging lyrics:start_idx--> 81, end_idx--> 102\n", "\n", "[verse 1]\n", "滚滚长江东逝水\n", "浪花淘尽英雄\n", "\n", "[verse 2]\n", "是非成败转头空\n", "青山依旧在\n", "几度夕阳红\n", "\n", "[bridge]\n", "白发渔樵江渚上\n", "惯看秋月春风\n", "\n", "[chorus]\n", "一壶浊酒喜相逢\n", "古今多少事\n", "都付笑谈中\n", "\n", "[Refrain]\n", "一壶浊酒喜相逢\n", "古今多少事\n", "都付笑谈中\n", "\n", "[end]\n", "\n", "一壶浊酒喜相逢\n", "古今多少事\n", "都付笑谈中\n", "\n", "[end]\n", "\n", "\n" ] } ], "source": [ "# this is the continue function test\n", "aligned_lyrics = encode_and_align(\n", " test_input_audio,\n", " prior_texts=prior_texts,\n", " do_multi_align=True,\n", ")\n", "print(aligned_lyrics)" ] }, { "cell_type": "code", "execution_count": 10, "id": "a3713615", "metadata": {}, "outputs": [], "source": [ "# chunk_duration = 30\n", "# for start_s in range(0, int(test_input_audio.duration_s), 30):\n", "# print(start_s)\n", "# aligned_lyrics = encode_and_align(\n", "# test_input_audio.get_segment(\n", "# from_s=max(0, start_s - 2), to_s=start_s + chunk_duration + 2\n", "# ),\n", "# prior_texts=prior_texts,\n", "# )\n", "# print(aligned_lyrics)" ] }, { "cell_type": "code", "execution_count": 11, "id": "1cabba3b", "metadata": {}, "outputs": [], "source": [ "model_list, model, tokenizer = _reload_models_if_needed()" ] }, { "cell_type": "code", "execution_count": 12, "id": "45c8ebaf", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.073120Z", "start_time": "2024-04-24T02:00:19.951452Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "FAILED alignments -- alignment length 0. Fallback to hoot.\n", "Failed to align lyrics at idx 5. skipping.\n", "FAILED alignments -- alignment length 0. Fallback to hoot.\n", "Failed to align lyrics at idx 6. skipping.\n", "FAILED alignments -- alignment length 0. Fallback to hoot.\n", "Failed to align lyrics at idx 10. skipping.\n", "FAILED alignments -- alignment length 0. Fallback to hoot.\n", "Failed to align lyrics at idx 11. skipping.\n", "Merging lyrics:start_idx--> 0, end_idx--> 102\n", "Merging lyrics:start_idx--> 81, end_idx--> 102\n", "{'word': '\\n'}\n", "{'word': '[verse 1]\\n滚', 'success': True, 'start_s': 14.76, 'end_s': 15.32, 'p_align': 0.0}\n", "{'word': '滚', 'success': True, 'start_s': 15.41, 'end_s': 16.16, 'p_align': 0.004}\n", "{'word': '长', 'success': True, 'start_s': 16.16, 'end_s': 16.56, 'p_align': 0.002}\n", "{'word': '江', 'success': True, 'start_s': 16.56, 'end_s': 17.0, 'p_align': 0.0}\n", "{'word': '东', 'success': True, 'start_s': 17.0, 'end_s': 17.63, 'p_align': 0.0}\n", "{'word': '逝', 'success': True, 'start_s': 17.63, 'end_s': 18.82, 'p_align': 0.001}\n", "{'word': '水\\n', 'success': True, 'start_s': 18.91, 'end_s': 20.21, 'p_align': 0.747}\n", "{'word': '浪', 'success': True, 'start_s': 20.61, 'end_s': 22.02, 'p_align': 0.036}\n", "{'word': '花', 'success': True, 'start_s': 22.12, 'end_s': 22.9, 'p_align': 0.449}\n", "{'word': '淘', 'success': True, 'start_s': 22.9, 'end_s': 23.3, 'p_align': 0.0}\n", "{'word': '尽', 'success': True, 'start_s': 23.3, 'end_s': 23.66, 'p_align': 0.008}\n", "{'word': '英', 'success': True, 'start_s': 23.66, 'end_s': 24.25, 'p_align': 0.028}\n", "{'word': '雄\\n\\n', 'success': True, 'start_s': 24.25, 'end_s': 25.52, 'p_align': 0.228}\n", "{'word': '[verse 2]\\n是', 'success': True, 'start_s': 27.17, 'end_s': 28.6, 'p_align': 0.0}\n", "{'word': '非', 'success': True, 'start_s': 28.73, 'end_s': 29.48, 'p_align': 0.9}\n", "{'word': '成', 'success': True, 'start_s': 29.48, 'end_s': 29.88, 'p_align': 0.347}\n", "{'word': '败', 'success': True, 'start_s': 29.88, 'end_s': 30.32, 'p_align': 0.003}\n", "{'word': '转', 'success': True, 'start_s': 30.32, 'end_s': 30.96, 'p_align': 0.012}\n", "{'word': '头', 'success': True, 'start_s': 30.96, 'end_s': 32.08, 'p_align': 0.019}\n", "{'word': '空\\n', 'success': True, 'start_s': 32.31, 'end_s': 33.64, 'p_align': 0.469}\n", "{'word': '青', 'success': True, 'start_s': 34.21, 'end_s': 35.26, 'p_align': 0.035}\n", "{'word': '山', 'success': True, 'start_s': 35.31, 'end_s': 36.14, 'p_align': 0.05}\n", "{'word': '依', 'success': True, 'start_s': 36.14, 'end_s': 36.58, 'p_align': 0.768}\n", "{'word': '旧', 'success': True, 'start_s': 36.58, 'end_s': 36.98, 'p_align': 0.008}\n", "{'word': '在\\n', 'success': True, 'start_s': 36.98, 'end_s': 37.62, 'p_align': 0.962}\n", "{'word': '几', 'success': True, 'start_s': 37.62, 'end_s': 38.26, 'p_align': 0.006}\n", "{'word': '度', 'success': True, 'start_s': 38.26, 'end_s': 38.66, 'p_align': 0.004}\n", "{'word': '夕', 'success': True, 'start_s': 38.66, 'end_s': 39.06, 'p_align': 0.055}\n", "{'word': '阳', 'success': True, 'start_s': 39.06, 'end_s': 39.45, 'p_align': 0.161}\n", "{'word': '红\\n\\n', 'success': True, 'start_s': 39.45, 'end_s': 40.41, 'p_align': 0.043}\n", "{'word': '[bridge]\\n白', 'success': True, 'start_s': 40.78, 'end_s': 41.95, 'p_align': 0.143}\n", "{'word': '发', 'success': True, 'start_s': 42.08, 'end_s': 42.81, 'p_align': 0.03}\n", "{'word': '渔', 'success': True, 'start_s': 42.81, 'end_s': 43.41, 'p_align': 0.0}\n", "{'word': '樵', 'success': True, 'start_s': 43.41, 'end_s': 44.24, 'p_align': 0.0}\n", "{'word': '江', 'success': True, 'start_s': 44.24, 'end_s': 45.08, 'p_align': 0.002}\n", "{'word': '渚', 'success': True, 'start_s': 45.08, 'end_s': 45.88, 'p_align': 0.0}\n", "{'word': '上\\n', 'success': True, 'start_s': 45.88, 'end_s': 47.09, 'p_align': 0.236}\n", "{'word': '惯', 'success': True, 'start_s': 47.52, 'end_s': 48.53, 'p_align': 0.002}\n", "{'word': '看', 'success': True, 'start_s': 48.7, 'end_s': 49.43, 'p_align': 0.668}\n", "{'word': '秋', 'success': True, 'start_s': 49.43, 'end_s': 49.83, 'p_align': 0.226}\n", "{'word': '月', 'success': True, 'start_s': 49.83, 'end_s': 51.77, 'p_align': 0.042}\n", "{'word': '春', 'success': True, 'start_s': 53.8, 'end_s': 57.73, 'p_align': 0.0}\n", "{'word': '风\\n\\n', 'success': True, 'start_s': 62.02, 'end_s': 65.99, 'p_align': 0.001}\n", "{'word': '[chorus]\\n一', 'success': True, 'start_s': 66.72, 'end_s': 68.46, 'p_align': 0.086}\n", "{'word': '壶', 'success': True, 'start_s': 68.46, 'end_s': 69.06, 'p_align': 0.0}\n", "{'word': '浊', 'success': True, 'start_s': 69.06, 'end_s': 69.46, 'p_align': 0.0}\n", "{'word': '酒', 'success': True, 'start_s': 69.46, 'end_s': 70.36, 'p_align': 0.083}\n", "{'word': '喜', 'success': True, 'start_s': 70.77, 'end_s': 71.73, 'p_align': 0.002}\n", "{'word': '相', 'success': True, 'start_s': 71.73, 'end_s': 72.57, 'p_align': 0.023}\n", "{'word': '逢\\n', 'success': True, 'start_s': 72.57, 'end_s': 73.44, 'p_align': 0.01}\n", "{'word': '古', 'success': True, 'start_s': 73.44, 'end_s': 74.08, 'p_align': 0.005}\n", "{'word': '今', 'success': True, 'start_s': 74.08, 'end_s': 74.48, 'p_align': 0.069}\n", "{'word': '多', 'success': True, 'start_s': 74.48, 'end_s': 74.88, 'p_align': 0.907}\n", "{'word': '少', 'success': True, 'start_s': 74.88, 'end_s': 75.28, 'p_align': 0.715}\n", "{'word': '事\\n', 'success': True, 'start_s': 75.28, 'end_s': 75.92, 'p_align': 0.529}\n", "{'word': '都', 'success': True, 'start_s': 75.92, 'end_s': 76.56, 'p_align': 0.946}\n", "{'word': '付', 'success': True, 'start_s': 76.56, 'end_s': 76.96, 'p_align': 0.001}\n", "{'word': '笑', 'success': True, 'start_s': 76.96, 'end_s': 77.39, 'p_align': 0.009}\n", "{'word': '谈', 'success': True, 'start_s': 77.39, 'end_s': 77.79, 'p_align': 0.005}\n", "{'word': '中\\n\\n', 'success': True, 'start_s': 77.79, 'end_s': 79.09, 'p_align': 0.325}\n", "{'word': '[Refrain]\\n一', 'success': True, 'start_s': 160.8, 'end_s': 161.73, 'p_align': 0.091}\n", "{'word': '壶', 'success': True, 'start_s': 161.73, 'end_s': 162.37, 'p_align': 0.0}\n", "{'word': '浊', 'success': True, 'start_s': 162.37, 'end_s': 162.81, 'p_align': 0.0}\n", "{'word': '酒', 'success': True, 'start_s': 162.81, 'end_s': 163.69, 'p_align': 0.034}\n", "{'word': '喜', 'success': True, 'start_s': 164.06, 'end_s': 165.08, 'p_align': 0.009}\n", "{'word': '相', 'success': True, 'start_s': 165.08, 'end_s': 165.92, 'p_align': 0.024}\n", "{'word': '逢\\n', 'success': True, 'start_s': 165.92, 'end_s': 166.75, 'p_align': 0.002}\n", "{'word': '古', 'success': True, 'start_s': 166.75, 'end_s': 167.35, 'p_align': 0.006}\n", "{'word': '今', 'success': True, 'start_s': 167.35, 'end_s': 167.79, 'p_align': 0.075}\n", "{'word': '多', 'success': True, 'start_s': 167.79, 'end_s': 168.19, 'p_align': 0.907}\n", "{'word': '少', 'success': True, 'start_s': 168.19, 'end_s': 168.59, 'p_align': 0.868}\n", "{'word': '事\\n', 'success': True, 'start_s': 168.59, 'end_s': 169.23, 'p_align': 0.593}\n", "{'word': '都', 'success': True, 'start_s': 169.23, 'end_s': 169.83, 'p_align': 0.919}\n", "{'word': '付', 'success': True, 'start_s': 169.83, 'end_s': 170.34, 'p_align': 0.004}\n", "{'word': '笑', 'success': True, 'start_s': 170.34, 'end_s': 170.82, 'p_align': 0.0}\n", "{'word': '谈', 'success': True, 'start_s': 170.82, 'end_s': 171.18, 'p_align': 0.002}\n", "{'word': '中\\n\\n', 'success': True, 'start_s': 171.18, 'end_s': 172.9, 'p_align': 0.511}\n", "{'word': '[end]\\n\\n一', 'success': True, 'start_s': 174.09, 'end_s': 175.13, 'p_align': 0.088}\n", "{'word': '壶', 'success': True, 'start_s': 175.13, 'end_s': 175.73, 'p_align': 0.0}\n", "{'word': '浊', 'success': True, 'start_s': 175.73, 'end_s': 176.17, 'p_align': 0.0}\n", "{'word': '酒', 'success': True, 'start_s': 176.17, 'end_s': 177.02, 'p_align': 0.004}\n", "{'word': '喜', 'success': True, 'start_s': 177.58, 'end_s': 178.49, 'p_align': 0.0}\n", "{'word': '相', 'success': True, 'start_s': 178.49, 'end_s': 179.28, 'p_align': 0.159}\n", "{'word': '逢\\n', 'success': True, 'start_s': 179.28, 'end_s': 180.12, 'p_align': 0.001}\n", "{'word': '古', 'success': True, 'start_s': 180.12, 'end_s': 180.76, 'p_align': 0.016}\n", "{'word': '今', 'success': True, 'start_s': 180.76, 'end_s': 181.2, 'p_align': 0.024}\n", "{'word': '多', 'success': True, 'start_s': 181.2, 'end_s': 181.6, 'p_align': 0.83}\n", "{'word': '少', 'success': True, 'start_s': 181.6, 'end_s': 181.95, 'p_align': 0.816}\n", "{'word': '事\\n', 'success': True, 'start_s': 181.95, 'end_s': 182.59, 'p_align': 0.399}\n", "{'word': '都', 'success': True, 'start_s': 182.59, 'end_s': 183.23, 'p_align': 0.701}\n", "{'word': '付', 'success': True, 'start_s': 183.23, 'end_s': 183.63, 'p_align': 0.001}\n", "{'word': '笑', 'success': True, 'start_s': 183.63, 'end_s': 184.07, 'p_align': 0.007}\n", "{'word': '谈', 'success': True, 'start_s': 184.07, 'end_s': 184.51, 'p_align': 0.003}\n", "{'word': '中\\n\\n', 'success': True, 'start_s': 184.51, 'end_s': 184.71, 'p_align': 0.593}\n", "{'word': '[end]\\n\\n'}\n", "{'hoot_lyrics': '滚上花多是水 浪花头今一雄是非成白转头空 青山依酒在几都夕阳空 白花一今长一上看看秋月长风 一付多酒一上风 古今多少事都非笑看中 一付多酒一上风 古今多少事都非笑看中 一付多酒喜上风古尽多少事都付看中 一花酒酒夕相风古尽多少事都花笑看中'}\n", "{'hoot_cer': 0.5688073394495413}\n", "done with basics\n" ] } ], "source": [ "padded_words_timed = get_word_timing_from_audio_and_lyrics(\n", " # test_input_audio, prior_texts\n", " test_input_audio, prior_texts, do_multi_align=True\n", ")\n", "for word_d in padded_words_timed:\n", " print(word_d)\n", "print(\"done with basics\")" ] }, { "cell_type": "code", "execution_count": 13, "id": "5cdf03f0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'word': '\\n'}\n", "{'word': '[verse 1]\\n滚', 'success': True, 'start_s': 14.76, 'end_s': 15.28, 'p_align': 0.0}\n", "{'word': '滚', 'success': True, 'start_s': 15.43, 'end_s': 16.16, 'p_align': 0.004}\n", "{'word': '长', 'success': True, 'start_s': 16.16, 'end_s': 16.56, 'p_align': 0.002}\n", "{'word': '江', 'success': True, 'start_s': 16.56, 'end_s': 17.0, 'p_align': 0.0}\n", "{'word': '东', 'success': True, 'start_s': 17.0, 'end_s': 17.63, 'p_align': 0.0}\n", "{'word': '逝', 'success': True, 'start_s': 17.63, 'end_s': 18.54, 'p_align': 0.001}\n", "{'word': '水\\n', 'success': True, 'start_s': 18.84, 'end_s': 20.24, 'p_align': 0.747}\n", "{'word': '浪', 'success': True, 'start_s': 20.69, 'end_s': 22.0, 'p_align': 0.036}\n", "{'word': '花', 'success': True, 'start_s': 22.12, 'end_s': 22.9, 'p_align': 0.449}\n", "{'word': '淘', 'success': True, 'start_s': 22.9, 'end_s': 23.3, 'p_align': 0.0}\n", "{'word': '尽', 'success': True, 'start_s': 23.3, 'end_s': 23.66, 'p_align': 0.008}\n", "{'word': '英', 'success': True, 'start_s': 23.66, 'end_s': 24.25, 'p_align': 0.028}\n", "{'word': '雄\\n\\n', 'success': True, 'start_s': 24.25, 'end_s': 25.93, 'p_align': 0.228}\n", "{'word': '[verse 2]\\n是', 'success': True, 'start_s': 27.28, 'end_s': 28.63, 'p_align': 0.0}\n", "{'word': '非', 'success': True, 'start_s': 28.75, 'end_s': 29.48, 'p_align': 0.9}\n", "{'word': '成', 'success': True, 'start_s': 29.48, 'end_s': 29.88, 'p_align': 0.347}\n", "{'word': '败', 'success': True, 'start_s': 29.88, 'end_s': 30.32, 'p_align': 0.003}\n", "{'word': '转', 'success': True, 'start_s': 30.32, 'end_s': 30.96, 'p_align': 0.012}\n", "{'word': '头', 'success': True, 'start_s': 30.96, 'end_s': 32.0, 'p_align': 0.019}\n", "{'word': '空\\n', 'success': True, 'start_s': 32.23, 'end_s': 33.79, 'p_align': 0.469}\n", "{'word': '青', 'success': True, 'start_s': 33.92, 'end_s': 35.21, 'p_align': 0.035}\n", "{'word': '山', 'success': True, 'start_s': 35.34, 'end_s': 36.14, 'p_align': 0.05}\n", "{'word': '依', 'success': True, 'start_s': 36.14, 'end_s': 36.58, 'p_align': 0.768}\n", "{'word': '旧', 'success': True, 'start_s': 36.58, 'end_s': 36.98, 'p_align': 0.008}\n", "{'word': '在\\n', 'success': True, 'start_s': 36.98, 'end_s': 37.62, 'p_align': 0.962}\n", "{'word': '几', 'success': True, 'start_s': 37.62, 'end_s': 38.26, 'p_align': 0.006}\n", "{'word': '度', 'success': True, 'start_s': 38.26, 'end_s': 38.66, 'p_align': 0.004}\n", "{'word': '夕', 'success': True, 'start_s': 38.66, 'end_s': 39.06, 'p_align': 0.055}\n", "{'word': '阳', 'success': True, 'start_s': 39.06, 'end_s': 39.45, 'p_align': 0.161}\n", "{'word': '红\\n\\n', 'success': True, 'start_s': 39.45, 'end_s': 40.16, 'p_align': 0.043}\n", "{'word': '[bridge]\\n白', 'success': True, 'start_s': 40.61, 'end_s': 41.98, 'p_align': 0.143}\n", "{'word': '发', 'success': True, 'start_s': 42.07, 'end_s': 42.81, 'p_align': 0.03}\n", "{'word': '渔', 'success': True, 'start_s': 42.81, 'end_s': 43.41, 'p_align': 0.0}\n", "{'word': '樵', 'success': True, 'start_s': 43.41, 'end_s': 44.24, 'p_align': 0.0}\n", "{'word': '江', 'success': True, 'start_s': 44.24, 'end_s': 45.08, 'p_align': 0.002}\n", "{'word': '渚', 'success': True, 'start_s': 45.08, 'end_s': 45.88, 'p_align': 0.0}\n", "{'word': '上\\n', 'success': True, 'start_s': 45.88, 'end_s': 46.84, 'p_align': 0.236}\n", "{'word': '惯', 'success': True, 'start_s': 47.25, 'end_s': 48.59, 'p_align': 0.002}\n", "{'word': '看', 'success': True, 'start_s': 48.66, 'end_s': 49.43, 'p_align': 0.668}\n", "{'word': '秋', 'success': True, 'start_s': 49.43, 'end_s': 49.83, 'p_align': 0.226}\n", "{'word': '月', 'success': True, 'start_s': 49.83, 'end_s': 52.13, 'p_align': 0.042}\n", "{'word': '春', 'success': True, 'start_s': 53.92, 'end_s': 60.11, 'p_align': 0.0}\n", "{'word': '风\\n\\n', 'success': True, 'start_s': 62.01, 'end_s': 65.27, 'p_align': 0.001}\n", "{'word': '[chorus]\\n一', 'success': True, 'start_s': 67.03, 'end_s': 68.46, 'p_align': 0.086}\n", "{'word': '壶', 'success': True, 'start_s': 68.46, 'end_s': 69.06, 'p_align': 0.0}\n", "{'word': '浊', 'success': True, 'start_s': 69.06, 'end_s': 69.46, 'p_align': 0.0}\n", "{'word': '酒', 'success': True, 'start_s': 69.46, 'end_s': 70.27, 'p_align': 0.083}\n", "{'word': '喜', 'success': True, 'start_s': 70.52, 'end_s': 71.73, 'p_align': 0.002}\n", "{'word': '相', 'success': True, 'start_s': 71.73, 'end_s': 72.57, 'p_align': 0.023}\n", "{'word': '逢\\n', 'success': True, 'start_s': 72.57, 'end_s': 73.44, 'p_align': 0.01}\n", "{'word': '古', 'success': True, 'start_s': 73.44, 'end_s': 74.08, 'p_align': 0.005}\n", "{'word': '今', 'success': True, 'start_s': 74.08, 'end_s': 74.48, 'p_align': 0.069}\n", "{'word': '多', 'success': True, 'start_s': 74.48, 'end_s': 74.88, 'p_align': 0.907}\n", "{'word': '少', 'success': True, 'start_s': 74.88, 'end_s': 75.28, 'p_align': 0.715}\n", "{'word': '事\\n', 'success': True, 'start_s': 75.28, 'end_s': 75.92, 'p_align': 0.529}\n", "{'word': '都', 'success': True, 'start_s': 75.92, 'end_s': 76.56, 'p_align': 0.946}\n", "{'word': '付', 'success': True, 'start_s': 76.56, 'end_s': 76.96, 'p_align': 0.001}\n", "{'word': '笑', 'success': True, 'start_s': 76.96, 'end_s': 77.39, 'p_align': 0.009}\n", "{'word': '谈', 'success': True, 'start_s': 77.39, 'end_s': 77.79, 'p_align': 0.005}\n", "{'word': '中\\n\\n', 'success': True, 'start_s': 77.79, 'end_s': 84.12, 'p_align': 0.325}\n", "{'word': '[Refrain]\\n一', 'success': True, 'start_s': 160.65, 'end_s': 161.73, 'p_align': 0.091}\n", "{'word': '壶', 'success': True, 'start_s': 161.73, 'end_s': 162.37, 'p_align': 0.0}\n", "{'word': '浊', 'success': True, 'start_s': 162.37, 'end_s': 162.81, 'p_align': 0.0}\n", "{'word': '酒', 'success': True, 'start_s': 162.81, 'end_s': 163.72, 'p_align': 0.034}\n", "{'word': '喜', 'success': True, 'start_s': 164.01, 'end_s': 165.08, 'p_align': 0.009}\n", "{'word': '相', 'success': True, 'start_s': 165.08, 'end_s': 165.92, 'p_align': 0.024}\n", "{'word': '逢\\n', 'success': True, 'start_s': 165.92, 'end_s': 172.04, 'p_align': 0.002}\n", "{'word': '古', 'success': True, 'start_s': 178.88, 'end_s': 180.76, 'p_align': 0.016}\n", "{'word': '今', 'success': True, 'start_s': 180.76, 'end_s': 181.2, 'p_align': 0.024}\n", "{'word': '多', 'success': True, 'start_s': 181.2, 'end_s': 181.6, 'p_align': 0.83}\n", "{'word': '少', 'success': True, 'start_s': 181.6, 'end_s': 181.95, 'p_align': 0.816}\n", "{'word': '事\\n', 'success': True, 'start_s': 181.95, 'end_s': 182.59, 'p_align': 0.399}\n", "{'word': '都', 'success': True, 'start_s': 182.59, 'end_s': 183.23, 'p_align': 0.701}\n", "{'word': '付', 'success': True, 'start_s': 183.23, 'end_s': 183.63, 'p_align': 0.001}\n", "{'word': '笑', 'success': True, 'start_s': 183.63, 'end_s': 184.07, 'p_align': 0.007}\n", "{'word': '谈', 'success': True, 'start_s': 184.07, 'end_s': 184.51, 'p_align': 0.003}\n", "{'word': '中\\n\\n', 'success': True, 'start_s': 184.51, 'end_s': 184.71, 'p_align': 0.593}\n", "{'word': '[end]\\n'}\n", "{'hoot_lyrics': '滚上花多是水 浪花头今一雄是非成白转头空 青山依酒在几都夕阳空 白花一今长一上看看秋月长风 一付多酒一上风 古今多少事都非笑看中 一付多酒一上风 古今多少事都非笑看中 一付多酒喜上风古尽多少事都付看中 一花酒酒夕相风古尽多少事都花笑看中'}\n", "{'hoot_cer': 0.7865168539325843}\n", "done with basics 2\n" ] } ], "source": [ "padded_words_timed = get_word_timing_from_audio_and_lyrics(\n", " test_input_audio, prior_texts\n", ")\n", "for word_d in padded_words_timed:\n", " print(word_d)\n", "print(\"done with basics 2\")" ] }, { "cell_type": "code", "execution_count": 14, "id": "95307a88", "metadata": {}, "outputs": [ { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[14], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mBREAK\u001b[49m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "BREAK" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "c2a4ea3c", "metadata": {}, "source": [ "## Try ctc" ] }, { "cell_type": "code", "execution_count": 13, "id": "807bea43", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.223968Z", "start_time": "2024-04-24T02:00:20.149679Z" } }, "outputs": [], "source": [ "frame_duration_s = 30 / 376" ] }, { "cell_type": "code", "execution_count": 14, "id": "670b31fd", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.356125Z", "start_time": "2024-04-24T02:00:20.225503Z" } }, "outputs": [], "source": [ "logits = encode(test_input_audio, return_logits=True)\n", "if not isinstance(logits, torch.Tensor):\n", " logits = torch.from_numpy(logits)\n", "if round(logits[0].sum().item(), 1) == 1.0:\n", " probs = logits\n", "elif round(torch.exp(logits[0]).sum().item(), 1) == 1.0:\n", " probs = torch.exp(logits)\n", "else:\n", " probs = torch.nn.functional.softmax(logits, dim=-1)\n", "probs = probs.numpy()" ] }, { "cell_type": "code", "execution_count": 15, "id": "e7475b36", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.367487Z", "start_time": "2024-04-24T02:00:20.357353Z" } }, "outputs": [], "source": [ "true_text = prior_texts" ] }, { "cell_type": "code", "execution_count": 16, "id": "bad4dfa0", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.477564Z", "start_time": "2024-04-24T02:00:20.368729Z" } }, "outputs": [], "source": [ "model_list = load_model_list()\n", "tokenizer = model_list[0][\"tokenizer\"]\n", "assert len(tokenizer.vocab) + 1 == probs.shape[1]\n", "inv_vocab = {v: k for k, v in tokenizer.vocab.items()}\n", "unk_id = tokenizer.vocab[\"\"]\n", "# add dummy char cause beginning is messed up\n", "word_transcripts = [\"a\"] + true_text.split()\n", "tokens = []\n", "for word_transcript in word_transcripts:\n", " tok_ids = tokenizer.encode(word_transcript)\n", " tokens.append(tok_ids[tok_ids != unk_id])\n", "char_list = [inv_vocab[i] for i in range(len(inv_vocab))]\n", "config = ctc_segmentation.CtcSegmentationParameters(\n", " char_list=char_list,\n", " blank_transition_cost_zero=True,\n", " replace_spaces_with_blanks=True,\n", " index_duration=frame_duration_s,\n", " backtrack_from_max_t=True,\n", " skip_prob=0.01,\n", ")" ] }, { "cell_type": "code", "execution_count": 17, "id": "4d59cb75", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.545299Z", "start_time": "2024-04-24T02:00:20.478695Z" } }, "outputs": [], "source": [ "ground_truth_mat, utt_begin_indices = ctc_segmentation.prepare_token_list(\n", " config, tokens\n", ")\n", "# print(ground_truth_mat, utt_begin_indices)\n", "timings, char_probs, state_list = ctc_segmentation.ctc_segmentation(\n", " config, probs, ground_truth_mat\n", ")" ] }, { "cell_type": "code", "execution_count": 18, "id": "9b57228a", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.596403Z", "start_time": "2024-04-24T02:00:20.547115Z" } }, "outputs": [], "source": [ "likely_probs = probs.argmax(axis=1).tolist()\n", "likely_probs = [i for i in likely_probs if i != 20480]" ] }, { "cell_type": "code", "execution_count": 19, "id": "7b767cc1", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.644876Z", "start_time": "2024-04-24T02:00:20.597506Z" } }, "outputs": [], "source": [ "gap_char = 0\n", "ground_truth_list = tokenizer.encode(true_text).tolist()" ] }, { "cell_type": "code", "execution_count": 20, "id": "f3796ea5", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.701253Z", "start_time": "2024-04-24T02:00:20.646313Z" } }, "outputs": [], "source": [ "aligner = Align.PairwiseAligner()\n", "aligner.mode = \"global\"\n", "aligner.match_score = 2\n", "aligner.mismatch_score = -1\n", "aligner.open_gap_score = -0.1\n", "aligner.extend_gap_score = -0.5\n", "aligner.target_end_gap_score = 0.0\n", "aligner.query_end_gap_score = 0.0\n", "aligner.alphabet = list(range(0, tokenizer.n_vocab + 1))" ] }, { "cell_type": "code", "execution_count": 21, "id": "cf0ee205", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.763042Z", "start_time": "2024-04-24T02:00:20.702893Z" } }, "outputs": [], "source": [ "# from math import log\n", "# def gap_function(x, y): # x is gap position in seq, y is gap length\n", "# if y == 0: # No gap\n", "# return 0\n", "# elif y == 1: # Gap open penalty\n", "# return -2\n", "# return -(2 + y / 4.0 + log(y) / 2.0)\n", "# alignment = pairwise2.align.globalmc(\n", "# ground_truth_list,\n", "# likely_probs,\n", "# 5, -4, gap_function, gap_function,\n", "# gap_char=[gap_char],\n", "# penalize_end_gaps=False,\n", "# penalize_extend_when_opening=False,\n", "# )[0]" ] }, { "cell_type": "code", "execution_count": null, "id": "a0372e5e", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.832987Z", "start_time": "2024-04-24T02:00:20.764043Z" } }, "outputs": [], "source": [ "alignment = aligner.align(ground_truth_list, likely_probs)[0]\n", "# # trim the two ends\n", "# seq_a, seq_b = alignment.seqA, alignment.seqB\n", "# seq_a = seq_a[::-1]\n", "# seq_b = seq_b[::-1]\n", "# while seq_a and seq_b and seq_b[-1] == gap_char:\n", "# seq_b.pop()\n", "# seq_a.pop()\n", "# seq_a = seq_a[::-1]\n", "# seq_b = seq_b[::-1]\n", "# while seq_a and seq_b and seq_b[-1] == gap_char:\n", "# seq_b.pop()\n", "# seq_a.pop()\n", "seq_a, seq_b = alignment.aligned\n", "seq_a = ground_truth_list[seq_a[0][0] : seq_a[-1][-1] + 1]\n", "real_truth = \" \".join(tokenizer.decode(int(i)) for i in seq_a)\n", "print(real_truth)" ] }, { "cell_type": "code", "execution_count": null, "id": "50b16445", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.890477Z", "start_time": "2024-04-24T02:00:20.834063Z" } }, "outputs": [], "source": [ "print(\n", " \"timings\",\n", " timings.shape,\n", ")\n", "print(\"char_probs\", char_probs.shape, char_probs[-0], char_probs[-1])\n", "print(\"word_transcripts\", len(word_transcripts))\n", "print(\"state_list\", len(state_list))\n", "print(\"utt_begin_indices\", len(utt_begin_indices))\n", "print(\"ground_truth_mat\", ground_truth_mat.shape)\n", "print(\"char prob\", char_probs.shape)\n", "print(tokenizer.n_vocab)\n", "print(tokenizer.decode(20480 - 2))\n", "print(tokenizer.encode(\"ah ah\"))\n", "print(tokenizer.encode(\"ah🛳 ah\"))" ] }, { "cell_type": "code", "execution_count": 24, "id": "0f4cd710", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.954095Z", "start_time": "2024-04-24T02:00:20.891514Z" } }, "outputs": [], "source": [ "# while char_probs.shape[0] > 0 and char_probs[-1] < 0.001:\n", "# char_probs = char_probs[:-1]\n", "# timings = timings[:-1]\n", "# utt_begin_indices = utt_begin_indices[:-1]\n", "# word_transcripts.pop()" ] }, { "cell_type": "code", "execution_count": null, "id": "37600574", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:21.016063Z", "start_time": "2024-04-24T02:00:20.955908Z" } }, "outputs": [], "source": [ "print(\"done with some basic ctc\")" ] }, { "cell_type": "markdown", "id": "53a8f788", "metadata": { "ExecuteTime": { "end_time": "2023-11-13T19:11:14.875248Z", "start_time": "2023-11-13T19:11:14.873671Z" } }, "source": [ "# Start from some test data" ] }, { "cell_type": "code", "execution_count": null, "id": "570dbaf1", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:33.594147Z", "start_time": "2024-04-24T02:00:21.017085Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/metas.json\", \"r\") as fp:\n", "# with open(\"/home/tony/Data/Hoot/ytm_metas.json\", \"r\") as fp:\n", "with open(\"/home/tony/Data/Hoot/deezer_metas.json\", \"r\") as fp:\n", " metas = json.load(fp)\n", "print(len(metas))\n", "metas_map = {m[\"id\"]: m for m in metas}\n", "original_id_metas_map = {m[\"original_id\"]: m for m in metas}" ] }, { "cell_type": "code", "execution_count": null, "id": "04ae3a08", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:33.610536Z", "start_time": "2024-04-24T02:00:33.597934Z" } }, "outputs": [], "source": [ "# train_info = []\n", "# with open(\n", "# \"/home/tony/Data/Hoot/multi_large_vocab_filtered_train_manifest.json\", \"r\"\n", "# ) as fp:\n", "# for l in tqdm.tqdm(fp):\n", "# my_l = json.loads(l)\n", "# if (\n", "# my_l[\"lang\"] == \"zh\"\n", "# and my_l[\"audio_filepath\"] == \"/app/suno/data/hoot/LL50Fu4UvG0.wav\"\n", "# ):\n", "# train_info.append(json.loads(l))" ] }, { "cell_type": "code", "execution_count": null, "id": "9e8350de", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:33.871366Z", "start_time": "2024-04-24T02:00:33.612179Z" } }, "outputs": [], "source": [ "train_info = []\n", "hooted_info = []\n", "for i, meta in enumerate(metas_map.values()):\n", " if meta[\"id\"] == \"sSA7CmtRkdQ\":\n", " # if meta[\"id\"] == \"74b2ba74-e58b-4750-a255-30c6fcdaf1e6\":\n", " # if meta[\"genius_slug\"] == \"Jay-chou-chrysanthemum-terrace-lyrics\":\n", " # if meta[\"genius_slug\"] == \"Alonzo-tout-va-bien-lyrics\":\n", " # if meta[\"genius_slug\"] == \"Lubalin-due-time-lyrics\":\n", " # if meta[\"genius_slug\"] == \"Taylor-swift-anti-hero-lyrics\":\n", " # if meta[\"genius_slug\"].startswith(\"Taylor-swift\"):\n", " # print(meta)\n", " train_info.append(meta)\n", " # print(i)\n", " with open(\n", " f\"/home/tony/Data/Hoot/alignments/outputs_3_deezer/batch_{(i // 100) * 100}.json\",\n", " # f\"/home/tony/Data/Hoot/alignments/outputs_3/batch_{(i // 100) * 100}.json\",\n", " \"r\",\n", " ) as fp:\n", " hooted_outputs = json.load(fp)\n", " for hooted_output in hooted_outputs:\n", " if hooted_output[\"id\"] == meta[\"id\"]:\n", " hooted_info.append(hooted_output)\n", "print(len(train_info), len(hooted_info))\n", "# print(train_info, hooted_info)" ] }, { "cell_type": "code", "execution_count": null, "id": "59429a37", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:33.885518Z", "start_time": "2024-04-24T02:00:33.873245Z" } }, "outputs": [], "source": [ "# # # for debug a single one\n", "# train_info = []\n", "# train_info.append(metas[925186])\n", "# train_info[0]" ] }, { "cell_type": "markdown", "id": "a3d112c7", "metadata": {}, "source": [ "# Redo hoot" ] }, { "cell_type": "code", "execution_count": null, "id": "bbd3e51e", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:33.951359Z", "start_time": "2024-04-24T02:00:33.887026Z" } }, "outputs": [], "source": [ "align_metas = []" ] }, { "cell_type": "code", "execution_count": null, "id": "380e3695", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:34.012892Z", "start_time": "2024-04-24T02:00:33.952919Z" } }, "outputs": [], "source": [ "_ = _reload_models_if_needed()" ] }, { "cell_type": "code", "execution_count": null, "id": "9ba66c36", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:38.719934Z", "start_time": "2024-04-24T02:00:34.014588Z" } }, "outputs": [], "source": [ "# for test_id in [10]:\n", "for test_id in tqdm.tqdm(range(len(train_info))):\n", " meta_id = train_info[test_id][\"id\"]\n", " out = encode_filepaths([metas_map[meta_id][\"audio_filepath\"]], return_logits=True)\n", " basic_cleaned_lyrics = (\n", " metas_map[meta_id][\"lyrics\"]\n", " if \"lyrics\" in metas_map[meta_id]\n", " else metas_map[meta_id][\"text\"]\n", " )\n", " decoded_preds = tokenizer.decode_logits(out[0], prior_text=basic_cleaned_lyrics)\n", " true_text_norm = clean_text(basic_cleaned_lyrics)\n", " cer_val = round(get_cer(true_text_norm, decoded_preds), 3)\n", " word_timings = ctc_align(out[0].astype(np.float16), basic_cleaned_lyrics)\n", " print(cer_val, decoded_preds)\n", " align_metas.append(\n", " {\n", " \"id\": meta_id,\n", " \"duration_s\": metas_map[meta_id][\"duration_s\"],\n", " \"alignment\": word_timings,\n", " \"cer_val\": cer_val,\n", " \"lyrics\": basic_cleaned_lyrics,\n", " }\n", " )" ] }, { "cell_type": "code", "execution_count": null, "id": "051fa022", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:38.737522Z", "start_time": "2024-04-24T02:00:38.721918Z" } }, "outputs": [], "source": [ "metas_map[meta_id]" ] }, { "cell_type": "code", "execution_count": null, "id": "19d8c87d", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:38.798090Z", "start_time": "2024-04-24T02:00:38.738506Z" } }, "outputs": [], "source": [ "aligned_lyrics_map = get_aligned_lyrics(align_metas)" ] }, { "cell_type": "code", "execution_count": null, "id": "577170f0", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:38.852006Z", "start_time": "2024-04-24T02:00:38.799574Z" } }, "outputs": [], "source": [ "# check we get the oraligned_lyrics_mapiginal text out\n", "meta_id = list(aligned_lyrics_map.keys())[0]\n", "original_id, lyrics = (\n", " metas_map[meta_id][\"original_id\"],\n", " basic_cleaned_lyrics,\n", ")\n", "assert \"\".join(w[\"text\"] for w in list(aligned_lyrics_map.values())[0]) == lyrics" ] }, { "cell_type": "code", "execution_count": null, "id": "ef49fc1c", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:38.909311Z", "start_time": "2024-04-24T02:00:38.853566Z" } }, "outputs": [], "source": [ "print(\"\".join(w[\"text\"] for w in list(aligned_lyrics_map.values())[0]))" ] }, { "cell_type": "code", "execution_count": null, "id": "e0d1e588", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:38.965222Z", "start_time": "2024-04-24T02:00:38.911006Z" } }, "outputs": [], "source": [ "print(lyrics)" ] }, { "cell_type": "code", "execution_count": null, "id": "f0ed65a8", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.214308Z", "start_time": "2024-04-24T02:00:38.966246Z" } }, "outputs": [], "source": [ "audio = Audio.from_file(os.path.join(\"/app/suno/data/hoot\", f\"{original_id}.wav\"))\n", "for s in aligned_lyrics_map[meta_id]:\n", " print(\n", " s[\"text\"],\n", " s[\"start_s\"],\n", " s[\"end_s\"],\n", " s[\"end_s\"] - s[\"start_s\"],\n", " s[\"vocal_start_s\"],\n", " )\n", " audio.get_segment(s[\"start_s\"], s[\"end_s\"]).play()\n", " print(\"-\" * 10)" ] }, { "cell_type": "markdown", "id": "9d7224f3", "metadata": {}, "source": [ "# Hooted lyrics alignment" ] }, { "cell_type": "code", "execution_count": null, "id": "ed1a160e", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.215997Z", "start_time": "2024-04-24T02:00:39.215987Z" } }, "outputs": [], "source": [ "# if debug, may need this:\n", "# model_list, model, tokenizer = _reload_models_if_needed()" ] }, { "cell_type": "code", "execution_count": null, "id": "8dbc0f18", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:01:45.582069Z", "start_time": "2024-04-24T02:01:45.053008Z" } }, "outputs": [], "source": [ "test_input_id = \"f249512f-67f6-4f83-8af3-954a7951fdae\"\n", "# audio = Audio.from_s3(f\"s3://suno-data-uploads/studio/uploads/{test_input_id}.mp3\")\n", "audio = Audio.from_file(\"../audios/walking_down.mp3\")\n", "print(audio.duration_s)" ] }, { "cell_type": "code", "execution_count": null, "id": "c48fa719", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:01:47.041238Z", "start_time": "2024-04-24T02:01:47.028757Z" } }, "outputs": [], "source": [ "prior_texts = \"\"\"\n", "[intro]\n", "\n", "[verse]\n", "\n", "Walking down the street, feeling so alive\n", "Got my head in the clouds, got a gleam in my eye\n", "Every step I take, it's like a brand new start\n", "No matter where I'm going, I'll always find my part\n", "(oh-oh-oh)\n", "\n", "[chorus]\n", "\n", "Life is like a high-wire act, we're dancing in the sky\n", "No need to worry, no need to ask why\n", "With a little bit of courage, we can chase our dreams\n", "No matter what comes our way, we'll always be a team\n", "(we're unstoppable, yeah)\n", "\n", "[outro]\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": null, "id": "eabecb44", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:01:53.468861Z", "start_time": "2024-04-24T02:01:53.456622Z" } }, "outputs": [], "source": [ "# prior_texts = \" [outro] \\n [outro] \\n [outro] \\n\" + prior_texts + \" [outro] \\n [outro] \\n [outro] \\n\"" ] }, { "cell_type": "code", "execution_count": null, "id": "b0bf5547", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:01:54.158047Z", "start_time": "2024-04-24T02:01:53.799535Z" } }, "outputs": [], "source": [ "padded_words_timed = get_word_timing_from_audio_and_lyrics(audio, prior_texts)" ] }, { "cell_type": "code", "execution_count": null, "id": "d132b7d1", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:02:40.531772Z", "start_time": "2024-04-24T02:02:40.516775Z" } }, "outputs": [], "source": [ "print(padded_words_timed)" ] }, { "cell_type": "code", "execution_count": null, "id": "4ee614fe", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:01:54.249149Z", "start_time": "2024-04-24T02:01:54.159954Z" } }, "outputs": [], "source": [ "annotated_lines = _merge_into_lines(padded_words_timed)\n", "print(annotated_lines)\n", "assert \"\".join(k[\"text\"] for k in annotated_lines) == prior_texts" ] }, { "cell_type": "code", "execution_count": null, "id": "2542dbff", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.220381Z", "start_time": "2024-04-24T02:00:39.220373Z" } }, "outputs": [], "source": [ "print(padded_words_timed)" ] }, { "cell_type": "code", "execution_count": null, "id": "95a8f592", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.221038Z", "start_time": "2024-04-24T02:00:39.221030Z" } }, "outputs": [], "source": [ "# try chinese\n", "print(metas_map[meta_id])\n", "audio = Audio.from_s3(f\"{metas_map[meta_id]['audio_filepath']}\")\n", "prior_texts = metas_map[meta_id][\"lyrics\"]" ] }, { "cell_type": "code", "execution_count": null, "id": "57936afa", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.221628Z", "start_time": "2024-04-24T02:00:39.221620Z" } }, "outputs": [], "source": [ "padded_words_timed = get_word_timing_from_audio_and_lyrics(audio, prior_texts)" ] }, { "cell_type": "code", "execution_count": null, "id": "0d99c763", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.222457Z", "start_time": "2024-04-24T02:00:39.222449Z" } }, "outputs": [], "source": [ "print(padded_words_timed)" ] }, { "cell_type": "markdown", "id": "7799768a", "metadata": {}, "source": [ "# Direct loop" ] }, { "cell_type": "code", "execution_count": null, "id": "36e317df", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.223142Z", "start_time": "2024-04-24T02:00:39.223134Z" } }, "outputs": [], "source": [ "# hooted_info = []\n", "# with open(\n", "# f\"/home/tony/Data/Hoot/alignments/outputs_2nd_2/batch_73200.json\", \"r\"\n", "# ) as fp:\n", "# hooted_outputs = json.load(fp)\n", "# for hooted_output in hooted_outputs[2:]:\n", "# hooted_info.append(hooted_output)\n", "# break\n", "# if hooted_info:\n", "# aligned_lyrics_map = get_aligned_lyrics(\n", "# hooted_info[:1], silent=True, return_only_valid=True\n", "# )" ] }, { "cell_type": "code", "execution_count": null, "id": "461f7325", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.223864Z", "start_time": "2024-04-24T02:00:39.223856Z" } }, "outputs": [], "source": [ "meta_id = list(aligned_lyrics_map.keys())[0]\n", "original_id, lyrics = (\n", " metas_map[meta_id][\"original_id\"],\n", " metas_map[meta_id][\"lyrics\"],\n", ")\n", "audio = Audio.from_file(os.path.join(\"/app/suno/data/hoot\", f\"{original_id}.wav\"))\n", "# print(lyrics)\n", "# audio.play()\n", "print(\"check aligned clips\")\n", "# for s in aligned_lyrics_map[meta_id]:\n", "# print(\n", "# s[\"text\"],\n", "# s[\"start_s\"],\n", "# s[\"end_s\"],\n", "# s[\"end_s\"] - s[\"start_s\"],\n", "# s[\"vocal_start_s\"],\n", "# )\n", "# audio.get_segment(s[\"start_s\"], s[\"end_s\"]).play()\n", "# print(\"-\" * 10)" ] }, { "cell_type": "markdown", "id": "d4171686", "metadata": {}, "source": [ "# alignment jobs" ] }, { "cell_type": "code", "execution_count": null, "id": "cf886804", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.224444Z", "start_time": "2024-04-24T02:00:39.224437Z" } }, "outputs": [], "source": [ "# python hoot_alignment.py --start_index 1200000 --end_index 1300000" ] }, { "cell_type": "code", "execution_count": null, "id": "8443b212", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.225217Z", "start_time": "2024-04-24T02:00:39.225209Z" } }, "outputs": [], "source": [ "# python hoot_alignment.py --start_index 1050000 --end_index 1100000" ] }, { "cell_type": "code", "execution_count": null, "id": "ec2cd5ba", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.225891Z", "start_time": "2024-04-24T02:00:39.225883Z" } }, "outputs": [], "source": [ "# 1836168 / 100" ] }, { "cell_type": "code", "execution_count": null, "id": "99f9e4ae", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.226581Z", "start_time": "2024-04-24T02:00:39.226573Z" } }, "outputs": [], "source": [ "# python hoot_alignment.py --start_index 0 --end_index 1900000" ] }, { "cell_type": "code", "execution_count": null, "id": "714da821", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.227168Z", "start_time": "2024-04-24T02:00:39.227161Z" } }, "outputs": [], "source": [ "# python hoot_alignment.py --start_index 290000 --end_index 300000" ] }, { "cell_type": "code", "execution_count": null, "id": "bc9f09d3", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.227879Z", "start_time": "2024-04-24T02:00:39.227872Z" } }, "outputs": [], "source": [ "# step = 200000\n", "# for i in range(0, 1836168, step):\n", "# print(f\"python hoot_alignment.py --start_index {i} --end_index {i + step}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "7781ac60", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.228612Z", "start_time": "2024-04-24T02:00:39.228604Z" } }, "outputs": [], "source": [ "import re" ] }, { "cell_type": "code", "execution_count": null, "id": "150dfc59", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.229294Z", "start_time": "2024-04-24T02:00:39.229286Z" } }, "outputs": [], "source": [ "print(re.match(\".\\w\", \"[intro : tag]\"))\n", "print(re.match(\".\\w\", \"je's\"))\n", "print(re.match(\".\\w+\", \"]\"))\n", "print(re.match(\".\\w+\", \"\\n\"))\n", "print(re.split(\"\\w\", \"\\n\\n\"))\n", "print(re.findall(r\"\\n\\n\", \"是谁在阁楼上冰冷的绝望\\n\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "f6825ffe", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.230003Z", "start_time": "2024-04-24T02:00:39.229996Z" } }, "outputs": [], "source": [ "a = list(range(20))" ] }, { "cell_type": "code", "execution_count": null, "id": "ac4a4319", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.230597Z", "start_time": "2024-04-24T02:00:39.230589Z" } }, "outputs": [], "source": [ "a[:-5]" ] }, { "cell_type": "code", "execution_count": null, "id": "8fb90a31", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.231372Z", "start_time": "2024-04-24T02:00:39.231364Z" } }, "outputs": [], "source": [ "a[-5:]" ] }, { "cell_type": "code", "execution_count": null, "id": "d74847c8", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env_dev", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }