{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "a8156ae5",
"metadata": {
"ExecuteTime": {
"end_time": "2024-02-19T01:17:45.859750Z",
"start_time": "2024-02-19T01:17:45.855723Z"
}
},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"2\""
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "d21a1cda",
"metadata": {
"ExecuteTime": {
"end_time": "2024-02-19T01:18:34.987107Z",
"start_time": "2024-02-19T01:18:32.769145Z"
}
},
"outputs": [],
"source": [
"import whisper\n",
"from suno_utils.audio import Audio\n",
"import tqdm\n",
"from torcheval.metrics import WordErrorRate\n",
"import re\n",
"from suno_utils.utils.text import normalize_whitespace\n",
"import copy\n",
"import tqdm\n",
"import numpy as np\n",
"from suno_utils.utils.lyrics import remove_speakers"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "79fd153b",
"metadata": {
"ExecuteTime": {
"end_time": "2024-02-19T01:18:38.970326Z",
"start_time": "2024-02-19T01:18:35.763638Z"
}
},
"outputs": [],
"source": [
"model = whisper.load_model(\"small\") # large-v2 takes like 20 sec per song"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "53236582",
"metadata": {
"ExecuteTime": {
"end_time": "2024-02-19T01:18:41.826752Z",
"start_time": "2024-02-19T01:18:38.973166Z"
}
},
"outputs": [],
"source": [
"model_2 = whisper.load_model(\"small.en\") # large-v2 takes like 20 sec per song"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "78524879",
"metadata": {
"ExecuteTime": {
"end_time": "2024-02-19T01:18:44.034010Z",
"start_time": "2024-02-19T01:18:41.829162Z"
}
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" And you're all alone so I can't you see You belong with me Standing by you, waiting at your back door All this time, how could you not know, baby\n"
]
}
],
"source": [
"input_audio = Audio.from_file(\"audios/test_swift.mp3\")\n",
"test_input_audio = input_audio.get_segment(from_s=180, to_s=200.1)\n",
"test_input_audio.play(compress=False)\n",
"result = model.transcribe(\n",
" test_input_audio.convert(16000, 2, 1).array_float.copy(),\n",
" initial_prompt=None,\n",
" condition_on_previous_text=False,\n",
")\n",
"print(result[\"text\"])"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "90397ac1",
"metadata": {
"ExecuteTime": {
"end_time": "2024-02-19T01:29:04.575701Z",
"start_time": "2024-02-19T01:29:04.024088Z"
}
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Thanks for watching!\n"
]
}
],
"source": [
"input_audio = Audio.from_file(\"/home/tony/test.mp3\")\n",
"test_input_audio = input_audio.get_segment(from_s=70)\n",
"test_input_audio.play(compress=False)\n",
"result = model.transcribe(\n",
" test_input_audio.convert(16000, 2, 1).array_float.copy(),\n",
" initial_prompt=None,\n",
" condition_on_previous_text=False,\n",
")\n",
"print(result[\"text\"])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "eb2b1568",
"metadata": {
"ExecuteTime": {
"end_time": "2023-11-13T15:32:39.452587Z",
"start_time": "2023-11-13T15:29:45.208807Z"
}
},
"outputs": [],
"source": [
"validation_datasets = {\n",
" \"english\": \"/home/tony/Data/Hoot/en_test_manifest_norm.json\",\n",
" \"multi\": \"/home/tony/Data/Hoot/all_test_manifest.json\",\n",
" \"multi_balanced\": \"/home/tony/Data/Hoot/multi_balanced_test_manifest.json\",\n",
"}\n",
"whisper_results = {}\n",
"for validation_set, validation_path in validation_datasets.items():\n",
" log_dict = {}\n",
" val_paths = []\n",
" val_truth_text = []\n",
" with open(validation_path, \"r\") as fp:\n",
" for l in fp:\n",
" val_meta = json.loads(l)\n",
" val_paths.append(val_meta[\"audio_filepath\"])\n",
" val_truth_text.append(val_meta[\"text\"])\n",
" whisper_outs = []\n",
" for val_path in tqdm.tqdm(val_paths):\n",
" audio = Audio.from_file(val_path)\n",
" result = model.transcribe(\n",
" audio.convert(16000, 2, 1).array_float.copy(),\n",
" initial_prompt=None,\n",
" condition_on_previous_text=False,\n",
" )\n",
" # result is: text, segment, lang\n",
" # print(result[\"language\"])\n",
" whisper_outs.append(result[\"text\"])\n",
" whisper_results[val_path] = result\n",
" # will take 1.5 mins\n",
" default_wer = WordErrorRate(device=\"cuda\")\n",
" default_wer.update(whisper_outs, val_truth_text)\n",
" wer_value = default_wer.compute().item()\n",
" print(validation_set, wer_value)\n",
" log_dict[validation_set] = wer_value\n",
" # wandb.log(log_dict)\n",
" break"
]
},
{
"cell_type": "markdown",
"id": "40e1b789",
"metadata": {},
"source": [
"# Direct Validation"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "27e5667b",
"metadata": {
"ExecuteTime": {
"end_time": "2023-11-13T21:54:39.914057Z",
"start_time": "2023-11-13T21:54:39.910260Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"1000"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(english_results)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "35eb6798",
"metadata": {
"ExecuteTime": {
"end_time": "2023-11-14T12:56:58.335917Z",
"start_time": "2023-11-14T12:56:58.332660Z"
}
},
"outputs": [],
"source": [
"def clean_text(text):\n",
" \"\"\"General text cleaning.\"\"\"\n",
" text = \"\\n\" + text\n",
" text = text.replace(\"’\", \"'\").lower()\n",
" text = text.replace('\"', \"\").lower()\n",
" text = re.sub(r\"\\[.+?\\]\", \" \", text) # tags\n",
" text = re.sub(r\"\\n.+?\\:\", \" \", text) # new line ends with :\n",
" text = re.sub(r\"\\n.+?\\：\", \" \", text) # new line ends with :\n",
" text = re.sub(r\"\\n\\(.+?\\)\", \" \", text) # new line with ()\n",
" text = re.sub(r\"[\\d]\", \" \", text) # digits\n",
" text = re.sub(r\"▁\", \"\", text) # special stuff\n",
" text = remove_speakers(text)\n",
" text = re.sub(r\"[^\\w\\'\\s]\", \" \", text) # keep only the words\n",
" text = normalize_whitespace(text)\n",
" return text"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "d34d04f1",
"metadata": {
"ExecuteTime": {
"end_time": "2023-11-14T13:00:00.791166Z",
"start_time": "2023-11-14T13:00:00.632996Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"english.json multi.json multi_balanced.json\r\n"
]
}
],
"source": [
"!ls /home/tony/Data/Hoot/whipser/"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "eb1a51e8",
"metadata": {
"ExecuteTime": {
"end_time": "2023-11-14T13:03:15.685465Z",
"start_time": "2023-11-14T13:01:25.725214Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"multi 0.448591411113739\n",
"english 0.4171019196510315\n",
"multi_balanced 0.5840970277786255\n"
]
}
],
"source": [
"validation_datasets = {\n",
" \"multi\": \"/home/tony/Data/Hoot/all_test_manifest.json\",\n",
" \"english\": \"/home/tony/Data/Hoot/en_test_manifest_norm.json\",\n",
" \"multi_balanced\": \"/home/tony/Data/Hoot/multi_balanced_test_manifest.json\",\n",
"}\n",
"log_dict = {}\n",
"for validation_set, validation_path in validation_datasets.items():\n",
" val_paths = []\n",
" val_truth_text = []\n",
" whisper_outs = []\n",
" with open(f\"/home/tony/Data/Hoot/whipser/{validation_set}.json\", \"r\") as fp:\n",
" whisper_results = json.load(fp)\n",
" with open(validation_path, \"r\") as fp:\n",
" for l in fp:\n",
" val_meta = json.loads(l)\n",
" val_paths.append(val_meta[\"audio_filepath\"])\n",
" val_truth_text.append(clean_text(val_meta[\"text\"]))\n",
" try:\n",
" whisper_outs.append(\n",
" clean_text(whisper_results[val_meta[\"audio_filepath\"]][\"text\"])\n",
" )\n",
" except:\n",
" print(val_meta[\"audio_filepath\"])\n",
"\n",
" default_wer = WordErrorRate(device=\"cpu\")\n",
" default_wer.update(whisper_outs, val_truth_text)\n",
" wer_value = default_wer.compute().item()\n",
"\n",
" print(validation_set, wer_value)\n",
" log_dict[validation_set] = wer_value"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "9420d71b",
"metadata": {
"ExecuteTime": {
"end_time": "2023-11-14T13:04:05.808170Z",
"start_time": "2023-11-14T13:03:46.583886Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mtonytongsuno\u001b[0m (\u001b[33msuno\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
]
},
{
"data": {
"text/html": [
"wandb version 0.16.0 is available! To upgrade, please run:\n",
" $ pip install wandb --upgrade"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Tracking run with wandb version 0.15.8"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Run data is saved locally in /home/tony/Work/tony/wandb/run-20231114_130348-2jrz4m61"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Syncing run whisper_large_v2 to Weights & Biases (docs)
"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
" View project at https://wandb.ai/suno/hoot_evals"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
" View run at https://wandb.ai/suno/hoot_evals/runs/2jrz4m61"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import wandb\n",
"wandb.init(project=\"hoot_evals\", name=f\"whisper_large_v2\")"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "2a5b0673",
"metadata": {
"ExecuteTime": {
"end_time": "2023-11-14T13:04:12.896693Z",
"start_time": "2023-11-14T13:04:12.892736Z"
}
},
"outputs": [],
"source": [
"# wandb.log(log_dict)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6264105e",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.13"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": true,
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 5
}