{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5fdf7b20",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-08T00:52:42.191954Z",
     "start_time": "2023-11-08T00:52:41.907297Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "from suno_utils.audio import Audio\n",
    "\n",
    "import torch\n",
    "from torcheval.metrics import WordErrorRate\n",
    "import json\n",
    "\n",
    "from suno_utils.tasks.hoot import (\n",
    "    preload_models,\n",
    "    encode_filepaths,\n",
    "    encode,\n",
    "    Tokenizer,\n",
    "    clean_models,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4f527b0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-08T00:52:47.174825Z",
     "start_time": "2023-11-08T00:52:44.508724Z"
    }
   },
   "outputs": [],
   "source": [
    "## suno hoot\n",
    "_ = preload_models(\n",
    "    checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-01-15_03-43-37/25k_ckpt.pt\",\n",
    "    tokenizer_filepath=\"/app/suno/models/hoot_v3_tokenizer.model\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "929ce86d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-08T00:52:47.179643Z",
     "start_time": "2023-11-08T00:52:47.177144Z"
    }
   },
   "outputs": [],
   "source": [
    "# val_input_path = \"/home/tony/Data/Hoot/en_test_manifest_norm.json\"\n",
    "# val_input_path = \"/home/tony/Data/Hoot/all_test_manifest.json\"\n",
    "val_input_path = \"/home/tony/Data/Hoot/multi_balanced_test_manifest.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8baa93e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-08T00:52:47.496174Z",
     "start_time": "2023-11-08T00:52:47.181157Z"
    }
   },
   "outputs": [],
   "source": [
    "val_paths = []\n",
    "val_truth_text = []\n",
    "with open(val_input_path, \"r\") as fp:\n",
    "    for l in fp:\n",
    "        val_meta = json.loads(l)\n",
    "        val_paths.append(val_meta[\"audio_filepath\"])\n",
    "        val_truth_text.append(val_meta[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27dcd680",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-08T00:52:47.498404Z",
     "start_time": "2023-11-08T00:52:47.498394Z"
    }
   },
   "outputs": [],
   "source": [
    "print(len(val_paths))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43a49b45",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-08T00:52:47.499179Z",
     "start_time": "2023-11-08T00:52:47.499170Z"
    }
   },
   "outputs": [],
   "source": [
    "# will take 1.5 mins\n",
    "default_hoot_outs = encode_filepaths(val_paths, batch_size=120)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19830150",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-08T00:52:47.499903Z",
     "start_time": "2023-11-08T00:52:47.499895Z"
    }
   },
   "outputs": [],
   "source": [
    "len(default_hoot_outs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22ffcbdf",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-08T00:52:47.500601Z",
     "start_time": "2023-11-08T00:52:47.500593Z"
    }
   },
   "outputs": [],
   "source": [
    "default_wer = WordErrorRate(device=\"cuda\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "417623aa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-08T00:52:47.501258Z",
     "start_time": "2023-11-08T00:52:47.501250Z"
    }
   },
   "outputs": [],
   "source": [
    "# will take 20 sec\n",
    "default_wer.update(default_hoot_outs, val_truth_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0a7f4f2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-08T00:52:47.502048Z",
     "start_time": "2023-11-08T00:52:47.502040Z"
    }
   },
   "outputs": [],
   "source": [
    "default_wer.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8808f297",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-08T00:52:47.502737Z",
     "start_time": "2023-11-08T00:52:47.502729Z"
    }
   },
   "outputs": [],
   "source": [
    "default_hoot_outs[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46259fc7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-08T00:52:47.503391Z",
     "start_time": "2023-11-08T00:52:47.503383Z"
    }
   },
   "outputs": [],
   "source": [
    "val_truth_text[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b4101c5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-07T16:03:08.669603Z",
     "start_time": "2023-11-07T16:03:08.621416Z"
    }
   },
   "source": [
    "on EN (val set)\n",
    "- default hoot wer is 0.5666\n",
    "- new en hoot wer is 0.492\n",
    "- new multi hoot wer is (unfinished): 0.5761\n",
    "\n",
    "on Multi (val set)\n",
    "- default hoot wer is 0.7333\n",
    "- new en hoot wer is 0.6695\n",
    "- new multi hoot wer is (unfinished): 0.6914\n",
    "- balanced multi hoot wer is (unfinished): 0.8114\n",
    "\n",
    "on new re-balanced Multi (val set)\n",
    "- default hoot wer is: \n",
    "- new en hoot wer is:\n",
    "- new multi hoot wer is (unfinished): \n",
    "- balanced multi hoot wer is (unfinished): "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5cef6651",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_env_dev",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}