{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GPT Embed\n",
    "Extract embeddings from intermediate layers of pretrained GPT for use in downstream tasks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "\n",
    "# setup GPT\n",
    "from suno_utils.gpt.chirp_v2_5 import (\n",
    "    GenerationConfig,\n",
    "    decode_stream,\n",
    "    preload_models,\n",
    "    prep_gconf,\n",
    "    codec_decode_stream_to_full_audio,\n",
    "    generate,\n",
    "    generate_stream,\n",
    ")\n",
    "\n",
    "from suno_utils.gpt.generation import load_model, prepare_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "gpt_ckpt_path = \"/app/suno/data/dpo/models/model_13b_full.pt\"  # model before fine-tuning\n",
    "#gpt_ckpt_path = \"/app/suno/checkpoints/2024-07-12_13-28-28/last_ckpt.pt\"  # 2b trained only on semantic\n",
    "#gpt_ckpt_path = \"/app/suno/checkpoints/2024-07-13_20-42-16/last_ckpt.pt\"\n",
    "#gpt_ckpt_path = \"/app/suno/checkpoints/2024-07-14_21-24-26/last_ckpt.pt\"\n",
    "#gpt_ckpt_path = \"/app/suno/checkpoints/2024-07-16_18-46-08/last_ckpt.pt\"\n",
    "tokenizer_path = \"/app/suno/data/dpo/models/tokenizer_60k.json\"\n",
    "\n",
    "model = load_model(gpt_ckpt_path, tokenizer_path, use_gpu=True)\n",
    "print(model)\n",
    "model = model[\"model\"]\n",
    "\n",
    "#preload_models(\n",
    "#    gpt_ckpt_path=gpt_ckpt_path,\n",
    "#    tokenizer_path=tokenizer_path,\n",
    "#    load_gpt=True,\n",
    "#    load_semantic=False,\n",
    "#    load_codec_device=\"cuda\",\n",
    "#)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load npz \n",
    "# load npz file with numpy\n",
    "gen_npz = np.load(\"/app/suno/christian/data/npz/a5e2198a-f352-4abb-9a24-7f81b143ded3.npz\")\n",
    "print(gen_npz.files)\n",
    "data = gen_npz[\"v3.0_raw\"]\n",
    "print(data.shape)\n",
    "\n",
    "history_arr = data[:1500,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shift history etc\n",
    "cfg = model.config\n",
    "n_semantic_offsets = cfg.semantic_n_codebooks * cfg.semantic_shift_factor\n",
    "\n",
    "text_arr = torch.full((1, 1, cfg.block_size), cfg.text_pad_token, dtype=torch.long)\n",
    "\n",
    "history_arr = prepare_history(history_arr)\n",
    "n_history = (\n",
    "    history_arr.shape[0]\n",
    "    - n_semantic_offsets\n",
    "    - (cfg.coarse_n_codebooks - 1) * cfg.coarse_shift_factor\n",
    ")\n",
    "audio_arr = torch.from_numpy(history_arr[:n_history].astype(np.int64))[None].swapaxes(-2, -1)\n",
    "\n",
    "x = torch.concat([text_arr[:, :, : audio_arr.shape[-1]], audio_arr], dim=1).cuda()\n",
    "x_pos = torch.arange(0, x.shape[-1], device=\"cuda\")\n",
    "print(x.shape, x_pos.shape)\n",
    "print(x_pos)\n",
    "\n",
    "out = {\"test\" : []}\n",
    "semantic_logits, coarse_logits = model(x, x_pos, debug_out=out)\n",
    "\n",
    "for key, val in out.items():\n",
    "    if \"block_\" in key:\n",
    "        print(key, val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run the model\n",
    "gconfig = GenerationConfig(\n",
    "    max_gen_duration_s=10,\n",
    "    temp_semantic=0.9,\n",
    "    #top_k_semantic=None,\n",
    "    #top_p_semantic=None,\n",
    "    history_arr=history_arr,\n",
    "    debug=True,\n",
    "    return_semantic=True,\n",
    ")\n",
    "\n",
    "out = {}\n",
    "for a in generate_stream(gconfig, out=out):\n",
    "    print(\"a\")\n",
    "\n",
    "debug_info = out[\"debug_info\"]\n",
    "gpt_debug_out = debug_info.gpt_debug_out\n",
    "gpt_debug_out_prefill = debug_info.gpt_debug_out_prefill\n",
    "\n",
    "\n",
    "for key, val in gpt_debug_out_prefill.items():\n",
    "    print(key, val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}