{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "a0d83737", "metadata": {}, "outputs": [], "source": [ "# !nvidia-smi\n", "# !echo $HOSTNAME" ] }, { "cell_type": "code", "execution_count": 1, "id": "fed7c348", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"7\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "88c3355d", "metadata": {}, "outputs": [], "source": [ "import sys\n", "sys.path.insert(0, \"/home/christian/code/neon/sunoDiff/\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "cd1bb050", "metadata": {}, "outputs": [], "source": [ "# base (25hz, 30s, ctx)\n", "# dit_model_filepath = \"/app/suno/checkpoints/2024-10-11_00-33-05_s7557/last_ckpt_infer.pt\"\n", "\n", "# prefix\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-10-11_02-04-25_s3533/last_ckpt_infer.pt\"\n", "\n", "# prefix2\n", "# dit_model_filepath = \"/app/suno/checkpoints/2024-10-13_13-04-30_s6052/last_ckpt_infer.pt\"\n", "\n", "# 100hz\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-10-11_16-53-57_s8050/last_ckpt_infer.pt\"\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-10-16_19-23-27_s9332/last_ckpt_infer.pt\"\n", "\n", "# my 100hz with context\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-10-21_15-43-06_s3766/best_ckpt.pt\" \n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-10-22_17-44-56_s9217/last_ckpt_infer.pt\"\n", "\n", "# 25hz prefix\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-10-18_17-53-25_s1400/last_ckpt_infer.pt\"\n", "\n", "# 25hz quality\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-10-21_18-29-39_s163/last_ckpt_infer.pt\"\n", "\n", "# latest 25hz prefix\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-10-26_04-39-23_s121/last_ckpt_infer.pt\"\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-10-26_04-39-23_s121/step_1000000_infer.pt\"\n", "\n", "# dpo 25hz\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-10-28_18-52-39_s5303/last_ckpt_infer.pt\"\n", "#it_model_filepath = \"/app/suno/checkpoints/2024-10-28_19-12-42_s4304/last_ckpt_infer.pt\"\n", "\n", "# finetunes 25hz\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-10-30_20-11-31_s2639/last_ckpt_infer.pt\" # first chunk only\n", "\n", "# audio loss\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-11-03_20-59-48_s4933/last_ckpt_infer.pt\"\n", "\n", "#100hz \n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-11-01_13-16-18_s411/last_ckpt_infer.pt\"\n", "\n", "# 25hz dpo v2\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-11-04_14-51-03_s9527/last_ckpt_infer.pt\"\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-11-04_15-08-19_s4277/last_ckpt_infer.pt\" # reverse labels\n", "\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-11-13_17-32-12_s3878/last_ckpt_infer.pt\"\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-10-26_04-39-23_s121/step_1000000_infer.pt\"\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-11-14_00-55-41_s2643/last_ckpt_infer.pt\"\n", "\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-10-30_15-14-54_s7904/last_ckpt_infer.pt\"\n", "\n", "#dpo 2\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-11-16_16-40-24_s2816/last_ckpt_infer.pt\"\n", "\n", "# tony dpo\n", "#dit_model_filepath = \"/app/suno/checkpoints/2024-11-08_15-04-27_s9054/last_ckpt_infer.pt\"\n", "dit_model_filepath = \"/app/suno/checkpoints/2024-11-18_01-35-12_s6013/last_ckpt_infer.pt\"\n", "\n", "dit_model_filepath = \"s3://suno-data/georg/tmp/2b_prefix_ft.pt\"\n", "\n", "# warning: this model is raw 10s\n", "dit_model_filepath = \"/app/suno/checkpoints/2024-11-18_22-22-35_s4893/last_ckpt_infer.pt\"\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "13171812", "metadata": {}, "outputs": [], "source": [ "import torch\n", "from suno_utils.audio import Audio\n", "from generation import preload_models, special_generate, _retrieve_models\n", "_ = preload_models(\n", " tokenizer_filepath=\"/home/georg/notebooks/gpu_nb/tmp/tokenizer_60k.json\",\n", " semantic_model_filepath=\"/home/georg/notebooks/gpu_nb/tmp/mert_25.pt\",\n", " semantic_clusters_filepath=\"/home/georg/notebooks/gpu_nb/tmp/mert_25_2x4k.npy\",\n", " #codec_filepath=\"/home/georg/notebooks/gpu_nb/tmp/100hz_vae_peaq_kl_0.005.pth\",\n", " codec_filepath=\"/home/georg/notebooks/gpu_nb/tmp/25hz_vae_peaq_kl_0.005.pth\",\n", " dit_model_filepath=dit_model_filepath,\n", " weights_precision=torch.bfloat16,\n", " #compile=True,\n", ")\n", "\n", "models = _retrieve_models()\n", "model_duration_s = 30\n", "if models[\"dit_model\"].ctx_len is not None:\n", " model_duration_s = 6 * 60\n", "else:\n", " model_duration_s = models[\"dit_model\"].block_size // models[\"dit_model\"].io_hz\n", "duration_s = 2*60 if model_duration_s >= 2*60 else 30" ] }, { "cell_type": "code", "execution_count": null, "id": "776b734c", "metadata": {}, "outputs": [], "source": [ "import json\n", "from suno_utils.audio import Audio\n", "import numpy as np\n", "from suno_utils.utils.s3 import read_from_s3\n", "\n", "\n", "#gen_id = \"99bffa17-7e59-47b4-a048-5528cbda05d5\" # sister\n", "#gen_id = \"081d73c4-7805-4212-9c80-8db1137ca3c4\" # friends\n", "#gen_id = \"562f762d-6ced-4080-9af1-910ee3d0a5dc\" # something real\n", "#gen_id = \"23c15c62-494d-422d-8a60-8b0454044322\" # rubber duck\n", "#gen_id = \"4b140a9e-964b-422c-85b5-5861ad1a9d38\" # once\n", "#gen_id = \"7b214347-fa38-4e9b-96f4-f7ec65adea45\" # rock n roll\n", "#gen_id = \"a5e2198a-f352-4abb-9a24-7f81b143ded3\" # stone\n", "#gen_id = \"7f5e7819-b7a1-449d-a76c-9c38950c4274\" # canada\n", "#gen_id = \"27135940-957f-4152-9e40-72e5c10a46bd\" # drunk\n", "#gen_id = \"caf1d7ae-ce81-4c75-b737-fd41e47a38ba\" # linger\n", "#gen_id = \"ac5aa29a-f18f-406f-a904-c47493f1beec\" # linger vox\n", "#gen_id = \"b04719c3-012a-4a4a-9b02-3cb2875d1744\" # courtney\n", "#gen_id = \"7407c7b3-8aa2-4f0a-82e8-fa19933f6e94\" # opera\n", "#gen_id = \"5b9624ef-b416-4095-a5f3-4945f09bba86\" # butterflies\n", "#gen_id = \"e757aa19-5c33-4945-aaa1-1519b7890f0b\" # bubblegum\n", "#gen_id = \"7f774078-1672-4858-a37f-acad373c5a84\" # golden\n", "#gen_id = \"6727880b-f5e8-4c48-a401-5831d14c71eb\" # rock instrumental/\n", "#gen_id = \"8e96a40b-f528-4cfd-9b25-edcdc50080d1\" # 80s\n", "#gen_id = \"88fe5e81-b07f-462b-8130-04fc7c964963\" # stary night\n", "#gen_id = \"3617dc97-2c11-4da4-8b26-773927fa9c7b\" # makeup\n", "#gen_id = \"7735f201-7cce-4067-9496-67f590ec01fc\" # back pain\n", "#gen_id = \"77260536-b692-4577-aac7-a53c13754ec4\" # love again\n", "#gen_id = \"707299a3-7668-484f-8479-421e516c6916\" # italian\n", "#gen_id = \"88427bb5-0b29-48bd-9f71-83bec2bdbd59\" # monarchy out of business\n", "\n", "# shimmer examples\n", "#gen_id = \"72a8c50d-7e16-469d-8283-56bed483374c\"\n", "#gen_id = \"382b1682-8503-4b3d-94b6-5fa296933975\" # orchestral ringing\n", "#gen_id = \"cee9773b-9514-43bf-bba4-6561264ea94a\" # strong ringning\n", "gen_id = \"7c62f810-f77d-4a98-b179-d52c538a996f\" # strong ringing 2\n", "#gen_id = \"8e176b20-7038-414d-8630-696be8911c09\" # metal\n", "\n", "s3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.npz\"\n", "mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.mp3\"\n", "\n", "\n", "print(s3_filepath)\n", "data = read_from_s3(s3_filepath, read_f=np.load)\n", "\n", "audio = Audio.from_s3(mp3_filepath, n_channels=2).get_slice(0, 60.01)\n", "\n", "if \"v3.0_raw\" in data:\n", " codes = data[\"v3.0_raw\"]\n", "elif \"v3.5_raw\" in data:\n", " codes = data[\"v3.5_raw\"]\n", "elif \"v4.0_raw\" in data:\n", " codes = data[\"v4.0_raw\"]\n", "else:\n", " raise ValueError(\"No codes found\")\n", "\n", "text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{gen_id}_hoot.json\")\n", "aligned_lyrics = json.loads(text_data)\n", "print(aligned_lyrics)\n", "audio.normalize_volume().play()\n", "\n", "lyrics = \"\"\n", "for elem in aligned_lyrics:\n", " if \"word\" in elem:\n", " lyrics += elem[\"word\"]\n" ] }, { "cell_type": "code", "execution_count": null, "id": "2daf1bc9", "metadata": {}, "outputs": [], "source": [ "semantic_codes = torch.from_numpy(codes[:, 0]).long().cuda()\n", "semantic_codes = semantic_codes[0:250]\n", "print(semantic_codes.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "98fb6f88", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import IPython\n", "\n", "steps = 128\n", "text_cfg_coef = 12.0\n", "ctx_cfg_coef = 1.0\n", "\n", "audios = []\n", "#max_sigmas = [1, 2.5, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]\n", "#min_sigmas = [0.05, 0.1, 0.5, 1]\n", "\n", "empty_semantic_codes = torch.ones([1500], device=\"cuda\").long() * 4000\n", "\n", "yum_lyrics = \"\"\"\n", "[verse]\n", "A is for the amazing grace that we recieve\n", "B is for the blessings, every day we blieve\n", "C is for the chorus, we sing it loud and clear\n", "D is for the devotion that we hold dear\n", "\n", "[Chorus]\n", "B is for Buttocks, ripe and slightly damp\n", "Yum Yum, boy oh boy do I like God\n", "Every day is a gift, when you won a skateboard ramp.\n", "Yum yum, boy oh boy do I like God.\n", "\n", "[outro]\n", "\"\"\"\n", "\n", "\n", "max_sigma = 50.0\n", "\n", "seeds = [np.random.randint(0, 2**32-1) for _ in range(3)]\n", "#seeds = [0]\n", "scaled_pred_z_list = []\n", "\n", "for sampling_method in [\"ddim\"]:\n", " #print(f\"sample_fn: {sampling_method}\")\n", " for seed in seeds:\n", " scaled_pred_z = special_generate(\n", " semantic_codes, \n", " lyrics=lyrics,\n", " #aligned_lyrics=aligned_lyrics,\n", " tags=\"rock\", \n", " text_cfg_coef=text_cfg_coef,\n", " ctx_cfg_coef=ctx_cfg_coef,\n", " steps=steps,\n", " seed=seed,\n", " sigma_max=max_sigma,\n", " #sampling_method=sampling_method\n", " )\n", " scaled_pred_z = scaled_pred_z.view(1, 2, 250, 1920)\n", " print(scaled_pred_z.shape)\n", " reconstructed_audio = reconstruct_audio(scaled_pred_z, 1920 * 25 * 10).squeeze(0)\n", " print(reconstructed_audio.shape)\n", " IPython.display.display(IPython.display.Audio(reconstructed_audio, rate=48000))\n", "\n", " #pred_audio.normalize_volume().play()\n", " #IPython.display.display(IPython.display.Audio(pred_audio, rate=48000))\n" ] }, { "cell_type": "code", "execution_count": null, "id": "506a4a32", "metadata": {}, "outputs": [], "source": [ "print(scaled_pred_z.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "656adb9a", "metadata": {}, "outputs": [], "source": [ "\n" ] }, { "cell_type": "code", "execution_count": 8, "id": "a7641d74", "metadata": {}, "outputs": [], "source": [ "\n", "def reconstruct_audio(\n", " frames: torch.Tensor, original_length: int, overlap: float = 0.0\n", ") -> torch.Tensor:\n", " \"\"\"\n", " Reconstruct audio signal from frames. Uses overlap-add only when overlap > 0.\n", "\n", " Args:\n", " frames: Input tensor of shape (batch_size, channels, num_frames, frame_size)\n", " original_length: Length of the original sequence\n", " overlap: Overlap used in framing\n", " \"\"\"\n", " batch_size, channels, num_frames, frame_size = frames.shape\n", " hop_size = int(frame_size * (1 - overlap))\n", "\n", " # For no overlap, we can just reshape\n", " if overlap == 0:\n", " # Check if the frames can be directly reshaped\n", " expected_length = num_frames * frame_size\n", " if expected_length == original_length:\n", " return frames.reshape(batch_size, channels, -1)\n", " else:\n", " # If not exact match, still do frame-by-frame to handle partial frames\n", " output = torch.zeros(\n", " batch_size,\n", " channels,\n", " original_length,\n", " dtype=frames.dtype,\n", " device=frames.device,\n", " )\n", " for i in range(num_frames):\n", " start_idx = i * frame_size\n", " end_idx = min(start_idx + frame_size, original_length)\n", " output[:, :, start_idx:end_idx] = frames[\n", " :, :, i, : (end_idx - start_idx)\n", " ]\n", " return output\n", "\n", " # For overlap > 0, use overlap-add (Hann window sum to 1)\n", " output = torch.zeros(\n", " batch_size, channels, original_length, dtype=frames.dtype, device=frames.device\n", " )\n", "\n", " for i in range(num_frames):\n", " start_idx = i * hop_size\n", " end_idx = start_idx + frame_size\n", " output[:, :, start_idx:end_idx] += frames[:, :, i]\n", "\n", " return output[:, :, :original_length]\n" ] }, { "cell_type": "code", "execution_count": null, "id": "09da74ab", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "908498d4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "bb494e8a", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": null, "id": "9843759c", "metadata": {}, "outputs": [], "source": [ "# # align audio/lyrics\n", "# from suno_utils.tasks.hoot import get_word_timing_from_audio_and_lyrics, preload_models\n", "# _ = preload_models(\n", "# checkpoint_filepath=\"/app/suno/models/hoot_v3.pt\",\n", "# tokenizer_filepath=\"/app/suno/models/hoot_v3_tokenizer.model\",\n", "# )\n", "# aligned_lyrics = get_word_timing_from_audio_and_lyrics(audio, lyrics)" ] }, { "cell_type": "code", "execution_count": null, "id": "9c946bc7", "metadata": {}, "outputs": [], "source": [ "# import torch\n", "# models = _retrieve_models()\n", "# extra_args = torch.load(\"extra_args.pt\", weights_only=True)\n", "# torch.manual_seed(0)\n", "# x = torch.randn([1, 128, 100*30], device=\"cuda\")*50\n", "# with torch.no_grad():\n", "# out = models[\"dit_model\"].forward_inference(\n", "# x,\n", "# x.new_ones([x.shape[0]])*50,\n", "# cross_attn_cond=extra_args[\"cross_attn_cond\"],\n", "# empty_cross_attn_cond=extra_args[\"empty_cross_attn_cond\"],\n", "# cfg_scale=extra_args[\"cfg_scale\"]\n", "# )\n", "# print(out.shape)\n", "# out" ] }, { "cell_type": "code", "execution_count": null, "id": "4a93b889", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7aa732de", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6e5084b6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "04c4bd6b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "acda5fe8", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env2", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" } }, "nbformat": 4, "nbformat_minor": 5 }