{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "execution": { "iopub.execute_input": "2025-10-16T15:35:56.954862Z", "iopub.status.busy": "2025-10-16T15:35:56.954627Z", "iopub.status.idle": "2025-10-16T15:36:04.187586Z", "shell.execute_reply": "2025-10-16T15:36:04.186865Z", "shell.execute_reply.started": "2025-10-16T15:35:56.954845Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Failed to import flash_attn_3.\n", "Running on non-Modal\n", "Dir1: 526 files\n", "Dir2: 196 files\n", "Common files: 196\n", "All unique files: 526\n", "Match percentage: 37.26%\n", "Similarity: 37.26%\n", "Directories are not identical. Use disk model instead, if you want to use ssd models. run python -m suno_utils.worker.modal_manager_model_store sync\n" ] } ], "source": [ "import os\n", "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"5\"\n", "\n", "import json\n", "from suno_utils.audio import Audio\n", "from suno_utils.utils.text import read_jsonl\n", "from suno_utils.utils.s3 import list_s3_dir, read_from_s3\n", "\n", "import torch\n", "import numpy as np\n", "\n", "from suno_utils.diffusion.generation import (\n", " preload_dit_model,\n", " preload_tokenizer,\n", " TOKENIZER_FILEPATH,\n", " SEMANTIC_MODEL_FILEPATH,\n", " SEMANTIC_CLUSTERS_FILEPATH,\n", " _retrieve_models,\n", ")\n", "\n", "\n", "from suno_utils.tasks.mert_25 import (\n", " preload_models as preload_semantic_models,\n", " encode as encode_semantic,\n", ")\n", "\n", "from suno_utils.tasks.dac_vae_fixed_25hz import (\n", " preload_models as preload_codec_models,\n", " decode as codec_decode,\n", " decode_stream_to_full_audio,\n", ")\n", "\n", "from suno_utils.diffusion import generation as diffusion_gen\n", "from suno_utils.tasks.upsample_engine import UpsampleEngine, Request, Job\n", "#from suno_utils.tasks.upsample_engine_old import UpsampleEngine, Request, Job\n", "\n", "# import sys\n", "# sys.path.insert(0, \"/home/m4burns/glockenspiel/suno_utils/notebooks/shimmerscore/\")" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "execution": { "iopub.execute_input": "2025-10-16T15:36:04.189894Z", "iopub.status.busy": "2025-10-16T15:36:04.189766Z", "iopub.status.idle": "2025-10-16T15:36:50.705655Z", "shell.execute_reply": "2025-10-16T15:36:50.705059Z", "shell.execute_reply.started": "2025-10-16T15:36:04.189878Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Found 1 GPUs. Using GPU 0.\n", "loading diffusion model...\n", "loading weights...\n", "converting model to precision torch.bfloat16...\n", "loading tokenizer...\n", "cropped extra layers of MERT model to 7\n" ] } ], "source": [ "# load diffusion model\n", "num_gpus = torch.cuda.device_count()\n", "cuda_device = torch.cuda.current_device()\n", "print(f\"Found {num_gpus} GPUs. Using GPU {cuda_device}.\")\n", "\n", "# dit models\n", "# dit_model_filepath = \"/app2/suno/checkpoints/2025-09-17_14-08-27_s1591/step_6000_infer.pt\"\n", "# dit_model_filepath = \"/app2/suno/checkpoints/2025-10-04_02-52-45_s4736/last_ckpt_infer.pt\"\n", "dit_model_filepath = \"/home/tony/Data/checkpoints/diff/diff_v2_v4_39_25pct_extrapolate_3.pt\"\n", "\n", "# other models\n", "tokenizer_filepath = \"s3://suno-data/georg/models/tokenizers/tokenizer_60k.json\"\n", "semantic_model_filepath = \"s3://suno-data/georg/models/semantic/mert_25.pt\"\n", "semantic_clusters_filepath = (\n", " \"s3://suno-data/georg/models/semantic/mert_25_2x4k.npy\"\n", ")\n", "codec_filepath = \"s3://suno-data/minz/models/dac_vae_tuned_25hz.pth\"\n", "CODEC_SCALE_FACTOR = 0.4\n", "SCALE_CTX_VECTOR = True\n", "\n", "_ = diffusion_gen.preload_dit_model(\n", " dit_model_filepath=dit_model_filepath,\n", " use_ema_if_exists=True,\n", " compile=False,\n", " weights_precision=torch.bfloat16,\n", ")\n", "_ = preload_tokenizer(tokenizer_filepath)\n", "_ = preload_semantic_models(semantic_model_filepath, semantic_clusters_filepath)\n", "_ = preload_codec_models(codec_filepath)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "execution": { "iopub.execute_input": "2025-10-16T15:36:50.707605Z", "iopub.status.busy": "2025-10-16T15:36:50.707472Z", "iopub.status.idle": "2025-10-16T15:36:50.710160Z", "shell.execute_reply": "2025-10-16T15:36:50.709731Z", "shell.execute_reply.started": "2025-10-16T15:36:50.707590Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Block size: 750\n" ] } ], "source": [ "diffusion_engine = UpsampleEngine(min_chunk_size=25*30) #, vae_version=\"v_vae_25_tuned_2\")" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2025-10-16T15:36:50.711757Z", "iopub.status.busy": "2025-10-16T15:36:50.711634Z", "iopub.status.idle": "2025-10-16T15:36:55.292880Z", "shell.execute_reply": "2025-10-16T15:36:55.292283Z", "shell.execute_reply.started": "2025-10-16T15:36:50.711744Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3b1830d7-5553-4785-be0b-ffeadf7756d8\n", "torch.Size([3000])\n", "KeysView()\n", "(3100, 128)\n", "None\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# load the test clip here\n", "item_id = \"3b1830d7-5553-4785-be0b-ffeadf7756d8\"\n", "\n", "print(item_id)\n", "# load the semantic codes from s3\n", "s3_filepath = f\"s3://suno-data-uploads/studio/uploads/{item_id}.npz\"\n", "\n", "try:\n", " data = read_from_s3(s3_filepath, read_f=np.load)\n", "except Exception as e:\n", " print(f\"Error loading {s3_filepath}: {e}\")\n", "\n", "if \"v3.0_raw\" in data:\n", " codes = data[\"v3.0_raw\"]\n", "elif \"v3.5_raw\" in data:\n", " codes = data[\"v3.5_raw\"]\n", "elif \"v4.0_raw\" in data:\n", " codes = data[\"v4.0_raw\"]\n", "elif \"v5.0_raw\" in data:\n", " codes = data[\"v5.0_raw\"]\n", "else:\n", " raise ValueError(\"No codes found\")\n", "\n", "semantic_codes = torch.from_numpy(codes[:, 0]).long() # .cuda()\n", "# semantic_codes = semantic_codes[:3000]\n", "print(semantic_codes.shape)\n", "\n", "# load the vae\n", "vae_data_filepath = f\"s3://suno-data-uploads/studio/uploads/{item_id}_vae.npz\"\n", "vae_data = read_from_s3(vae_data_filepath, read_f=np.load)\n", "print(vae_data.keys())\n", "init_vae_latents = vae_data[\"vae_latents\"]\n", "print(init_vae_latents.shape)\n", "\n", "gen_seed = vae_data.get(\"seed\", None)\n", "print(gen_seed)\n", "\n", "text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{item_id}_hoot.json\")\n", "aligned_lyrics = json.loads(text_data)\n", "# print(aligned_lyrics)\n", "\n", "tags = \"pop\"\n", "\n", "lyrics = \"\"\n", "for elem in aligned_lyrics:\n", " if \"word\" in elem:\n", " lyrics += elem[\"word\"]\n", "# print(lyrics)\n", "mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{item_id}.mp3\"\n", "data = read_from_s3(s3_filepath, read_f=np.load)\n", "audio = Audio.from_s3(mp3_filepath, n_channels=2)\n", "audio.play()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "execution": { "iopub.execute_input": "2025-10-16T15:36:55.294879Z", "iopub.status.busy": "2025-10-16T15:36:55.294738Z", "iopub.status.idle": "2025-10-16T15:37:19.956167Z", "shell.execute_reply": "2025-10-16T15:37:19.955494Z", "shell.execute_reply.started": "2025-10-16T15:36:55.294863Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Prepared audio tags: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 5it [00:02, 1.70it/s]\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 5it [00:02, 1.70it/s]\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 5it [00:02, 1.71it/s]\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "print()\n", "# Map slider value from 0–1 to a target range\n", "def map_slider(slider_value, min_out, max_out):\n", " \"\"\"Map slider from 0-1 range to min_out-max_out range, handling None.\"\"\"\n", " if slider_value is None:\n", " return None\n", " slider_value = max(0, min(1, slider_value)) # clamp to 0–1\n", " return min_out + (max_out - min_out) * slider_value\n", "\n", "# === Set global slider values from frontend input (0.0 to 1.0) ===\n", "slider_strength = 0.95 # example: strength from user input\n", "slider_quality = 0.5 # example: quality\n", "slider_tone = 0.5 # example: tone\n", "slider_stereo = 0.5 # example: stereo\n", "\n", "# === Map to model ranges ===\n", "strength_value = map_slider(slider_strength, 1.0, 4.0) # cfg\n", "quality_value = map_slider(slider_quality, 12, 28) # ear score\n", "tone_value = map_slider(slider_tone, -2.0, 2.0) # spectral centroid\n", "stereo_value = map_slider(slider_stereo, -2.0, 2.0) # stereo balance\n", "\n", "# === Format audio tags ===\n", "audio_tags_parts = []\n", "if quality_value is not None and quality_value != 20.0:\n", " audio_tags_parts.append(f\"quality: {int(quality_value)}\")\n", "if tone_value is not None and tone_value != 0.0:\n", " audio_tags_parts.append(f\"spectral_centroid: {tone_value:.1f}\")\n", "if stereo_value is not None and stereo_value != 0.0:\n", " #if stereo_value <= -1.5:\n", " # audio_tags_parts.append(f\"mono\")\n", " #else:\n", " audio_tags_parts.append(f\"stereo_width: {stereo_value:.1f}\")\n", "\n", "audio_tags = \", \".join(audio_tags_parts) if audio_tags_parts else \"\"\n", "print(f\"Prepared audio tags: {audio_tags}\")\n", "\n", "# === Example of using values in model logic ===\n", "text_cfg_coef = 2.5 if strength_value is None else strength_value\n", "\n", "steps = 10\n", "text_cfg_coef = 2.0\n", "noise_ctx_level = 0.75\n", "noise_ctx_pad_len = 0\n", "codec_scale_factor = 0.4\n", "num_seeds = 2\n", "\n", "seeds = list(range(3))\n", "upsampled_audios = []\n", "for idx, seed in enumerate(seeds):\n", " gen_cfg = diffusion_gen.DiffusionGenerationConfig(\n", " steps=steps,\n", " lyrics=lyrics,\n", " tags=tags,\n", " text_cfg_coef=text_cfg_coef,\n", " ctx_cfg_coef=1.0,\n", " codec_scale_factor=codec_scale_factor,\n", " scale_ctx_vector=True,\n", " noise_ctx_level=noise_ctx_level,\n", " noise_ctx_pad_len=noise_ctx_pad_len,\n", " drop_semantic_tokens=False,\n", " seed=seed,\n", " rho=1.0,\n", " sigma_min=0.5,\n", " sigma_max=50.0,\n", " semantic_mask_ratio=0.0,\n", " )\n", "\n", " request = Request(\n", " id=\"dummy\",\n", " generation_config=gen_cfg,\n", " tokens=semantic_codes,\n", " input_tokens_finished=True,\n", " )\n", "\n", " result = diffusion_engine.run_request(request)\n", "\n", " vae_latents = []\n", " for vae_latent in result.vae_latents:\n", " #print(vae_latent.shape)\n", " mean = vae_latent.mean()\n", " std = vae_latent.std()\n", " vae_latents.append(vae_latent)\n", "\n", " vae_latents = torch.concat(vae_latents)\n", " upsampled_audio = decode_stream_to_full_audio(vae_latents)\n", " upsampled_audio.play()\n", " upsampled_audios.append(upsampled_audio)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "execution": { "iopub.execute_input": "2025-10-16T15:37:19.958613Z", "iopub.status.busy": "2025-10-16T15:37:19.958477Z", "iopub.status.idle": "2025-10-16T15:37:22.451959Z", "shell.execute_reply": "2025-10-16T15:37:22.451344Z", "shell.execute_reply.started": "2025-10-16T15:37:19.958598Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "s3://suno-data-uploads/studio/uploads/ae137e1f-7562-48f6-a11c-b125e645282c.npz\n", "torch.Size([1446])\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "gen_id = \"ae137e1f-7562-48f6-a11c-b125e645282c\"\n", "gen_id_audio = \"cb105e06-6e20-4acc-9d1c-e7c9b36b24e2\"\n", "\n", "s3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.npz\"\n", "mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id_audio}.mp3\"\n", "print(s3_filepath)\n", "data = read_from_s3(s3_filepath, read_f=np.load)\n", "audio = Audio.from_s3(mp3_filepath, n_channels=2)\n", "\n", "if \"v3.0_raw\" in data:\n", " codes = data[\"v3.0_raw\"]\n", "elif \"v3.5_raw\" in data:\n", " codes = data[\"v3.5_raw\"]\n", "elif \"v4.0_raw\" in data:\n", " codes = data[\"v4.0_raw\"]\n", "else:\n", " raise ValueError(\"No codes found\")\n", "\n", "semantic_codes = torch.from_numpy(codes[:, 0]).long() # .cuda()\n", "# semantic_codes = semantic_codes[:3000]\n", "print(semantic_codes.shape)\n", "\n", "text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{gen_id}_hoot.json\")\n", "aligned_lyrics = json.loads(text_data)\n", "# print(aligned_lyrics)\n", "# lyrics = \"\".join(w[\"word\"] for w in aligned_lyrics if \"word\" in w)\n", "lyrics = \"walking down the streets feeling so alive i've got my head in the clouds got a gleam in my eye every step i take it's like a brand new start no matter where i'm going i'll always find my part life is like a hard wire act we're dancing in the sky no need to worry no need to ask why with a little bit of courage we can chase our dreams no matter what comes our way we'll always be a team we're unstoppable yeah\"\n", "audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "execution": { "iopub.execute_input": "2025-10-16T15:37:22.452751Z", "iopub.status.busy": "2025-10-16T15:37:22.452591Z", "iopub.status.idle": "2025-10-16T15:37:33.629861Z", "shell.execute_reply": "2025-10-16T15:37:33.629245Z", "shell.execute_reply.started": "2025-10-16T15:37:22.452734Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 2it [00:01, 1.84it/s]\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 2it [00:01, 1.85it/s]\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 2it [00:01, 1.85it/s]\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "upsampled_audios = []\n", "for idx, seed in enumerate(seeds):\n", " gen_cfg = diffusion_gen.DiffusionGenerationConfig(\n", " steps=steps,\n", " lyrics=lyrics,\n", " tags=tags,\n", " text_cfg_coef=text_cfg_coef,\n", " ctx_cfg_coef=1.0,\n", " codec_scale_factor=codec_scale_factor,\n", " scale_ctx_vector=True,\n", " noise_ctx_level=noise_ctx_level,\n", " noise_ctx_pad_len=noise_ctx_pad_len,\n", " drop_semantic_tokens=False,\n", " seed=seed,\n", " rho=1.0,\n", " sigma_min=0.5,\n", " sigma_max=50.0,\n", " semantic_mask_ratio=0.0,\n", " )\n", "\n", " request = Request(\n", " id=\"dummy\",\n", " generation_config=gen_cfg,\n", " tokens=semantic_codes,\n", " input_tokens_finished=True,\n", " )\n", "\n", " result = diffusion_engine.run_request(request)\n", "\n", " vae_latents = []\n", " for vae_latent in result.vae_latents:\n", " #print(vae_latent.shape)\n", " mean = vae_latent.mean()\n", " std = vae_latent.std()\n", " vae_latents.append(vae_latent)\n", "\n", " vae_latents = torch.concat(vae_latents)\n", " upsampled_audio = decode_stream_to_full_audio(vae_latents)\n", " upsampled_audio.play()\n", " upsampled_audios.append(upsampled_audio)" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "execution": { "iopub.execute_input": "2025-11-14T04:22:21.198107Z", "iopub.status.busy": "2025-11-14T04:22:21.197759Z", "iopub.status.idle": "2025-11-14T04:22:21.200860Z", "shell.execute_reply": "2025-11-14T04:22:21.200392Z", "shell.execute_reply.started": "2025-11-14T04:22:21.198090Z" } }, "outputs": [], "source": [ "# data_path = \"/app2/suno/data/semantic_code/sft/YK3SZUbQcdI.npz\"\n", "# lyrics = \"\"\"\n", "# Me enamor\\u00e9\\nCon esa tibieza que tiene toda tu piel\\nCon la furia loca que s\\u00f3lo vive el amor\\nEse amor que es parte\n", "# de todo, todo mi ser\\n\\nMe enamor\\u00e9\\nComo s\\u00f3lo un ni\\u00f1o peque\\u00f1o sue\\u00f1a el amor\\nComo una paloma que vue\n", "# la en busca del sol\\nPara ser un \\u00e1ngel que implora por nuestro amor\\n\\nMe enamor\\u00e9\\nDel clavel del aire, de tus caric\n", "# ias, de tu sonrisa\\nPor eso quiero darte mi vida\\nAy, amor\\nPor ti me juego el alma\\nSin ti ya no hay ma\\u00f1ana\\nSin ti, mi \n", "# amor\\nMi vida no vale nada\\n\\nAy, amor\\nPor ti me juego el alma\\nSin ti ya no hay ma\\u00f1ana\\nSin ti, mi amor\\nMi vida no val\n", "# e nada\\n\\nMe enamor\\u00e9\\nComo s\\u00f3lo un ni\\u00f1o peque\\u00f1o sue\\u00f1a el amor\\nComo una paloma que vuela en busca del\n", "# sol\\nPara ser un \\u00e1ngel que implora por nuestro amor\\n\\nMe enamor\\u00e9\\nDel clavel del aire, de tus caricias, de tu sonr\n", "# isa\\nPor eso quiero darte mi vida\\n\\nAy, amor\\nPor ti me juego el alma\\nSin ti ya no hay ma\\u00f1ana\\nSin ti, mi amor\\nMi vida\n", "# no vale nada\\n\\n\\nAy, amor\\nPor ti me juego el alma\\nSin ti ya no hay ma\\u00f1ana\\nSin ti, mi amor\\nMi vida no vale nada\\n\\nS\n", "# in ti, mi amor\\nMi vida no vale nada\\nNo vale nada\n", "# \"\"\"\n", "# codes = np.load(data_path)[\"codes\"]\n", "\n", "# semantic_codes = torch.from_numpy(codes[:, 0]).long() # .cuda()\n", "# # semantic_codes = semantic_codes[:3000]\n", "# print(semantic_codes.shape)\n", "# audio = Audio.from_file(\"/app2/suno/data/raw_audio_opus_v0/YK3SZUbQcdI.opus\", n_channels=2)\n", "# audio.play()" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "execution": { "iopub.execute_input": "2025-11-14T04:22:09.506785Z", "iopub.status.busy": "2025-11-14T04:22:09.506461Z", "iopub.status.idle": "2025-11-14T04:22:09.509968Z", "shell.execute_reply": "2025-11-14T04:22:09.509513Z", "shell.execute_reply.started": "2025-11-14T04:22:09.506766Z" } }, "outputs": [], "source": [ "# data_path = \"/app2/suno/data/semantic_code/crow/b7b60ccb-41e5-41b6-b5cf-06d7ad74738b.npz\"\n", "# lyrics = \"\"\"\n", "# [Verse 1 – detached, smooth]\n", "# Scroll past my life, you’ll see a curated frame\n", "# I’m floating through the noise, I don’t need your name\n", "# Your eyes are on my pixels, but my soul’s unseen\n", "# I’m living in the quiet, in places in between\n", "\n", "# [Pre-Chorus – detached confidence]\n", "# You can trace my steps, but I’m already gone\n", "# I move without a shadow, I don’t need your dawn\n", "\n", "# [Chorus – midtempo, hypnotic groove]\n", "# Unfollow me, I don’t need the crowd\n", "# I’m invisible but shining, no screen to be proud\n", "# You can’t cancel someone who doesn’t want to be known\n", "# I breathe in solitude, I thrive alone\n", "# Unfollow me, I’m dancing in the grey\n", "# I’m more than what your eyes can display\n", "# I’m untouchable, I’m free, I don’t have to comply\n", "# I live outside the lens, I let my spirit fly\n", "\n", "# [Verse 2 – smooth, playful]\n", "# Every post, every story, just a fragment of my hue\n", "# I’m the silence in the chaos, the shadow passing through\n", "# I don’t need applause, I don’t need a name\n", "# I’m alive in my own rhythm, I’ll never play your game\n", "\n", "# [Pre-Chorus 2 – confident]\n", "# Trace all you want, but you’ll never see\n", "# The hidden corners where I choose to be\n", "\n", "# [Chorus – repeat, layered harmonies]\n", "# Unfollow me, I don’t need the crowd\n", "# I’m invisible but shining, no screen to be proud\n", "# You can’t cancel someone who doesn’t want to be known\n", "# I breathe in solitude, I thrive alone\n", "# Unfollow me, I’m dancing in the grey\n", "# I’m more than what your eyes can display\n", "# I’m untouchable, I’m free, I don’t have to comply\n", "# I live outside the lens, I let my spirit fly\n", "\n", "# [Bridge – airy, reflective]\n", "# No notifications, no likes, no sound\n", "# Just me, the quiet, where my truth is found\n", "# I slip through the algorithms, I dissolve from the stream\n", "# I exist in the in-between, I live my own dream\n", "\n", "# [Chorus – final, cinematic]\n", "# Unfollow me, I don’t need the crowd\n", "# I’m invisible but shining, no screen to be proud\n", "# You can’t cancel someone who doesn’t want to be known\n", "# I breathe in solitude, I thrive alone\n", "# Unfollow me, I’m dancing in the grey\n", "# I’m more than what your eyes can display\n", "# I’m untouchable, I’m free, I don’t have to comply\n", "# I live outside the lens, I let my spirit fly\n", "\n", "# [outro]\n", "# \"\"\"\n", "# codes = np.load(data_path)[\"codes\"]\n", "\n", "# semantic_codes = torch.from_numpy(codes[:, 0]).long() # .cuda()\n", "# # semantic_codes = semantic_codes[:3000]\n", "# print(semantic_codes.shape)\n", "# audio = Audio.from_file(\"/app2/suno/data/dpo/audios/crow/b7b60ccb-41e5-41b6-b5cf-06d7ad74738b.m4a\", n_channels=2)\n", "# audio.play()" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "execution": { "iopub.execute_input": "2025-11-15T01:10:58.113527Z", "iopub.status.busy": "2025-11-15T01:10:58.113102Z", "iopub.status.idle": "2025-11-15T01:10:58.144623Z", "shell.execute_reply": "2025-11-15T01:10:58.143885Z", "shell.execute_reply.started": "2025-11-15T01:10:58.113508Z" } }, "outputs": [ { "ename": "FileNotFoundError", "evalue": "[Errno 2] No such file or directory: '/app2/suno/data/semantic_code/crow/0f9b253b-605f-4e15-8abb-e1588fbf0e5d.npz'", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[31], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m data_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/app2/suno/data/semantic_code/crow/0f9b253b-605f-4e15-8abb-e1588fbf0e5d.npz\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m----> 2\u001b[0m codes \u001b[38;5;241m=\u001b[39m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata_path\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcodes\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n", "File \u001b[0;32m~/anaconda3/envs/suno_env_dev/lib/python3.10/site-packages/numpy/lib/npyio.py:405\u001b[0m, in \u001b[0;36mload\u001b[0;34m(file, mmap_mode, allow_pickle, fix_imports, encoding, max_header_size)\u001b[0m\n\u001b[1;32m 403\u001b[0m own_fid \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 404\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 405\u001b[0m fid \u001b[38;5;241m=\u001b[39m stack\u001b[38;5;241m.\u001b[39menter_context(\u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mos_fspath\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrb\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m)\n\u001b[1;32m 406\u001b[0m own_fid \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 408\u001b[0m \u001b[38;5;66;03m# Code to distinguish from NumPy binary files and pickles.\u001b[39;00m\n", "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/app2/suno/data/semantic_code/crow/0f9b253b-605f-4e15-8abb-e1588fbf0e5d.npz'" ] } ], "source": [ "data_path = \"/app2/suno/data/semantic_code/crow/0f9b253b-605f-4e15-8abb-e1588fbf0e5d.npz\"\n", "codes = np.load(data_path)[\"codes\"]" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "execution": { "iopub.execute_input": "2025-11-15T01:12:22.386311Z", "iopub.status.busy": "2025-11-15T01:12:22.385913Z", "iopub.status.idle": "2025-11-15T01:12:25.984541Z", "shell.execute_reply": "2025-11-15T01:12:25.983848Z", "shell.execute_reply.started": "2025-11-15T01:12:22.386291Z" } }, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# audio = Audio.from_file(\"/app2/suno/data/dpo/audios/crow/0f9b253b-605f-4e15-8abb-e1588fbf0e5d.opus\", n_channels=2)\n", "# audio.play()" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "execution": { "iopub.execute_input": "2025-11-14T04:22:05.877732Z", "iopub.status.busy": "2025-11-14T04:22:05.877395Z", "iopub.status.idle": "2025-11-14T04:22:05.880437Z", "shell.execute_reply": "2025-11-14T04:22:05.879985Z", "shell.execute_reply.started": "2025-11-14T04:22:05.877714Z" } }, "outputs": [], "source": [ "# upsampled_audios = []\n", "# for idx, seed in enumerate(seeds):\n", "# gen_cfg = diffusion_gen.DiffusionGenerationConfig(\n", "# steps=steps,\n", "# lyrics=lyrics,\n", "# tags=tags,\n", "# text_cfg_coef=text_cfg_coef,\n", "# ctx_cfg_coef=1.0,\n", "# codec_scale_factor=codec_scale_factor,\n", "# scale_ctx_vector=True,\n", "# noise_ctx_level=noise_ctx_level,\n", "# noise_ctx_pad_len=noise_ctx_pad_len,\n", "# drop_semantic_tokens=False,\n", "# seed=seed,\n", "# rho=1.0,\n", "# sigma_min=0.5,\n", "# sigma_max=50.0,\n", "# semantic_mask_ratio=0.0,\n", "# )\n", "\n", "# request = Request(\n", "# id=\"dummy\",\n", "# generation_config=gen_cfg,\n", "# tokens=semantic_codes,\n", "# input_tokens_finished=True,\n", "# )\n", "\n", "# result = diffusion_engine.run_request(request)\n", "\n", "# vae_latents = []\n", "# for vae_latent in result.vae_latents:\n", "# #print(vae_latent.shape)\n", "# mean = vae_latent.mean()\n", "# std = vae_latent.std()\n", "# vae_latents.append(vae_latent)\n", "\n", "# vae_latents = torch.concat(vae_latents)\n", "# upsampled_audio = decode_stream_to_full_audio(vae_latents)\n", "# upsampled_audio.play()\n", "# upsampled_audios.append(upsampled_audio)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 8, "metadata": { "execution": { "iopub.execute_input": "2025-10-16T15:37:33.630621Z", "iopub.status.busy": "2025-10-16T15:37:33.630467Z", "iopub.status.idle": "2025-10-16T15:37:33.934725Z", "shell.execute_reply": "2025-10-16T15:37:33.934101Z", "shell.execute_reply.started": "2025-10-16T15:37:33.630605Z" } }, "outputs": [ { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mBREAK\u001b[49m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "BREAK" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Infill" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-10-16T15:37:33.935109Z", "iopub.status.idle": "2025-10-16T15:37:33.935291Z", "shell.execute_reply": "2025-10-16T15:37:33.935203Z", "shell.execute_reply.started": "2025-10-16T15:37:33.935194Z" } }, "outputs": [], "source": [ "list(vae_data.keys())\n", "ctx_vae_latents = vae_data[\"vae_latents\"]\n", "ctx_vae_latents.shape\n", "print(ctx_vae_latents.shape)\n", "decode_stream_to_full_audio(ctx_vae_latents).play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-10-16T15:37:33.935606Z", "iopub.status.idle": "2025-10-16T15:37:33.935761Z", "shell.execute_reply": "2025-10-16T15:37:33.935689Z", "shell.execute_reply.started": "2025-10-16T15:37:33.935681Z" } }, "outputs": [], "source": [ "lyrics_30s = \"\"\"\n", "[Instrumental]\n", "\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-10-16T15:37:33.936317Z", "iopub.status.idle": "2025-10-16T15:37:33.936480Z", "shell.execute_reply": "2025-10-16T15:37:33.936400Z", "shell.execute_reply.started": "2025-10-16T15:37:33.936392Z" } }, "outputs": [], "source": [ "SEMANTIC_RATE_HZ = 25\n", "\n", "start_s = 10\n", "end_s = 20\n", "infill_dur_s = end_s - start_s\n", "context_dur_s = 30.0 - infill_dur_s\n", "\n", "# count total tokens of the context + infill\n", "infill_tokens = int(np.ceil(infill_dur_s * SEMANTIC_RATE_HZ))\n", "print(infill_dur_s)\n", "print(infill_tokens)\n", "\n", "prefix_start_s = start_s - (context_dur_s // 2)\n", "prefix_end_s = start_s\n", "suffix_start_s = end_s\n", "suffix_end_s = end_s + (context_dur_s // 2)\n", "print(\"prefix_start_s\", prefix_start_s)\n", "print(\"suffix_end_s\", suffix_end_s)\n", "\n", "prefix_start_idx = int(np.ceil(prefix_start_s * SEMANTIC_RATE_HZ))\n", "prefix_end_idx = int(np.ceil(prefix_end_s * SEMANTIC_RATE_HZ))\n", "prefix_tokens = prefix_end_idx - prefix_start_idx\n", "\n", "print(\"prefix_start_idx\", prefix_start_idx)\n", "print(\"prefix_end_idx\", prefix_end_idx)\n", "\n", "suffix_start_idx = int(np.ceil(suffix_start_s * SEMANTIC_RATE_HZ))\n", "suffix_end_idx = int(np.ceil(suffix_end_s * SEMANTIC_RATE_HZ))\n", "suffix_tokens = suffix_end_idx - suffix_start_idx\n", "\n", "print(\"suffix_start_idx\", suffix_start_idx)\n", "print(\"suffix_end_idx\", suffix_end_idx)\n", "\n", "total_context_tokens = prefix_tokens + suffix_tokens\n", "print(\"total_context_tokens\", total_context_tokens)\n", "\n", "total_tokens = total_context_tokens + infill_tokens\n", "if total_tokens < 750: \n", " # add more context to the right side\n", " context_to_add = 750 - total_tokens\n", " print(context_to_add)\n", " suffix_end_idx += context_to_add\n", "\n", "\n", "infill_prefix_latents = torch.tensor(ctx_vae_latents[prefix_start_idx: prefix_end_idx, :])\n", "infill_suffix_latents = torch.tensor(ctx_vae_latents[suffix_start_idx: suffix_end_idx, :])\n", "print(infill_prefix_latents.shape)\n", "print(infill_suffix_latents.shape)\n", "\n", "\n", "gen_cfg = diffusion_gen.DiffusionGenerationConfig(\n", " audio=None,\n", " steps=16,\n", " lyrics=lyrics_30s,\n", " tags=tags,\n", " text_cfg_coef=2.0,\n", " ctx_cfg_coef=1.0,\n", " codec_scale_factor=0.4,\n", " scale_ctx_vector=True,\n", " infill_prefix_latents=infill_prefix_latents,\n", " infill_suffix_latents=infill_suffix_latents,\n", " noise_ctx_level=0.0,\n", " drop_semantic_tokens=False,\n", " semantic_skip_factor=1,\n", " semantic_mask_ratio=1.0,\n", " is_diff_infill=True,\n", " #objective=\"rectified_flow\",\n", ")\n", "input_tokens = codes[prefix_start_idx : suffix_end_idx, 0]\n", "print(input_tokens.shape)\n", "request = Request(\n", " id=\"dummy\",\n", " generation_config=gen_cfg,\n", " tokens=input_tokens,\n", " input_tokens_finished=True,\n", ")\n", "result = diffusion_engine.run_request(request)\n", "\n", "vae_latents = torch.concat(result.vae_latents)\n", "print(vae_latents.shape)\n", "decode_stream_to_full_audio(vae_latents).play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-10-16T15:37:33.936750Z", "iopub.status.idle": "2025-10-16T15:37:33.936892Z", "shell.execute_reply": "2025-10-16T15:37:33.936824Z", "shell.execute_reply.started": "2025-10-16T15:37:33.936817Z" } }, "outputs": [], "source": [ "decode_stream_to_full_audio(ctx_vae_latents).play()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# MODEL AVERAGE" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "execution": { "iopub.execute_input": "2025-10-16T23:22:55.312499Z", "iopub.status.busy": "2025-10-16T23:22:55.311891Z", "iopub.status.idle": "2025-10-16T23:22:55.315171Z", "shell.execute_reply": "2025-10-16T23:22:55.314675Z", "shell.execute_reply.started": "2025-10-16T23:22:55.312478Z" } }, "outputs": [], "source": [ "dit_model_filepath = \"/app2/suno/checkpoints/2025-07-14_15-27-20_s7143/last_ckpt_infer.pt\" # prod v4_39\n", "dit_model_filepath_base = \"/app/suno/data/dpo/models/diff_v2_2b_2mil_ft_infill_20250421_v2.pt\"\n", "dit_model_with_vox = \"/app2/suno/modal/models/tony/tmp/diff/v2_diff_vox_0920.pt\"" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "execution": { "iopub.execute_input": "2025-10-16T23:30:35.495602Z", "iopub.status.busy": "2025-10-16T23:30:35.495176Z", "iopub.status.idle": "2025-10-16T23:31:00.363453Z", "shell.execute_reply": "2025-10-16T23:31:00.362757Z", "shell.execute_reply.started": "2025-10-16T23:30:35.495583Z" } }, "outputs": [], "source": [ "# # # create a model mix\n", "mix_ratio = 1.1 # how much of raw to use\n", "import torch\n", "from collections import OrderedDict\n", "# cpd = \"/app/suno/checkpoints/\"\n", "sd1 = torch.load(dit_model_filepath_base, map_location=\"cpu\", weights_only=True)\n", "sd2 = torch.load(dit_model_filepath, map_location=\"cpu\", weights_only=True)\n", "sd3 = torch.load(dit_model_with_vox, map_location=\"cpu\", weights_only=True)\n", "assert set(sd1.keys()) == set(sd2.keys())\n", "sd1[\"model\"] = {k: sd2[\"model\"][k] * mix_ratio + sd1[\"model\"][k] * (1 - mix_ratio) for k in sd1[\"model\"].keys()}\n", "torch.save(sd1, \"/home/tony/Data/checkpoints/diff/diff_v2_v4_39_25pct_extrapolate_2.pt\")\n", "\n", "# # intersection_keys = set(sd1[\"model\"].keys()).intersection(sd3[\"model\"].keys())\n", "# # sd3[\"model\"] = {k: sd3[\"model\"][k] * mix_ratio + (sd2[\"model\"][k] * mix_ratio - sd1[\"model\"][k]) for k in intersection_keys}\n", "# # # torch.save(sd3, \"/home/tony/Data/checkpoints/diff/v2_diff_vox_0920_25pct_extrapolate.pt\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" } }, "nbformat": 4, "nbformat_minor": 4 }