{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "0e09c52f-da2a-4a21-b659-b58383cba6f4", "metadata": {}, "outputs": [], "source": [ "# !nvidia-smi\n", "# !echo $HOSTNAME\n", "\n", "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"5\"\n", "\n", "import sys\n", "#sys.path.insert(0, \"/home/georg/code/neon/sunoDiff/\")\n", "sys.path.insert(0, \"/home/tony/Work/neon/sunoDiff/\")" ] }, { "cell_type": "code", "execution_count": null, "id": "c046f7b4", "metadata": {}, "outputs": [], "source": [ "import torch\n", "from suno_utils.audio import Audio\n", "from generation import preload_models, generate, _retrieve_models\n", "CKPT_DIR = \"/home/christian/code/neon/stable-audio-tools/harmonai_train/\"\n", "_ = preload_models(\n", " tokenizer_filepath=\"/app/suno/data/dpo/models/tokenizer_60k.json\",\n", " semantic_model_filepath=\"/app/suno/data/dpo/models/mert_25.pt\",\n", " semantic_clusters_filepath=\"/app/suno/data/dpo/models/mert_25_2x4k.npy\",\n", " weights_precision=torch.bfloat16,\n", " compile=True,\n", " ### 25hz ###\n", " # codec_filepath=\"/home/georg/notebooks/gpu_nb/tmp/25hz_vae_peaq_kl_0.005.pth\",\n", " codec_filepath=\"/app/suno/data/dpo/models/dac_vae_tuned_25hz.pth\",\n", " ### 100hz ###\n", " # codec_filepath=\"/home/georg/notebooks/gpu_nb/tmp/100hz_vae_peaq_kl_0.005.pth\",\n", " # 30s (overlap newest)\n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-10-17_04-15-19_s4176/last_ckpt_infer.pt\",\n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-10-11_16-53-57_s8050/last_ckpt_infer.pt\", # default\n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-10-27_00-05-50_s2159/last_ckpt_infer.pt\", # artificial data dpo\n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-10-26_04-39-23_s121/last_ckpt_infer.pt\", # default check point\n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-11-25_03-10-14_s4910/last_ckpt_infer.pt\", # quality model\n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-11-23_14-22-16_s994/last_ckpt_infer.pt\", # \n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-11-30_09-50-23_s5271/step_3000_infer.pt\", # t1\n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-12-10_19-53-58_s7792/step_3000_infer.pt\", # t3\n", " # dit_model_filepath=\"/app/suno/checkpoints/2025-02-20_21-37-57_s6971/step_3000_infer.pt\", # t5\n", " # dit_model_filepath=\"/app/suno/checkpoints/2025-03-04_05-30-37_s7677/step_30000_infer.pt\", # step_5000_infer.pt\",\n", " # dit_model_filepath=\"/app/suno/checkpoints/2025-04-10_04-13-39_s12/step_9000_infer.pt\", # t6_cs\n", " dit_model_filepath=\"/app2/suno/checkpoints/2025-07-14_15-27-20_s7143/last_ckpt_infer.pt\", # t6_cs\n", " codec_scale_factor=0.4\n", ")\n", "models = _retrieve_models()\n", "model_duration_s = 30\n", "if models[\"dit_model\"].ctx_len is not None:\n", " model_duration_s = 6 * 60\n", "else:\n", " model_duration_s = models[\"dit_model\"].block_size // models[\"dit_model\"].io_hz\n", "duration_s = 2*60 if model_duration_s >= 2*60 else 30" ] }, { "cell_type": "code", "execution_count": null, "id": "bac651fe-cfda-49f4-a4f2-b8878b89b128", "metadata": {}, "outputs": [], "source": [ "# from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "# from suno_utils.tasks.dac_2c_12cb import (\n", "# encode as dac_codec_encode,\n", "# decode_stream_to_full_audio as dac_codec_decode,\n", "# decode as dac_decode\n", "# )\n", "# _ = preload_codec_models(\"/app/suno/data/dpo/models/dac_2c_25x12.pt\", device=\"cuda\")\n", "\n", "# from suno_utils.tasks.dac_vae_fixed_25hz import preload_models as preload_codec_models\n", "# from suno_utils.tasks.dac_vae_fixed_25hz import (\n", "# encode as dac_codec_encode,\n", "# decode_stream_to_full_audio as dac_codec_decode,\n", "# decode as dac_decode\n", "# )\n", "# _ = preload_codec_models(\"/app/suno/data/dpo/models/dac_vae_tuned_25hz.pth\", device=\"cuda\")" ] }, { "cell_type": "code", "execution_count": null, "id": "cf696991", "metadata": { "scrolled": true }, "outputs": [], "source": [ "import json\n", "import numpy as np\n", "from suno_utils.utils.s3 import read_from_s3\n", "\n", "#gen_id = \"99bffa17-7e59-47b4-a048-5528cbda05d5\" # sister\n", "#gen_id = \"081d73c4-7805-4212-9c80-8db1137ca3c4\" # friends\n", "#gen_id = \"562f762d-6ced-4080-9af1-910ee3d0a5dc\" # something real\n", "#gen_id = \"23c15c62-494d-422d-8a60-8b0454044322\" # rubber duck\n", "#gen_id = \"4b140a9e-964b-422c-85b5-5861ad1a9d38\" # once\n", "#gen_id = \"7b214347-fa38-4e9b-96f4-f7ec65adea45\" # rock n roll\n", "# gen_id = \"a5e2198a-f352-4abb-9a24-7f81b143ded3\" # stone\n", "#gen_id = \"7f5e7819-b7a1-449d-a76c-9c38950c4274\" # canada\n", "#gen_id = \"27135940-957f-4152-9e40-72e5c10a46bd\" # drunk\n", "#gen_id = \"caf1d7ae-ce81-4c75-b737-fd41e47a38ba\" # linger\n", "#gen_id = \"ac5aa29a-f18f-406f-a904-c47493f1beec\" # linger vox\n", "#gen_id = \"b04719c3-012a-4a4a-9b02-3cb2875d1744\" # courtney\n", "#gen_id = \"7407c7b3-8aa2-4f0a-82e8-fa19933f6e94\" # opera\n", "#gen_id = \"5b9624ef-b416-4095-a5f3-4945f09bba86\" # butterflies\n", "#gen_id = \"e757aa19-5c33-4945-aaa1-1519b7890f0b\" # bubblegum\n", "#gen_id = \"7f774078-1672-4858-a37f-acad373c5a84\" # golden\n", "# gen_id = \"1cee79db-a0ee-44e4-9646-d312cd620997\" # chinese\n", "# gen_id = \"3b1830d7-5553-4785-be0b-ffeadf7756d8\" # debug\n", "# gen_id_audio = gen_id\n", "# gen_id = \"6845dc51-71b1-491c-b0be-1f6c691a2f42\"\n", "\n", "gen_id = \"ae137e1f-7562-48f6-a11c-b125e645282c\"\n", "gen_id_audio = \"cb105e06-6e20-4acc-9d1c-e7c9b36b24e2\"\n", "# gen_id_audio = gen_id\n", "\n", "s3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.npz\"\n", "mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id_audio}.mp3\"\n", "print(s3_filepath)\n", "data = read_from_s3(s3_filepath, read_f=np.load)\n", "\n", "audio = Audio.from_s3(mp3_filepath, n_channels=2)\n", "\n", "if \"v3.0_raw\" in data:\n", " codes = data[\"v3.0_raw\"]\n", "elif \"v3.5_raw\" in data:\n", " codes = data[\"v3.5_raw\"]\n", "elif \"v4.0_raw\" in data:\n", " codes = data[\"v4.0_raw\"]\n", "else:\n", " raise ValueError(\"No codes found\")\n", "\n", "text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{gen_id}_hoot.json\")\n", "aligned_lyrics = json.loads(text_data)\n", "# print(aligned_lyrics)\n", "# lyrics = \"\".join(w[\"word\"] for w in aligned_lyrics if \"word\" in w)\n", "lyrics = \"walking down the streets feeling so alive i've got my head in the clouds got a gleam in my eye every step i take it's like a brand new start no matter where i'm going i'll always find my part life is like a hard wire act we're dancing in the sky no need to worry no need to ask why with a little bit of courage we can chase our dreams no matter what comes our way we'll always be a team we're unstoppable yeah\"\n", "audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "id": "a46034ef-098c-4ef6-bc38-20994af79abb", "metadata": {}, "outputs": [], "source": [ "# codes = np.load(\"/app2/suno/data/dpo/sft/semantic_codes_v11/koHo5-0DVlI.npz\")\n", "# codes = codes[\"codes\"].reshape(-1, 1)\n", "# print(codes.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "63a83ce4-b913-4838-8744-a0185bd7f75c", "metadata": {}, "outputs": [], "source": [ "# import numpy as np\n", "# from suno_utils.utils.text import read_jsonl, read_json\n", "# N_TOKENS_AUDIO = 25 * 8 * 60\n", "\n", "# mm = np.memmap(os.path.join(\"/app2/suno/data/dpo/sft/\", f\"data_sft.bin\"), dtype=np.uint16, mode=\"r\")\n", "# test_metas = read_jsonl(os.path.join(\"/app2/suno/data/dpo/sft/\", f\"meta_sft.jsonl\"))\n", "# test_info = read_json(os.path.join(\"/app2/suno/data/dpo/sft/\", f\"info_sft.json\"))\n", "# mm = mm.reshape(-1, N_TOKENS_AUDIO, 1)\n", "# assert len(mm) == len(test_metas)\n", "# assert mm[:100, :, 0].min() >= 0\n", "# assert mm[:100, :, 0].max() <= 4000\n", "# codes = mm[0, :, 0]\n", "# codes = codes[codes != 4000].reshape(-1, 1)\n", "# print(codes.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "9b6e8953", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "np.random.seed(42)\n", "# seeds = np.random.randint(0, 2**32 - 1, 3)\n", "seeds = list(range(1))\n", "print(seeds)\n", "sem_codes = torch.tensor(codes[:, 0]).to(torch.long)" ] }, { "cell_type": "code", "execution_count": null, "id": "c0c34cd3", "metadata": {}, "outputs": [], "source": [ "steps = 8\n", "text_cfg_coef = 2.0\n", "ctx_cfg_coef = 1.0\n", "\n", "audios = []\n", "loudness = []\n", "for n in seeds:\n", " pred_audio = generate(\n", " sem_codes, \n", " lyrics=lyrics,\n", " # history_lyrics=history_text,\n", " # tags='shoegaze, ethereal, art, ego',\n", " tags='orchestral, orchestra, classical, epic movie',\n", " text_cfg_coef=text_cfg_coef, \n", " ctx_cfg_coef=ctx_cfg_coef,\n", " steps=steps,\n", " seed=n,\n", " semantic_skip_factor=1,\n", " downscale_ctx_vector=False,\n", " noise_ctx_vector=0.5,\n", " noise_ctx_vector_pad_size=25,\n", " ) #.normalize_volume()\n", " audios.append(pred_audio)\n", " pred_audio.play()\n", " loudness.append(pred_audio.loudness)\n", "# break" ] }, { "cell_type": "code", "execution_count": null, "id": "44c7002d-eac1-4150-95a1-0be1575f37e7", "metadata": {}, "outputs": [], "source": [ "# 2024-10-26_04-39-23_s121 -- default\n", "# -15.728760602829365\n", "# 2024-11-30_09-50-23_s5271 -- t1\n", "# -13.758060684048314\n", "# 2024-12-05_03-42-43_s4538 -- t2\n", "# -14.186474583001399 -- no cfg -14.392693014792687\n", "# 2024-12-10_19-53-58_s7792 -- t3\n", "# -13.905701024133066\n", "# 2025-01-26_02-13-54_s2704 -- t4\n", "# -14.534069602355155 3000 ckpt -- no cfg -14.754058349827892\n", "# 2025-02-20_21-37-57_s6971 -- t5\n", "# -13.999082847979414 --3000 ckpt -- no cfg -14.170253265724998\n", "# 2025-03-04_05-30-37_s7677 -- t6\n", "# -13.294572563705719 -- 3000 ckpt\n", "# -12.95814316018512 -- 6000 ckpt\n", "# -12.775032448273599 -- 9000 ckpt\n", "# -12.1183249055171 -- 18000 ckpt\n", "# -12.137043287018688 -- 30000 ckpt -- no cfg -12.303569310305011\n", "# 2025-03-23_01-14-02_s1142\n", "# -10.42984739347292 -- 3000 ckpt immediately...\n", "# 2025-03-23_02-32-06_s2927\n", "# -10.100477842758405 -- 3000 ckpt immediately...\n", "# 2025-03-23_06-25-25_s7193\n", "# -10.315832152117018 -- no cfg -10.42959817423211 -- 3000 ckpt immediately...\n", "# 2025-03-23_13-41-41_s6800 3000 ckpt\n", "# -10.96124472725545\n", "# 2025-03-23_17-34-28_s7038 3000 ckpt\n", "# -9.455910982026754\n", "# -9.487254994786863 9000 ckpt\n", "print(f\"all {loudness}, \\n mean: {round(np.mean(loudness), 3)}, \\n median: {round(np.median(loudness), 3)}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "b8c54957-d67d-4add-9538-c1a05dd6d0b8", "metadata": {}, "outputs": [], "source": [ "BREAK" ] }, { "cell_type": "code", "execution_count": null, "id": "f6d3d5f3-fcfa-4b95-af01-98e6c01d8f86", "metadata": {}, "outputs": [], "source": [ "# for i in ([0, 1, 2]):\n", "# ref_audio = Audio.from_file(f\"/home/tony/Work/tony/audios/walking_down_diff_ft_1_ref_{i}.wav\")\n", "# ref_audio.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "d1507be8-e2f2-40d1-9542-97510d2d29bb", "metadata": {}, "outputs": [], "source": [ "seeds = list(range(3))" ] }, { "cell_type": "code", "execution_count": null, "id": "1e2f7d0b-37d5-485e-a153-a53e5e7b4c01", "metadata": { "scrolled": true }, "outputs": [], "source": [ "gen_id = \"3b1830d7-5553-4785-be0b-ffeadf7756d8\" # debug\n", "# gen_id = \"5d95fbf0-7072-439e-aa58-ebb8989d49c9\"\n", "gen_id_audio = gen_id\n", "\n", "s3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.npz\"\n", "mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id_audio}.mp3\"\n", "print(s3_filepath)\n", "data = read_from_s3(s3_filepath, read_f=np.load)\n", "\n", "audio = Audio.from_s3(mp3_filepath, n_channels=2)\n", "\n", "if \"v3.0_raw\" in data:\n", " codes = data[\"v3.0_raw\"]\n", "elif \"v3.5_raw\" in data:\n", " codes = data[\"v3.5_raw\"]\n", "elif \"v4.0_raw\" in data:\n", " codes = data[\"v4.0_raw\"]\n", "else:\n", " raise ValueError(\"No codes found\")\n", "\n", "text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{gen_id}_hoot.json\")\n", "aligned_lyrics = json.loads(text_data)\n", "# print(aligned_lyrics)\n", "lyrics = \"\".join(w[\"word\"] for w in aligned_lyrics if \"word\" in w)\n", "audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "id": "00e3d45c", "metadata": {}, "outputs": [], "source": [ "sem_codes = torch.tensor(codes[:, 0]).to(torch.long)\n", "audios = []\n", "for n in seeds:\n", " pred_audio = generate(\n", " sem_codes, \n", " lyrics=lyrics,\n", " # history_lyrics=history_text,\n", " # tags='shoegaze, ethereal, art, ego',\n", " tags='80s, new wave, female power',\n", " text_cfg_coef=text_cfg_coef, \n", " ctx_cfg_coef=ctx_cfg_coef,\n", " steps=steps,\n", " seed=n,\n", " semantic_skip_factor=1,\n", " downscale_ctx_vector=False,\n", " noise_ctx_vector=0.0,\n", " noise_ctx_vector_pad_size=25,\n", " ).normalize_volume()\n", " audios.append(pred_audio)\n", " pred_audio.play()\n", "# break" ] }, { "cell_type": "code", "execution_count": null, "id": "19afba82-4fd6-429a-9be1-bb31f9e00523", "metadata": {}, "outputs": [], "source": [ "gen_id = \"ebb4ce1d-8f79-4a12-8aa8-858bcc65b44a\" # debug\n", "gen_id_audio = gen_id\n", "\n", "s3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.npz\"\n", "mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id_audio}.mp3\"\n", "print(s3_filepath)\n", "data = read_from_s3(s3_filepath, read_f=np.load)\n", "\n", "audio = Audio.from_s3(mp3_filepath, n_channels=2)\n", "\n", "if \"v3.0_raw\" in data:\n", " codes = data[\"v3.0_raw\"]\n", "elif \"v3.5_raw\" in data:\n", " codes = data[\"v3.5_raw\"]\n", "elif \"v4.0_raw\" in data:\n", " codes = data[\"v4.0_raw\"]\n", "else:\n", " raise ValueError(\"No codes found\")\n", "\n", "text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{gen_id}_hoot.json\")\n", "aligned_lyrics = json.loads(text_data)\n", "# print(aligned_lyrics)\n", "lyrics = \"\".join(w[\"word\"] for w in aligned_lyrics if \"word\" in w)\n", "audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "id": "5ec15f19-f5f8-4b99-a6f6-6d33b1a4d4c7", "metadata": {}, "outputs": [], "source": [ "sem_codes = torch.tensor(codes[:, 0]).to(torch.long)\n", "audios = []\n", "for n in seeds:\n", " pred_audio = generate(\n", " sem_codes, \n", " lyrics=lyrics,\n", " # history_lyrics=history_text,\n", " # tags='shoegaze, ethereal, art, ego',\n", " tags='pop',\n", " text_cfg_coef=text_cfg_coef, \n", " ctx_cfg_coef=ctx_cfg_coef,\n", " steps=steps,\n", " seed=n,\n", " semantic_skip_factor=1,\n", " downscale_ctx_vector=False,\n", " noise_ctx_vector=0.5,\n", " noise_ctx_vector_pad_size=25,\n", " ).normalize_volume()\n", " audios.append(pred_audio)\n", " pred_audio.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "5608019b-9896-4990-bd45-3276c5308d75", "metadata": {}, "outputs": [], "source": [ "gen_id = \"ce168306-b6af-4120-b938-9c2ada58af81\" # debug\n", "gen_id_audio = gen_id\n", "\n", "s3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.npz\"\n", "mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id_audio}.mp3\"\n", "print(s3_filepath)\n", "data = read_from_s3(s3_filepath, read_f=np.load)\n", "\n", "audio = Audio.from_s3(mp3_filepath, n_channels=2)\n", "\n", "if \"v3.0_raw\" in data:\n", " codes = data[\"v3.0_raw\"]\n", "elif \"v3.5_raw\" in data:\n", " codes = data[\"v3.5_raw\"]\n", "elif \"v4.0_raw\" in data:\n", " codes = data[\"v4.0_raw\"]\n", "else:\n", " raise ValueError(\"No codes found\")\n", "\n", "text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{gen_id}_hoot.json\")\n", "aligned_lyrics = json.loads(text_data)\n", "# print(aligned_lyrics)\n", "lyrics = \"\".join(w[\"word\"] for w in aligned_lyrics if \"word\" in w)\n", "audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "id": "ea5212ad-1b87-4589-a024-045d2f183af7", "metadata": {}, "outputs": [], "source": [ "sem_codes = torch.tensor(codes[:, 0]).to(torch.long)\n", "audios = []\n", "for n in seeds:\n", " pred_audio = generate(\n", " sem_codes, \n", " lyrics=lyrics,\n", " # history_lyrics=history_text,\n", " # tags='shoegaze, ethereal, art, ego',\n", " tags='pop',\n", " text_cfg_coef=text_cfg_coef, \n", " ctx_cfg_coef=ctx_cfg_coef,\n", " steps=steps,\n", " seed=n,\n", " semantic_skip_factor=1,\n", " downscale_ctx_vector=False,\n", " noise_ctx_vector=0.5,\n", " noise_ctx_vector_pad_size=25,\n", " ).normalize_volume()\n", " audios.append(pred_audio)\n", " pred_audio.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "77725223", "metadata": {}, "outputs": [], "source": [ "# chunk_window_size = 3000\n", "\n", "# for n_chunk in range(8):\n", "# idx_pad = 5\n", "# chunk_start_s = n_chunk * chunk_window_size / models[\"dit_model\"].io_hz\n", "# chunk_end_s = (n_chunk + 1) * chunk_window_size / models[\"dit_model\"].io_hz\n", "# print(chunk_start_s, chunk_end_s)\n", "\n", "# chunk_start_indices = [\n", "# idx\n", "# for idx, m in enumerate(aligned_lyrics)\n", "# if \"start_s\" in m and m[\"start_s\"] >= chunk_start_s\n", "# ]\n", "# chunk_end_indices = [\n", "# idx for idx, m in enumerate(aligned_lyrics) if \"end_s\" in m and m[\"end_s\"] <= chunk_end_s\n", "# ]\n", "\n", "\n", "# if len(chunk_start_indices) > 0 and len(chunk_end_indices) > 0:\n", "# chunk_start_idx = max(0, chunk_start_indices[0] - idx_pad)\n", "# chunk_end_idx = min(len(aligned_lyrics), chunk_end_indices[-1] + idx_pad)\n", "# print(chunk_start_idx, chunk_end_idx)\n", "# lyrics_chunk = \"\".join(\n", "# [m[\"word\"] for m in aligned_lyrics[chunk_start_idx:chunk_end_idx]]\n", "# )\n", "# #lyrics_chunk = simplify_whitespace(lyrics_chunk, retain_newlines=True)\n", "# else:\n", "# lyrics_chunk = \"\"\n", "# print(lyrics_chunk)" ] }, { "cell_type": "markdown", "id": "441ef260", "metadata": {}, "source": [ "# Testbench" ] }, { "cell_type": "code", "execution_count": null, "id": "2115a8e4", "metadata": {}, "outputs": [], "source": [ "# import json\n", "# import numpy as np\n", "# from suno_utils.utils.s3 import read_from_s3\n", "# import shutil\n", "# output_dir = \"outputs/custom_infer_stable_audio-10102024\"\n", "# if os.path.exists(output_dir):\n", "# shutil.rmtree(output_dir)\n", "# os.makedirs(output_dir, exist_ok=True)\n", "\n", "# gen_ids = [\n", "# (\"99bffa17-7e59-47b4-a048-5528cbda05d5\", \"country\"), # sister\n", "# (\"081d73c4-7805-4212-9c80-8db1137ca3c4\", \"rock\"), # friends\n", "# (\"562f762d-6ced-4080-9af1-910ee3d0a5dc\", \"rock\"), # something real\n", "# (\"23c15c62-494d-422d-8a60-8b0454044322\", \"shanty\"), # rubber duck\n", "# (\"4b140a9e-964b-422c-85b5-5861ad1a9d38\", \"indie rock\"), # once\n", "# (\"7b214347-fa38-4e9b-96f4-f7ec65adea45\", \"rock n roll\"), # rock n roll\n", "# (\"7f5e7819-b7a1-449d-a76c-9c38950c4274\", \"pop\"), # canada\n", "# (\"27135940-957f-4152-9e40-72e5c10a46bd\", \"pop\"), # drunk\n", "# (\"caf1d7ae-ce81-4c75-b737-fd41e47a38ba\", \"pop\"), # linger\n", "# (\"ac5aa29a-f18f-406f-a904-c47493f1beec\", \"pop\"), # linger vox\n", "# (\"b04719c3-012a-4a4a-9b02-3cb2875d1744\", \"pop\"), # courtney\n", "# (\"7407c7b3-8aa2-4f0a-82e8-fa19933f6e94\", \"opera\"), # opera\n", "# (\"5b9624ef-b416-4095-a5f3-4945f09bba86\", \"math rock\"), # butterflies\n", "# ]\n", "\n", "# for gen_id, tags in gen_ids:\n", "# mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.mp3\"\n", "# print(tags, mp3_filepath)\n", "# audio = Audio.from_s3(mp3_filepath, n_channels=2)\n", "# audio.normalize_volume()\n", "# text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{gen_id}_hoot.json\")\n", "# aligned_lyrics = json.loads(text_data)\n", "\n", "# steps = 32\n", "# text_cfg_coef = 4.0\n", "# ctx_cfg_coef = 2.0\n", "# seeds = np.random.randint(0, 2**32 - 1, 3)\n", "\n", "# for n in seeds:\n", "# pred_audio = generate(\n", "# audio, \n", "# aligned_lyrics=aligned_lyrics,\n", "# tags=tags, \n", "# text_cfg_coef=text_cfg_coef, \n", "# ctx_cfg_coef=ctx_cfg_coef,\n", "# steps=steps,\n", "# seed=n,\n", "# ).normalize_volume()\n", "\n", "# # save audio to disk\n", "# pred_audio.write_mp3(f\"{output_dir}/pred_{gen_id}_{n}.mp3\")" ] }, { "cell_type": "markdown", "id": "bb494e8a", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": null, "id": "2ce62b93", "metadata": {}, "outputs": [], "source": [ "# import json\n", "# with open(\"walking_aligned_lyrics.json\") as f:\n", "# aligned_lyrics = json.load(f)" ] }, { "cell_type": "code", "execution_count": null, "id": "75675270", "metadata": {}, "outputs": [], "source": [ "# # TODO: pad with silence for integer multiple of sliding\n", "# if models[\"dit_model\"].ctx_len is not None:\n", "# window_s = int(round((models[\"dit_model\"].block_size - models[\"dit_model\"].ctx_len) / models[\"dit_model\"].io_hz))\n", "# if audio.duration_s % window_s > 0.1:\n", "# audio = Audio.concatenate([\n", "# audio, \n", "# Audio.from_silence(window_s - audio.duration_s % window_s + 0.01, sample_rate=audio.sample_rate, n_channels=audio.n_channels)\n", "# ])" ] }, { "cell_type": "code", "execution_count": null, "id": "e2d70822", "metadata": {}, "outputs": [], "source": [ "# # get latents for preconditioning\n", "# import torch\n", "# from suno_utils.tasks.dac_vae_100hz_peaq import encode as codec_encode\n", "# # latent_audio = audio\n", "# latent_audio = Audio.from_file(\"test.mp3\", sample_rate=48_000, n_channels=2).get_segment(from_s=offset_s, to_s=offset_s+duration_s)\n", "# init_latents = torch.from_numpy(codec_encode(latent_audio).T[None]) * 2.5\n", "# latent_audio.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "9c946bc7", "metadata": {}, "outputs": [], "source": [ "# import torch\n", "# models = _retrieve_models()\n", "# extra_args = torch.load(\"extra_args.pt\", weights_only=True)\n", "# torch.manual_seed(0)\n", "# x = torch.randn([1, 128, 100*30], device=\"cuda\")*50\n", "# with torch.no_grad():\n", "# out = models[\"dit_model\"].forward_inference(\n", "# x,\n", "# x.new_ones([x.shape[0]])*50,\n", "# cross_attn_cond=extra_args[\"cross_attn_cond\"],\n", "# empty_cross_attn_cond=extra_args[\"empty_cross_attn_cond\"],\n", "# cfg_scale=extra_args[\"cfg_scale\"]\n", "# )\n", "# print(out.shape)\n", "# out" ] }, { "cell_type": "code", "execution_count": null, "id": "818cf6a9", "metadata": {}, "outputs": [], "source": [ "# PAG\n", "# https://arxiv.org/pdf/2403.17377\n", "# https://huggingface.co/hyoungwoncho/sd_perturbed_attention_guidance/blob/main/pipeline.py\n", "# https://github.com/KU-CVLAB/Perturbed-Attention-Guidance?tab=readme-ov-file" ] }, { "cell_type": "code", "execution_count": null, "id": "abe70d19", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b12de9ec", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" } }, "nbformat": 4, "nbformat_minor": 5 }