{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", "import glob\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"4\"\n", "import time\n", "import torch\n", "import torchaudio\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "from suno_utils.tasks.ear import load_model\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "from suno_utils.audio import Audio\n", "from suno_utils.tasks.mert_25 import (\n", " preload_models as preload_semantic_models,\n", " load_model as load_semantic_model,\n", " encode as encode_semantic,\n", " EMBEDDING_RATE as SEMANTIC_HZ,\n", ")\n", "import numpy as np\n", "\n", "semantic_model_filepath=\"/home/georg/notebooks/gpu_nb/tmp/mert_25.pt\"\n", "semantic_clusters_filepath=\"/home/georg/notebooks/gpu_nb/tmp/mert_25_2x4k.npy\"\n", "\n", "_ = preload_semantic_models(semantic_model_filepath, semantic_clusters_filepath)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#model = load_model(\"s3://suno-data/christian/checkpoints/ear/ear_v2_s9275.pt\", compile=True)\n", "#model = load_model(\"/app/suno/christian/checkpoints/ear-v2/2025-03-13_01-42-16_s3080/last_ckpt.pt\", compile=True)\n", "#model = load_model(\"s3://suno-data/christian/checkpoints/ear/ear_v2_s3080.pt\", compile=True)\n", "\n", "model = load_model(\"/app/suno/christian/checkpoints/ear-v2/2025-04-04_09-51-12_s9595/last_ckpt.pt\", compile=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# score semantc\n", "from suno_utils.utils.s3 import read_from_s3\n", "\n", "gen_id = \"32784d29-ff57-4769-a9b9-b853d8af7953\"\n", "\n", "s3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.npz\"\n", "data = read_from_s3(s3_filepath, read_f=np.load)\n", "\n", "if \"v3.0_raw\" in data:\n", " codes = data[\"v3.0_raw\"]\n", "elif \"v3.5_raw\" in data:\n", " codes = data[\"v3.5_raw\"]\n", "elif \"v4.0_raw\" in data:\n", " codes = data[\"v4.0_raw\"]\n", "else:\n", " raise ValueError(\"No codes found\")\n", "\n", "gen_semantic_codes = torch.from_numpy(codes[:, 0]).long()#.cuda()\n", "print(gen_semantic_codes.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "\n", "# encode real audio \n", "audio_filepath = \"/home/christian/audio/reference-audio-wav/02 Dreams.wav\"\n", "audio = Audio.from_file(audio_filepath)\n", "\n", "# also encode semantic codes\n", "real_semantic_codes = encode_semantic(\n", " audio.convert(sample_rate=24_000, byte_width=2, n_channels=1)\n", ").astype(np.int64)[:, 0]\n", "real_semantic_codes = torch.from_numpy(real_semantic_codes)\n", "print(real_semantic_codes.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "gen_score = model.get_score_batch_semantic(gen_semantic_codes.unsqueeze(0).cuda())\n", "print(gen_score)\n", "\n", "real_score = model.get_score_batch_semantic(real_semantic_codes.unsqueeze(0).cuda())\n", "print(real_score)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Score audio files" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audio file we want to test\n", "original_audio_filepath = \"/home/christian/audio/reference-audio-wav/02 Dreams.wav\"\n", "diff_no_ctx_audio_filepath = \"/home/christian/code/christian/notebooks/audio/dreams-no-ctx.mp3\"\n", "diff_ctx_audio_filepath = \"/home/christian/code/christian/notebooks/audio/dreams-with-ctx-2.mp3\"\n", "\n", "original_score = model.get_score(original_audio_filepath)\n", "no_ctx_score = model.get_score(diff_no_ctx_audio_filepath)\n", "ctx_score = model.get_score(diff_ctx_audio_filepath)\n", "\n", "# higher is better quality\n", "print(f\"Original score: {original_score}\")\n", "print(f\"No ctx score: {no_ctx_score}\")\n", "print(f\"Ctx score: {ctx_score}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Score tensors" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# can also work with tensors\n", "\n", "original_tensor, sr = torchaudio.load(original_audio_filepath)\n", "\n", "# corrupt the audio\n", "corrupt_tensor = original_tensor.clone()\n", "#corrupt_tensor = torchaudio.functional.lowpass_biquad(corrupt_tensor, sr, 10000)\n", "corrupt_tensor = torchaudio.functional.highpass_biquad(corrupt_tensor, sr, 200)\n", "\n", "\n", "# get the scores\n", "original_score = model.get_score(original_tensor, sample_rate=sr)\n", "corrupt_score = model.get_score(corrupt_tensor, sample_rate=sr)\n", "\n", "print(f\"Original score: {original_score}\")\n", "print(f\"Corrupt score: {corrupt_score}\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Score over time" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "diff_ctx_audio_filepath2 = \"/home/christian/code/christian/notebooks/audio/dreams-with-ctx-2.mp3\"\n", "diff_ctx_audio_filepath3 = \"/home/christian/code/christian/notebooks/audio/dreams-with-ctx-3.mp3\"\n", "\n", "diff_no_ctx_audio_filepath = \"/home/christian/code/christian/notebooks/audio/dreams-no-ctx.mp3\"\n", "\n", "ctx_audio_tensor2, sr = torchaudio.load(diff_ctx_audio_filepath2)\n", "ctx_audio_tensor3, sr = torchaudio.load(diff_ctx_audio_filepath3)\n", "\n", "no_ctx_audio_tensor, sr = torchaudio.load(diff_no_ctx_audio_filepath)\n", "\n", "\n", "ctx_scores2, ctx_mean_score2 = model.get_score(ctx_audio_tensor2, sample_rate=sr, return_scores=True)\n", "ctx_scores3, ctx_mean_score3 = model.get_score(ctx_audio_tensor3, sample_rate=sr, return_scores=True)\n", "no_ctx_scores, no_ctx_mean_score = model.get_score(no_ctx_audio_tensor, sample_rate=sr, return_scores=True)\n", "\n", "# plot the scores\n", "\n", "# time indices, each frame is 5 seconds \n", "time_indices = np.arange(0, len(ctx_scores2)) * 5\n", "\n", "plt.plot(time_indices, ctx_scores2, label=f\"ctx {ctx_mean_score2:0.1f}\")\n", "plt.plot(time_indices, ctx_scores3, label=f\"ctx {ctx_mean_score3:0.1f}\")\n", "plt.plot(time_indices, no_ctx_scores, label=f\"no ctx {no_ctx_mean_score:0.1f}\")\n", "plt.xlabel(\"Time (seconds)\")\n", "plt.ylabel(\"Score\")\n", "plt.title(\"Score over time\")\n", "plt.grid(c=\"lightgray\")\n", "plt.legend()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# compare v3.5 to v4\n", "from suno_utils.utils.s3 import read_from_s3\n", "\n", "v3_5_gen_id = \"32784d29-ff57-4769-a9b9-b853d8af7953\"\n", "v4_gen_id = \"ca9a7acc-cd86-42d4-b168-272394e5f07d\"\n", "\n", "\n", "v3_5_audio_filepath = f\"s3://suno-data-uploads/studio/uploads/{v3_5_gen_id}.mp3\"\n", "v4_audio_filepath = f\"s3://suno-data-uploads/studio/uploads/{v4_gen_id}.mp3\"\n", "\n", "v3_5_audio_tensor, sr = read_from_s3(v3_5_audio_filepath, read_f=torchaudio.load)\n", "v4_audio_tensor, sr = read_from_s3(v4_audio_filepath, read_f=torchaudio.load)\n", "\n", "v3_5_score = model.get_score(v3_5_audio_tensor, sample_rate=sr)\n", "v4_score = model.get_score(v4_audio_tensor, sample_rate=sr)\n", "\n", "print(f\"v3.5 score: {v3_5_score}\")\n", "print(f\"v4 score: {v4_score}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import glob\n", "filepaths = glob.glob(\"/home/christian/code/christian/notebooks/audio/ctx2/*.mp3\")\n", "\n", "for filepath in filepaths:\n", " audio_tensor, sr = torchaudio.load(filepath)\n", " scores, mean_score = model.get_score(audio_tensor, sample_rate=sr, return_scores=True)\n", " print(f\"{filepath}: {mean_score}\")\n", " time_indices = np.arange(0, len(scores)) * 5\n", " plt.plot(time_indices, scores, label=f\"ctx {mean_score:0.1f}\")\n", "\n", "# time indices, each frame is 5 seconds \n", "\n", "plt.xlabel(\"Time (seconds)\")\n", "plt.ylabel(\"Score\")\n", "plt.title(\"Score over time\")\n", "plt.grid(c=\"lightgray\")\n", "plt.legend()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import glob\n", "filepaths = glob.glob(\"/home/christian/code/christian/notebooks/audio/steps2/*.mp3\")\n", "\n", "steps_list = []\n", "score_list = []\n", "\n", "for filepath in filepaths:\n", " # get the steps from the filename\n", " steps = int(filepath.replace(\".mp3\", \"\").split(\"_\")[-2])\n", "\n", " audio_tensor, sr = torchaudio.load(filepath)\n", " scores, mean_score = model.get_score(audio_tensor, sample_rate=sr, return_scores=True)\n", " print(f\"{steps}: {mean_score}\")\n", " score_list.append(mean_score)\n", " steps_list.append(steps)\n", "# time indices, each frame is 5 seconds \n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "plt.plot(steps_list, score_list)\n", "plt.xlabel(\"Steps\")\n", "plt.ylabel(\"Score\")\n", "plt.title(\"Score over steps\")\n", "plt.grid(c=\"lightgray\")\n", "plt.show()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "audio_filepaths = glob.glob(\"/home/christian/code/christian/notebooks/audio/noise_ctx3/*.mp3\")\n", "print(len(audio_filepaths))\n", "\n", "results = {}\n", "for filepath in audio_filepaths:\n", " audio_tensor, sr = torchaudio.load(filepath)\n", " scores, mean_score = model.get_score(audio_tensor, sample_rate=sr, return_scores=True)\n", " print(f\"{filepath}: {mean_score}\")\n", " results[filepath] = (mean_score, scores)\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "plt.figure(figsize=(8, 3))\n", "colormap = plt.cm.viridis(np.linspace(0.0, 0.75, len(audio_filepaths)*1))\n", "\n", "# time indices, each frame is 5 seconds \n", "time_indices = np.arange(0, len(scores)) * 5\n", "for i, (filepath, (score, scores)) in enumerate(results.items()):\n", " noise_ctx = filepath.replace(\".mp3\", \"\").split(\"_\")[-1]\n", " if noise_ctx == \"0.0\" or noise_ctx == \"0.5\": \n", " plt.plot(time_indices, scores, label=f\"noise ctx={noise_ctx} score: {score:0.1f}\", color=colormap[i])\n", "\n", "plt.xlim(0, time_indices[-1])\n", "plt.xlabel(\"Time (seconds)\")\n", "plt.ylabel(\"Score (higher is better)\")\n", "plt.title(\"Ear audio quality score over time\")\n", "plt.grid(c=\"lightgray\")\n", "plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left', borderaxespad=0.)\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "filepaths = glob.glob(\"/app/suno/data/diff_sft/v45_2b_step_2_600_000/pos_interesting_clips_up_u_1_20241201_full_v2/*.mp3\")\n", "print(len(filepaths))" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [], "source": [ "# group the filepaths based on the base s3 id \n", "base_s3_ids = [filepath.split(\"_upsample_\")[0] for filepath in filepaths]\n", "\n", "# get the unique base s3 ids\n", "unique_base_s3_ids = list(set(base_s3_ids))\n", "\n", "# group the filepaths based on the base s3 id\n", "grouped_filepaths = {base_s3_id: [] for base_s3_id in unique_base_s3_ids}\n", "for filepath in filepaths:\n", " base_s3_id = filepath.split(\"_upsample_\")[0]\n", " grouped_filepaths[base_s3_id].append(filepath)\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(len(grouped_filepaths))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# make a plot for the \n", "filepaths = grouped_filepaths[unique_base_s3_ids[2]]\n", "\n", "results = {}\n", "for filepath in filepaths:\n", " audio_tensor, sr = torchaudio.load(filepath)\n", " scores, mean_score = model.get_score(audio_tensor, sample_rate=sr, return_scores=True)\n", " print(f\"{filepath}: {mean_score}\")\n", " results[filepath] = (mean_score, scores)\n", "\n", "# sort the results by the mean score\n", "results = sorted(results.items(), key=lambda x: x[1][0], reverse=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "\n", "# print the best and worst 5\n", "# best \n", "print(results[0][0], np.mean(results[0][1][0]))\n", "# worst \n", "print(results[-1][0], np.mean(results[-1][1][0]))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "colormap = plt.cm.Blues(np.linspace(0.25, 1, len(audio_filepaths)*1))\n", "\n", "# time indices, each frame is 5 seconds \n", "time_indices = np.arange(0, len(scores)) * 5\n", "for i, (filepath, (score, scores)) in enumerate(results.items()):\n", " plt.plot(time_indices, scores, label=f\"score: {score:0.1f}\")\n", "\n", "plt.xlim(30, time_indices[-1])\n", "plt.xlabel(\"Time (seconds)\")\n", "plt.ylabel(\"Score\")\n", "plt.title(\"Score over time\")\n", "plt.grid(c=\"lightgray\")\n", "plt.legend()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.9" } }, "nbformat": 4, "nbformat_minor": 2 }