{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "# given two directories of audio files from two different models\n", "# get the ear scores for each file in the two directories\n", "# and plot the results in a bar chart\n", "\n", "import os\n", "import glob\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"4\"\n", "import time\n", "import torch\n", "import torchaudio\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "from suno_utils.tasks.ear import load_model" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model = load_model(\"/app/suno/christian/checkpoints/ear-v2/2025-03-13_01-42-16_s3080/last_ckpt.pt\", compile=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "dir_a = \"/home/christian/code/christian/notebooks/outputs/v4-25hz-s6971-prod\"\n", "dir_b = \"/home/christian/code/christian/notebooks/outputs/v4-25hz-s5157-dpo\"\n", "\n", "filepaths_a = glob.glob(os.path.join(dir_a, \"*.mp3\"))\n", "filepaths_b = glob.glob(os.path.join(dir_b, \"*.mp3\"))\n", "\n", "print(len(filepaths_a))\n", "print(len(filepaths_b))\n", "# merge the two lists into a tuple with the files that have the same name\n", "merged_filepaths = []\n", "for filepath_a in filepaths_a:\n", " for filepath_b in filepaths_b:\n", " if os.path.basename(filepath_a) == os.path.basename(filepath_b):\n", " merged_filepaths.append((filepath_a, filepath_b))\n", "\n", "print(len(merged_filepaths))\n", "scores_a = []\n", "scores_b = []\n", "loudness_a = []\n", "loudness_b = []\n" ] }, { "cell_type": "code", "execution_count": 66, "metadata": {}, "outputs": [], "source": [ "import pyloudnorm as pyln\n", "import soundfile as sf\n", "from tqdm import tqdm" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for idx, (filepath_a, filepath_b) in enumerate(tqdm(merged_filepaths)):\n", "\n", " # measure the loudness of the two files\n", " audio_a, sr_a = sf.read(filepath_a)\n", " audio_b, sr_b = sf.read(filepath_b)\n", " loudness_a.append(pyln.Meter(sr_a).integrated_loudness(audio_a))\n", " loudness_b.append(pyln.Meter(sr_b).integrated_loudness(audio_b))\n", "\n", " score_a = model.get_score(filepath_a)\n", " score_b = model.get_score(filepath_b)\n", " scores_a.append(score_a)\n", " scores_b.append(score_b)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(scores_a)\n", "print(scores_b)\n", "\n", "import IPython\n", "\n", "\n", "score_diff = [score_b - score_a for score_a, score_b in zip(scores_a, scores_b)]\n", "# get the index of the largest diff\n", "index_max_diff = np.argmax(score_diff)\n", "print(f\"Max difference: {score_diff[index_max_diff]} at index {index_max_diff}\")\n", "\n", "# sort merged filepaths by score_diff\n", "sorted_indices = np.argsort(score_diff)[::-1] # Sort in descending order\n", "sorted_filepaths = [merged_filepaths[i] for i in sorted_indices]\n", "sorted_diffs = [score_diff[i] for i in sorted_indices]\n", "scorted_scores_a = [scores_a[i] for i in sorted_indices]\n", "scorted_scores_b = [scores_b[i] for i in sorted_indices]\n", "\n", "\n", "print(f\"Top 5 differences: {sorted_diffs[:5]}\")\n", "\n", "# Get the file with the largest difference\n", "filepath_a, filepath_b = merged_filepaths[index_max_diff]\n", "\n", "for idx, (filepath_a, filepath_b) in enumerate(sorted_filepaths[:5]):\n", " print(f\"Model A score: {scorted_scores_a[idx]}, Model B score: {scorted_scores_b[idx]}\")\n", " IPython.display.display(IPython.display.Audio(filepath_a))\n", " IPython.display.display(IPython.display.Audio(filepath_b))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(np.mean(score_diff))\n", "# can we compute the percent better of model b over model a?\n", "percent_better = [100 * (score_b - score_a) / score_a for score_a, score_b in zip(scores_a, scores_b)]\n", "print(percent_better)\n", "print(np.mean(percent_better))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# plot a histogram of the scores\n", "#plt.hist(scores_a, bins=np.linspace(5, 40, 50), alpha=0.5, label='Model A')\n", "#plt.hist(scores_b, bins=np.linspace(5, 40, 50), alpha=0.5, label='Model B')\n", "plt.hist(score_diff, bins=np.linspace(-20, 20, 40), alpha=0.5, label='Model B - Model A')\n", "plt.legend(loc='upper right')\n", "plt.show()\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "plt.hist(loudness_a, bins=np.linspace(-20, 0, 40), alpha=0.5, label='prod')\n", "plt.hist(loudness_b, bins=np.linspace(-20, 0, 40), alpha=0.5, label='new dpo')\n", "plt.legend(loc='upper right')\n", "plt.xlabel('Loudness (LUFS)')\n", "plt.show()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "rand_idx = np.random.randint(len(merged_filepaths))\n", "filepath_a, filepath_b = merged_filepaths[rand_idx]\n", "# measure the loudness of the two files\n", "audio_a, sr_a = sf.read(filepath_a)\n", "audio_b, sr_b = sf.read(filepath_b)\n", "loudness_a.append(pyln.Meter(sr_a).integrated_loudness(audio_a))\n", "loudness_b.append(pyln.Meter(sr_b).integrated_loudness(audio_b))\n", "\n", "score_a, mean_score_a = model.get_score(filepath_a, return_scores=True)\n", "score_b, mean_score_b = model.get_score(filepath_b, return_scores=True)\n", "\n", "print(mean_score_a)\n", "\n", "# plot over time\n", "plt.plot(score_a)\n", "plt.plot(score_b)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_clone_diff", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" } }, "nbformat": 4, "nbformat_minor": 2 }