{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import os\n", "import json\n", "\n", "from suno_utils.utils.text import read_jsonl" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "\n", "# we need a special case for genius\n", "old_genius_metas = read_jsonl(\"/home/christian/code/christian/metadata/genius_hq_metas.jsonl\")\n", "# make a new dict that maps from the youtube id to genius id\n", "genius_id_to_youtube_id = {meta[\"original_id\"]: meta[\"id\"] for meta in old_genius_metas}\n" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "len(ids_set): 1,115,041\n" ] } ], "source": [ "# sft subset for gpt\n", "sft_filepath = \"/home/christian/code/christian/metadata/v45_splits/ids_keep_sets_v11.json\"\n", "\n", "with open(sft_filepath, \"r\") as f:\n", " sft_ids = json.load(f)\n", "\n", "# also need to fix the genius ids here\n", "sft_ids[\"genius\"] = [genius_id_to_youtube_id[k] for k in sft_ids[\"genius\"]]\n", "\n", "ids_set = set()\n", "# merge these down to just ids\n", "for key, val in sft_ids.items():\n", " ids_set.update(val)\n", "\n", "print(f\"len(ids_set): {len(ids_set):,}\")\n" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['4a537928-088d-481d-88e6-9441c1cbb012', '2f1fac49-0cae-4bbc-8509-ecadb0eb9944', '69e9f49c-d2fd-4fb5-bc7a-09f8975ad49d', 'b6682d6a-ca6e-4a94-8398-6bf2679e6a32', 'a5e0fe9d-e04b-4b54-a35d-ebe3d149a56b', 'b0e31e50-b932-4335-a95a-da034a882e1f', 'b26ad8dc-1baa-4a4b-93c9-217a04b8c247', '459e8455-d731-4a6d-94d0-88aed92543d9', 'c68ef8f0-35a6-4de6-a201-27473bd09754', '9fdbe77b-c07e-42e0-80b0-7999ac7321d4']\n", "5008040\n" ] } ], "source": [ "# load ear data\n", "discogs_subset_ear_scores_filepath = \"/home/christian/code/christian/metadata/ear/discogs_subset_ear_scores.json\"\n", "genius_ear_scores_filepath = \"/home/christian/code/christian/metadata/ear/genius_ear_scores.json\"\n", "imslp_ear_scores_filepath = \"/home/christian/code/christian/metadata/ear/imslp_ear_scores.json\"\n", "\n", "with open(discogs_subset_ear_scores_filepath, \"r\") as f:\n", " discogs_subset_ear_scores = json.load(f)\n", "\n", "with open(genius_ear_scores_filepath, \"r\") as f:\n", " genius_ear_scores = json.load(f)\n", "\n", "genius_ear_scores = {genius_id_to_youtube_id[k]: v for k, v in genius_ear_scores.items()}\n", "print(list(genius_ear_scores.keys())[:10])\n", "\n", "with open(imslp_ear_scores_filepath, \"r\") as f:\n", " imslp_ear_scores = json.load(f)\n", "\n", "ear_scores = {**discogs_subset_ear_scores, **genius_ear_scores, **imslp_ear_scores}\n", "print(len(ear_scores))\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "mean_ear_scores = {k: v[\"mean_score\"] for k, v in ear_scores.items()}" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAGdCAYAAAD+JxxnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAuF0lEQVR4nO3df3RUdX7/8VcC5IfITAiYDFMCZJXlR0FYAUP8QdeSQxC0mxVbgunKahaqm1AgKgSFgFt2YUNtIYKkdHsWzylRpGdhFTSaBiFdiQGCKT+WZNGiYHGCu5AZiRICud8/+OaW4Xdwhknm83ycM+eY+3nP3M/9eN157efe+5kIy7IsAQAAGCgy1B0AAAAIFYIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYnUPdgfaspaVFx44dU7du3RQRERHq7gAAgOtgWZa++uorud1uRUZefc6HIHQVx44dU1JSUqi7AQAAbsDRo0fVu3fvq9YQhK6iW7duks4PpMPhCHFvAADA9fD5fEpKSrK/x6+GIHQVrZfDHA4HQQgAgA7mem5r4WZpAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhACgA+mXv0X98reEuhtA2CAIAUAHcWEAIhABgUEQAgAAxuoc6g4AAC6PGR8g+No8I1RRUaGHH35YbrdbERER2rRpk93W3NysuXPnaujQoeratavcbrcef/xxHTt2zO8zTpw4oaysLDkcDsXFxSk7O1unTp3yq9m7d6/uv/9+xcTEKCkpSYWFhZf0ZcOGDRo4cKBiYmI0dOhQvf32237tlmWpoKBAvXr1UmxsrNLS0nTo0KG2HjIAAAhTbQ5CjY2NGjZsmFatWnVJ29dff609e/ZowYIF2rNnj37zm9+orq5Of/VXf+VXl5WVpQMHDqisrEybN29WRUWFpk+fbrf7fD6NGzdOffv2VXV1tZYtW6ZFixZpzZo1ds2OHTs0ZcoUZWdn66OPPlJGRoYyMjK0f/9+u6awsFBFRUUqLi5WVVWVunbtqvT0dJ0+fbqthw0AAMJQhGVZ1g2/OSJCGzduVEZGxhVrdu3apbvvvlufffaZ+vTpo4MHD2rw4MHatWuXRo4cKUkqLS3VhAkT9Pnnn8vtdmv16tV64YUX5PF4FBUVJUnKz8/Xpk2bVFtbK0maPHmyGhsbtXnzZntfo0eP1vDhw1VcXCzLsuR2u/XMM8/o2WeflSR5vV4lJiZq7dq1yszMvObx+Xw+OZ1Oeb1eORyOGx0mAGiTtlwS+3TpxCD2BOiY2vL9HfSbpb1eryIiIhQXFydJqqysVFxcnB2CJCktLU2RkZGqqqqya8aMGWOHIElKT09XXV2dTp48adekpaX57Ss9PV2VlZWSpMOHD8vj8fjVOJ1OpaSk2DUXa2pqks/n83sBQHvG02PAtxPUIHT69GnNnTtXU6ZMsROZx+NRQkKCX13nzp0VHx8vj8dj1yQmJvrVtP59rZoL2y983+VqLrZkyRI5nU77lZSU1OZjBgAAHUfQglBzc7P+5m/+RpZlafXq1cHaTUDNmzdPXq/Xfh09ejTUXQIAAEEUlMfnW0PQZ599pq1bt/pdn3O5XDp+/Lhf/dmzZ3XixAm5XC67pr6+3q+m9e9r1VzY3rqtV69efjXDhw+/bL+jo6MVHR3d1sMFAAAdVMBnhFpD0KFDh/Sf//mf6tGjh197amqqGhoaVF1dbW/bunWrWlpalJKSYtdUVFSoubnZrikrK9OAAQPUvXt3u6a8vNzvs8vKypSamipJSk5Olsvl8qvx+XyqqqqyawCgPeF+H+Dma3MQOnXqlGpqalRTUyPp/E3JNTU1OnLkiJqbm/Xoo49q9+7dWrdunc6dOyePxyOPx6MzZ85IkgYNGqTx48dr2rRp2rlzpz744APl5uYqMzNTbrdbkvTYY48pKipK2dnZOnDggNavX68VK1YoLy/P7sfMmTNVWlqql156SbW1tVq0aJF2796t3NxcSeefaJs1a5YWL16sN998U/v27dPjjz8ut9t91afcAKAjIkQBN6bNj89v27ZNDzzwwCXbp06dqkWLFik5Ofmy73v//ff1/e9/X9L5BRVzc3P11ltvKTIyUpMmTVJRUZFuvfVWu37v3r3KycnRrl271LNnT82YMUNz5871+8wNGzZo/vz5+vTTT9W/f38VFhZqwoQJdrtlWVq4cKHWrFmjhoYG3XfffXrllVf03e9+97qOlcfnAdxMgQgyPE4PtO37+1utIxTuCEIAbiaCEBAY7WodIQAAgPaKIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgDQDrAGEBAaBCEAAGAsghAAADAWQQgAwgiX2IC2IQgBAABjEYQAAICxCEIAAMBYnUPdAQAwGff0AKHFjBAAADAWQQgAwky//C3MNAHXiSAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYrCMEACHAU11A+8CMEAAAMBZBCAAAGIsgBAAAjEUQAoAwxQrTwLURhAAAgLEIQgAAwFg8Pg8ANxGXqoD2hRkhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEACHP85hhwZQQhAABgLIIQAAAwFkEIAAAYix9dBYCbgHt0gPaJGSEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMZqcxCqqKjQww8/LLfbrYiICG3atMmv3bIsFRQUqFevXoqNjVVaWpoOHTrkV3PixAllZWXJ4XAoLi5O2dnZOnXqlF/N3r17df/99ysmJkZJSUkqLCy8pC8bNmzQwIEDFRMTo6FDh+rtt99uc18AAIC52hyEGhsbNWzYMK1ateqy7YWFhSoqKlJxcbGqqqrUtWtXpaen6/Tp03ZNVlaWDhw4oLKyMm3evFkVFRWaPn263e7z+TRu3Dj17dtX1dXVWrZsmRYtWqQ1a9bYNTt27NCUKVOUnZ2tjz76SBkZGcrIyND+/fvb1BcAMAU/tQFcKsKyLOuG3xwRoY0bNyojI0PS+RkYt9utZ555Rs8++6wkyev1KjExUWvXrlVmZqYOHjyowYMHa9euXRo5cqQkqbS0VBMmTNDnn38ut9ut1atX64UXXpDH41FUVJQkKT8/X5s2bVJtba0kafLkyWpsbNTmzZvt/owePVrDhw9XcXHxdfXlWnw+n5xOp7xerxwOx40OEwC0qwDy6dKJoe4CEFRt+f4O6D1Chw8flsfjUVpamr3N6XQqJSVFlZWVkqTKykrFxcXZIUiS0tLSFBkZqaqqKrtmzJgxdgiSpPT0dNXV1enkyZN2zYX7aa1p3c/19OViTU1N8vl8fi8AABC+AhqEPB6PJCkxMdFve2Jiot3m8XiUkJDg1965c2fFx8f71VzuMy7cx5VqLmy/Vl8utmTJEjmdTvuVlJR0HUcNAAA6Kp4au8C8efPk9Xrt19GjR0PdJQBhoD1dFgPgL6BByOVySZLq6+v9ttfX19ttLpdLx48f92s/e/asTpw44Vdzuc+4cB9Xqrmw/Vp9uVh0dLQcDoffCwAAhK+ABqHk5GS5XC6Vl5fb23w+n6qqqpSamipJSk1NVUNDg6qrq+2arVu3qqWlRSkpKXZNRUWFmpub7ZqysjINGDBA3bt3t2su3E9rTet+rqcvAADAbG0OQqdOnVJNTY1qamoknb8puaamRkeOHFFERIRmzZqlxYsX680339S+ffv0+OOPy+1220+WDRo0SOPHj9e0adO0c+dOffDBB8rNzVVmZqbcbrck6bHHHlNUVJSys7N14MABrV+/XitWrFBeXp7dj5kzZ6q0tFQvvfSSamtrtWjRIu3evVu5ubmSdF19AQAAZuvc1jfs3r1bDzzwgP13aziZOnWq1q5dqzlz5qixsVHTp09XQ0OD7rvvPpWWliomJsZ+z7p165Sbm6uxY8cqMjJSkyZNUlFRkd3udDr13nvvKScnRyNGjFDPnj1VUFDgt9bQPffco5KSEs2fP1/PP/+8+vfvr02bNmnIkCF2zfX0BQAAmOtbrSMU7lhHCEAgtLebpVlHCOEuZOsIAQAAdCQEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgDDtLen2IBQavM6QgCA60PgANo/ZoQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFgEIQAAYCx+awwAAqwj/MZYax8/XToxxD0BQosZIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMxY+uAkCAdIQfW70YP74K0zEjBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABj8VtjAPAtdcTfGANwHjNCAADAWAQhAABgLIIQAAAwVsCD0Llz57RgwQIlJycrNjZWt99+u/7hH/5BlmXZNZZlqaCgQL169VJsbKzS0tJ06NAhv885ceKEsrKy5HA4FBcXp+zsbJ06dcqvZu/evbr//vsVExOjpKQkFRYWXtKfDRs2aODAgYqJidHQoUP19ttvB/qQAQBABxXwIPTLX/5Sq1ev1sqVK3Xw4EH98pe/VGFhoV5++WW7prCwUEVFRSouLlZVVZW6du2q9PR0nT592q7JysrSgQMHVFZWps2bN6uiokLTp0+3230+n8aNG6e+ffuqurpay5Yt06JFi7RmzRq7ZseOHZoyZYqys7P10UcfKSMjQxkZGdq/f3+gDxsAOrR++Vu46RtGirAunKoJgIceekiJiYn6t3/7N3vbpEmTFBsbq3//93+XZVlyu9165pln9Oyzz0qSvF6vEhMTtXbtWmVmZurgwYMaPHiwdu3apZEjR0qSSktLNWHCBH3++edyu91avXq1XnjhBXk8HkVFRUmS8vPztWnTJtXW1kqSJk+erMbGRm3evNnuy+jRozV8+HAVFxdf81h8Pp+cTqe8Xq8cDkfAxghAeAmnAPHp0omh7gLwrbXl+zvgM0L33HOPysvL9Yc//EGS9N///d/63e9+pwcffFCSdPjwYXk8HqWlpdnvcTqdSklJUWVlpSSpsrJScXFxdgiSpLS0NEVGRqqqqsquGTNmjB2CJCk9PV11dXU6efKkXXPhflprWvdzsaamJvl8Pr8XAAAIXwFfRyg/P18+n08DBw5Up06ddO7cOf385z9XVlaWJMnj8UiSEhMT/d6XmJhot3k8HiUkJPh3tHNnxcfH+9UkJydf8hmtbd27d5fH47nqfi62ZMkSvfjiizdy2AAAoAMK+IzQG2+8oXXr1qmkpER79uzRq6++qn/8x3/Uq6++GuhdBdy8efPk9Xrt19GjR0PdJQAAEEQBnxF67rnnlJ+fr8zMTEnS0KFD9dlnn2nJkiWaOnWqXC6XJKm+vl69evWy31dfX6/hw4dLklwul44fP+73uWfPntWJEyfs97tcLtXX1/vVtP59rZrW9otFR0crOjr6Rg4bAAB0QAGfEfr6668VGen/sZ06dVJLS4skKTk5WS6XS+Xl5Xa7z+dTVVWVUlNTJUmpqalqaGhQdXW1XbN161a1tLQoJSXFrqmoqFBzc7NdU1ZWpgEDBqh79+52zYX7aa1p3Q8AADBbwIPQww8/rJ///OfasmWLPv30U23cuFH/9E//pB/+8IeSpIiICM2aNUuLFy/Wm2++qX379unxxx+X2+1WRkaGJGnQoEEaP368pk2bpp07d+qDDz5Qbm6uMjMz5Xa7JUmPPfaYoqKilJ2drQMHDmj9+vVasWKF8vLy7L7MnDlTpaWleumll1RbW6tFixZp9+7dys3NDfRhAzBUOD0xBpgo4JfGXn75ZS1YsEA//elPdfz4cbndbv3d3/2dCgoK7Jo5c+aosbFR06dPV0NDg+677z6VlpYqJibGrlm3bp1yc3M1duxYRUZGatKkSSoqKrLbnU6n3nvvPeXk5GjEiBHq2bOnCgoK/NYauueee1RSUqL58+fr+eefV//+/bVp0yYNGTIk0IcNAAA6oICvIxROWEcIwLWE24wQ6wghHIR0HSEAAICOgiAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAwNYvf0vYrY0EXA1BCAAAGCvgP7EBACZg1gQID8wIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAgEvwEyIwBUEIAAAYix9dBYA2YKYECC/MCAEAAGMRhAAAgLEIQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAY7GyNABcB1aUBsITM0IAAMBYBCEAAGAsghAA4LL65W/hkiDCHkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLH10FgKtgZWUgvAVlRuh///d/9bd/+7fq0aOHYmNjNXToUO3evdtutyxLBQUF6tWrl2JjY5WWlqZDhw75fcaJEyeUlZUlh8OhuLg4ZWdn69SpU341e/fu1f3336+YmBglJSWpsLDwkr5s2LBBAwcOVExMjIYOHaq33347GIcMAAA6oIAHoZMnT+ree+9Vly5d9M477+j3v/+9XnrpJXXv3t2uKSwsVFFRkYqLi1VVVaWuXbsqPT1dp0+ftmuysrJ04MABlZWVafPmzaqoqND06dPtdp/Pp3Hjxqlv376qrq7WsmXLtGjRIq1Zs8au2bFjh6ZMmaLs7Gx99NFHysjIUEZGhvbv3x/owwYAAB1QhGVZViA/MD8/Xx988IH+67/+67LtlmXJ7XbrmWee0bPPPitJ8nq9SkxM1Nq1a5WZmamDBw9q8ODB2rVrl0aOHClJKi0t1YQJE/T555/L7XZr9erVeuGFF+TxeBQVFWXve9OmTaqtrZUkTZ48WY2Njdq8ebO9/9GjR2v48OEqLi6+5rH4fD45nU55vV45HI5vNS4AOiYujUmfLp0Y6i4AbdKW7++Azwi9+eabGjlypP76r/9aCQkJ+t73vqd//dd/tdsPHz4sj8ejtLQ0e5vT6VRKSooqKyslSZWVlYqLi7NDkCSlpaUpMjJSVVVVds2YMWPsECRJ6enpqqur08mTJ+2aC/fTWtO6n4s1NTXJ5/P5vQAAQPgKeBD6n//5H61evVr9+/fXu+++q6efflp///d/r1dffVWS5PF4JEmJiYl+70tMTLTbPB6PEhIS/No7d+6s+Ph4v5rLfcaF+7hSTWv7xZYsWSKn02m/kpKS2nz8ABBu+uVvYWYMYSvgQailpUV33XWXfvGLX+h73/uepk+frmnTpl3XpahQmzdvnrxer/06evRoqLsEAACCKOBBqFevXho8eLDftkGDBunIkSOSJJfLJUmqr6/3q6mvr7fbXC6Xjh8/7td+9uxZnThxwq/mcp9x4T6uVNPafrHo6Gg5HA6/FwAACF8BD0L33nuv6urq/Lb94Q9/UN++fSVJycnJcrlcKi8vt9t9Pp+qqqqUmpoqSUpNTVVDQ4Oqq6vtmq1bt6qlpUUpKSl2TUVFhZqbm+2asrIyDRgwwH5CLTU11W8/rTWt+wEAAGYLeBCaPXu2PvzwQ/3iF7/Qxx9/rJKSEq1Zs0Y5OTmSpIiICM2aNUuLFy/Wm2++qX379unxxx+X2+1WRkaGpPMzSOPHj9e0adO0c+dOffDBB8rNzVVmZqbcbrck6bHHHlNUVJSys7N14MABrV+/XitWrFBeXp7dl5kzZ6q0tFQvvfSSamtrtWjRIu3evVu5ubmBPmwAANABBXxl6VGjRmnjxo2aN2+efvaznyk5OVnLly9XVlaWXTNnzhw1NjZq+vTpamho0H333afS0lLFxMTYNevWrVNubq7Gjh2ryMhITZo0SUVFRXa70+nUe++9p5ycHI0YMUI9e/ZUQUGB31pD99xzj0pKSjR//nw9//zz6t+/vzZt2qQhQ4YE+rABAEAHFPB1hMIJ6wgB4Gmp/8N6QugoQrqOEACEC0IQEP4IQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAADXpV/+FtZWQtgJ+E9sAEBHx5c9YA5mhAAAgLEIQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAECb8JtjCCf81hgA/H98uQPmYUYIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAs1hECYDzWDwLMxYwQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAG4Iv0KPcEAQAgAAxiIIAQAAYxGEAACAsVhQEYCxuL8FADNCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjBT0ILV26VBEREZo1a5a97fTp08rJyVGPHj106623atKkSaqvr/d735EjRzRx4kTdcsstSkhI0HPPPaezZ8/61Wzbtk133XWXoqOjdccdd2jt2rWX7H/VqlXq16+fYmJilJKSop07dwbjMAHAWKzHhI4sqEFo165d+pd/+Rfdeeedfttnz56tt956Sxs2bND27dt17NgxPfLII3b7uXPnNHHiRJ05c0Y7duzQq6++qrVr16qgoMCuOXz4sCZOnKgHHnhANTU1mjVrln7yk5/o3XfftWvWr1+vvLw8LVy4UHv27NGwYcOUnp6u48ePB/OwAXQAfHkDkKQIy7KsYHzwqVOndNddd+mVV17R4sWLNXz4cC1fvlxer1e33XabSkpK9Oijj0qSamtrNWjQIFVWVmr06NF655139NBDD+nYsWNKTEyUJBUXF2vu3Ln68ssvFRUVpblz52rLli3av3+/vc/MzEw1NDSotLRUkpSSkqJRo0Zp5cqVkqSWlhYlJSVpxowZys/Pv+Yx+Hw+OZ1Oeb1eORyOQA8RgBAiCAXWp0snhroLgK0t399BmxHKycnRxIkTlZaW5re9urpazc3NftsHDhyoPn36qLKyUpJUWVmpoUOH2iFIktLT0+Xz+XTgwAG75uLPTk9Ptz/jzJkzqq6u9quJjIxUWlqaXQMAAMwWlN8ae/3117Vnzx7t2rXrkjaPx6OoqCjFxcX5bU9MTJTH47FrLgxBre2tbVer8fl8+uabb3Ty5EmdO3fusjW1tbWX7XdTU5Oamprsv30+33UcLQAA6KgCPiN09OhRzZw5U+vWrVNMTEygPz6olixZIqfTab+SkpJC3SUAABBEAQ9C1dXVOn78uO666y517txZnTt31vbt21VUVKTOnTsrMTFRZ86cUUNDg9/76uvr5XK5JEkul+uSp8ha/75WjcPhUGxsrHr27KlOnTpdtqb1My42b948eb1e+3X06NEbHgcAAND+BTwIjR07Vvv27VNNTY39GjlypLKysux/7tKli8rLy+331NXV6ciRI0pNTZUkpaamat++fX5Pd5WVlcnhcGjw4MF2zYWf0VrT+hlRUVEaMWKEX01LS4vKy8vtmotFR0fL4XD4vQAAQPgK+D1C3bp105AhQ/y2de3aVT169LC3Z2dnKy8vT/Hx8XI4HJoxY4ZSU1M1evRoSdK4ceM0ePBg/ehHP1JhYaE8Ho/mz5+vnJwcRUdHS5KeeuoprVy5UnPmzNGTTz6prVu36o033tCWLf/3JEheXp6mTp2qkSNH6u6779by5cvV2NioJ554ItCHDQAAOqCg3Cx9Lf/8z/+syMhITZo0SU1NTUpPT9crr7xit3fq1EmbN2/W008/rdTUVHXt2lVTp07Vz372M7smOTlZW7Zs0ezZs7VixQr17t1bv/rVr5Senm7XTJ48WV9++aUKCgrk8Xg0fPhwlZaWXnIDNQBz8Ng8gAsFbR2hcMA6QkD4IQgFB+sIoT1pF+sIAQAAtHcEIQDAt9YvfwuzbeiQCEIAAMBYBCEAAGAsghAAADAWQQgAABgrJOsIAcDNxo28AC6HGSEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMbiqTEAYY2nxW6u1vHmR1jRUTAjBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFusIAQhLrB8E4HowIwQAAIxFEAIAAMYiCAEAAq5f/hYuT6JDIAgBAABjEYQAAICxCEIAAMBYBCEAYYd7UwBcL4IQAAAwFkEIAAAYiyAEAACMRRACAAQN6wmhvSMIAQAAY/GjqwDCBjMPANqKGSEAAGAsghAAADAWQQgAABiLIAQAAIzFzdIAOjxukgZwo5gRAgAAxiIIAQCCjlk7tFcEIQAAYCyCEAAAMBY3SwPosLjcAuDbYkYIAAAYiyAEAACMRRACAADGCngQWrJkiUaNGqVu3bopISFBGRkZqqur86s5ffq0cnJy1KNHD916662aNGmS6uvr/WqOHDmiiRMn6pZbblFCQoKee+45nT171q9m27ZtuuuuuxQdHa077rhDa9euvaQ/q1atUr9+/RQTE6OUlBTt3Lkz0IcMAAA6qIAHoe3btysnJ0cffvihysrK1NzcrHHjxqmxsdGumT17tt566y1t2LBB27dv17Fjx/TII4/Y7efOndPEiRN15swZ7dixQ6+++qrWrl2rgoICu+bw4cOaOHGiHnjgAdXU1GjWrFn6yU9+onfffdeuWb9+vfLy8rRw4ULt2bNHw4YNU3p6uo4fPx7owwZwE/XL38KN0h0Q/97QHkVYlmUFcwdffvmlEhIStH37do0ZM0Zer1e33XabSkpK9Oijj0qSamtrNWjQIFVWVmr06NF655139NBDD+nYsWNKTEyUJBUXF2vu3Ln68ssvFRUVpblz52rLli3av3+/va/MzEw1NDSotLRUkpSSkqJRo0Zp5cqVkqSWlhYlJSVpxowZys/Pv2bffT6fnE6nvF6vHA5HoIcGwA3iy7Rj+3TpxFB3AWGuLd/fQb9HyOv1SpLi4+MlSdXV1WpublZaWppdM3DgQPXp00eVlZWSpMrKSg0dOtQOQZKUnp4un8+nAwcO2DUXfkZrTetnnDlzRtXV1X41kZGRSktLs2su1tTUJJ/P5/cCAADhK6hBqKWlRbNmzdK9996rIUOGSJI8Ho+ioqIUFxfnV5uYmCiPx2PXXBiCWttb265W4/P59M033+iPf/yjzp07d9ma1s+42JIlS+R0Ou1XUlLSjR04gKBhNghAIAU1COXk5Gj//v16/fXXg7mbgJk3b568Xq/9Onr0aKi7BAAAgihoK0vn5uZq8+bNqqioUO/eve3tLpdLZ86cUUNDg9+sUH19vVwul11z8dNdrU+VXVhz8ZNm9fX1cjgcio2NVadOndSpU6fL1rR+xsWio6MVHR19YwcMAAA6nIDPCFmWpdzcXG3cuFFbt25VcnKyX/uIESPUpUsXlZeX29vq6up05MgRpaamSpJSU1O1b98+v6e7ysrK5HA4NHjwYLvmws9orWn9jKioKI0YMcKvpqWlReXl5XYNAAAwW8BnhHJyclRSUqLf/va36tatm30/jtPpVGxsrJxOp7Kzs5WXl6f4+Hg5HA7NmDFDqampGj16tCRp3LhxGjx4sH70ox+psLBQHo9H8+fPV05Ojj1j89RTT2nlypWaM2eOnnzySW3dulVvvPGGtmz5v/sH8vLyNHXqVI0cOVJ33323li9frsbGRj3xxBOBPmwAQca9QQCCIeCPz0dERFx2+69//Wv9+Mc/lnR+QcVnnnlGr732mpqampSenq5XXnnF75LVZ599pqefflrbtm1T165dNXXqVC1dulSdO/9fdtu2bZtmz56t3//+9+rdu7cWLFhg76PVypUrtWzZMnk8Hg0fPlxFRUVKSUm5rmPh8Xmg/SAIhR8eo0ewtOX7O+jrCHVkBCGg/SAIhR+CEIKlXa0jBAAA0F4F7akxAAgEZoIABBMzQgAAwFgEIQAAYCwujQFol7gkFv5a/x1z0zRCiRkhAABgLIIQAAAwFpfGALQrXBIDcDMxIwQAAIxFEAIAAMbi0hiAdoFLYubi6TGEEjNCAADAWAQhACHHbBCAUCEIAQAAYxGEAACAsQhCAEKmX/4WLovBxrmAUOCpMQA3HV94ANoLZoQAAICxmBECcNMwEwSgvSEIAQg6AhCA9opLYwCAdoMb6HGzEYQABBVfagDaMy6NAQgKAhCAjoAgBCCgCEAAOhKCEICAIAAhkPhFetwsBCEAN4zwA6CjIwgBaDMCEIBwQRACcE0EHwDhiiAE4IoIQAg17hVCsLGOEIDLIgQBMAEzQgD8EIAAmIQgBEASAQjtG5fIECwEIcBwBCAAJiMIAYYiAAEAN0sDRiIEoaPi3EWgMSMEGIQvEQDwRxACDEAAQjjhxmkEEkEICGMEIAC4OoIQEIYIQABwfbhZGgDQIfXL30Lox7fGjBAQJvhCgKm4ZwjfBkEI6OAIQABw4whCQAdFAAL8MTOEG8E9QkAHw30RwNXx3wfagiAEdCD8Dzxwffg/DLheXBoDOgD+Bx0AgsOIILRq1SotW7ZMHo9Hw4YN08svv6y777471N0CrojgAwTGxf8tcf8QLhb2QWj9+vXKy8tTcXGxUlJStHz5cqWnp6uurk4JCQmh7h5gI/wAwccN1bhYhGVZVqg7EUwpKSkaNWqUVq5cKUlqaWlRUlKSZsyYofz8/Ku+1+fzyel0yuv1yuFw3IzuwgAEHqB9IRSFn7Z8f4f1jNCZM2dUXV2tefPm2dsiIyOVlpamysrKS+qbmprU1NRk/+31eiWdH1DgSoYsfDfUXQDwLfSZveGaNftfTL8JPUGgtH5vX89cT1gHoT/+8Y86d+6cEhMT/bYnJiaqtrb2kvolS5boxRdfvGR7UlJS0PoIAGj/nMtD3QPciK+++kpOp/OqNWEdhNpq3rx5ysvLs/9uaWnRiRMn1KNHD0VERISwZ9fP5/MpKSlJR48eNfpyHuNwHuPAGLRiHM5jHMwYA8uy9NVXX8ntdl+zNqyDUM+ePdWpUyfV19f7ba+vr5fL5bqkPjo6WtHR0X7b4uLigtnFoHE4HGF7grcF43Ae48AYtGIczmMcwn8MrjUT1CqsF1SMiorSiBEjVF5ebm9raWlReXm5UlNTQ9gzAADQHoT1jJAk5eXlaerUqRo5cqTuvvtuLV++XI2NjXriiSdC3TUAABBiYR+EJk+erC+//FIFBQXyeDwaPny4SktLL7mBOlxER0dr4cKFl1ziMw3jcB7jwBi0YhzOYxwYg4uF/TpCAAAAVxLW9wgBAABcDUEIAAAYiyAEAACMRRACAADGIgiFiUWLFikiIsLvNXDgwFB3K+gqKir08MMPy+12KyIiQps2bfJrtyxLBQUF6tWrl2JjY5WWlqZDhw6FprNBdK1x+PGPf3zJ+TF+/PjQdDZIlixZolGjRqlbt25KSEhQRkaG6urq/GpOnz6tnJwc9ejRQ7feeqsmTZp0yYKrHdn1jMH3v//9S86Fp556KkQ9Do7Vq1frzjvvtBcMTE1N1TvvvGO3h/t50Opa42DCuXA9CEJh5M///M/1xRdf2K/f/e53oe5S0DU2NmrYsGFatWrVZdsLCwtVVFSk4uJiVVVVqWvXrkpPT9fp06dvck+D61rjIEnjx4/3Oz9ee+21m9jD4Nu+fbtycnL04YcfqqysTM3NzRo3bpwaGxvtmtmzZ+utt97Shg0btH37dh07dkyPPPJICHsdWNczBpI0bdo0v3OhsLAwRD0Ojt69e2vp0qWqrq7W7t279Zd/+Zf6wQ9+oAMHDkgK//Og1bXGQQr/c+G6WAgLCxcutIYNGxbqboSUJGvjxo323y0tLZbL5bKWLVtmb2toaLCio6Ot1157LQQ9vDkuHgfLsqypU6daP/jBD0LSn1A5fvy4Jcnavn27ZVnn/9136dLF2rBhg11z8OBBS5JVWVkZqm4G1cVjYFmW9Rd/8RfWzJkzQ9epEOnevbv1q1/9ysjz4EKt42BZ5p4LF2NGKIwcOnRIbrdb3/nOd5SVlaUjR46EukshdfjwYXk8HqWlpdnbnE6nUlJSVFlZGcKehca2bduUkJCgAQMG6Omnn9af/vSnUHcpqLxeryQpPj5eklRdXa3m5ma/82HgwIHq06dP2J4PF49Bq3Xr1qlnz54aMmSI5s2bp6+//joU3bspzp07p9dff12NjY1KTU018jyQLh2HViadC1cS9itLmyIlJUVr167VgAED9MUXX+jFF1/U/fffr/3796tbt26h7l5IeDweSbpkFfHExES7zRTjx4/XI488ouTkZH3yySd6/vnn9eCDD6qyslKdOnUKdfcCrqWlRbNmzdK9996rIUOGSDp/PkRFRV3yQ8rhej5cbgwk6bHHHlPfvn3ldru1d+9ezZ07V3V1dfrNb34Twt4G3r59+5SamqrTp0/r1ltv1caNGzV48GDV1NQYdR5caRwkc86FayEIhYkHH3zQ/uc777xTKSkp6tu3r9544w1lZ2eHsGdoDzIzM+1/Hjp0qO68807dfvvt2rZtm8aOHRvCngVHTk6O9u/fb8R9cldypTGYPn26/c9Dhw5Vr169NHbsWH3yySe6/fbbb3Y3g2bAgAGqqamR1+vVf/zHf2jq1Knavn17qLt1011pHAYPHmzMuXAtXBoLU3Fxcfrud7+rjz/+ONRdCRmXyyVJlzwNUl9fb7eZ6jvf+Y569uwZludHbm6uNm/erPfff1+9e/e2t7tcLp05c0YNDQ1+9eF4PlxpDC4nJSVFksLuXIiKitIdd9yhESNGaMmSJRo2bJhWrFhh1HkgXXkcLidcz4VrIQiFqVOnTumTTz5Rr169Qt2VkElOTpbL5VJ5ebm9zefzqaqqyu8auYk+//xz/elPfwqr88OyLOXm5mrjxo3aunWrkpOT/dpHjBihLl26+J0PdXV1OnLkSNicD9cag8upqamRpLA6Fy6npaVFTU1NRpwHV9M6DpdjyrlwMS6NhYlnn31WDz/8sPr27atjx45p4cKF6tSpk6ZMmRLqrgXVqVOn/P7fy+HDh1VTU6P4+Hj16dNHs2bN0uLFi9W/f38lJydrwYIFcrvdysjICF2ng+Bq4xAfH68XX3xRkyZNksvl0ieffKI5c+bojjvuUHp6egh7HVg5OTkqKSnRb3/7W3Xr1s2+38PpdCo2NlZOp1PZ2dnKy8tTfHy8HA6HZsyYodTUVI0ePTrEvQ+Ma43BJ598opKSEk2YMEE9evTQ3r17NXv2bI0ZM0Z33nlniHsfOPPmzdODDz6oPn366KuvvlJJSYm2bdumd99914jzoNXVxsGUc+G6hPqxNQTG5MmTrV69ellRUVHWn/3Zn1mTJ0+2Pv7441B3K+jef/99S9Ilr6lTp1qWdf4R+gULFliJiYlWdHS0NXbsWKuuri60nQ6Cq43D119/bY0bN8667bbbrC5dulh9+/a1pk2bZnk8nlB3O6Aud/ySrF//+td2zTfffGP99Kc/tbp3727dcsst1g9/+EPriy++CF2nA+xaY3DkyBFrzJgxVnx8vBUdHW3dcccd1nPPPWd5vd7QdjzAnnzySatv375WVFSUddttt1ljx4613nvvPbs93M+DVlcbB1POhesRYVmWdTODFwAAQHvBPUIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFgEIQAAYCyCEAAAMBZBCAAAGOv/AVxQpIMC5DMJAAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# plot distribution of ear scores\n", "\n", "import json\n", "import matplotlib.pyplot as plt\n", "\n", "\n", "mean_scores = []\n", "std_scores = []\n", "\n", "for key, val in mean_ear_scores.items():\n", " mean_scores.append(val)\n", "\n", "# plot distribution of ear scores\n", "plt.hist(mean_scores, bins=250)\n", "plt.show()\n" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3,572,640\n" ] } ], "source": [ "# filter each scores in the interval\n", "\n", "min_score = 22\n", "max_score = 28\n", "\n", "filtered_ear_scores_ids = set()\n", "\n", "for key, val in mean_ear_scores.items():\n", " if val >= min_score and val <= max_score:\n", " filtered_ear_scores_ids.add(key)\n", "\n", "print(f\"{len(filtered_ear_scores_ids):,}\")\n", "\n", "# filter ids_set\n" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "discogs_audio_features = pd.read_csv(\"/home/christian/code/christian/metadata/v4/discogs_subset_audio_production_features_v2.csv\")\n", "imslp_audio_features = pd.read_csv(\"/home/christian/code/christian/metadata/v4/imslp_audio_production_features_v2.csv\")\n", "genius_audio_features = pd.read_csv(\"/home/christian/code/christian/metadata/v4/genius_audio_production_features_v2.csv\")\n", "\n", "# Combine all audio features into a single dict indexed by id\n", "# For genius, convert ids to match the format (e.g., prepend \"genius_\" if needed)\n", "audio_features_map = {}\n", "\n", "# Add discogs_subset features\n", "for m in discogs_audio_features.to_dict(orient=\"records\"):\n", " audio_features_map[m[\"id\"]] = m\n", "\n", "# Add imslp features\n", "for m in imslp_audio_features.to_dict(orient=\"records\"):\n", " audio_features_map[m[\"id\"]] = m\n", "\n", "# Add genius features, converting ids as needed (match how we did it for qualit)\n", "for m in genius_audio_features.to_dict(orient=\"records\"):\n", " gid = m[\"id\"]\n", " # Use the mapping from genius_id_to_youtube_id if available, else skip\n", " if gid in genius_id_to_youtube_id:\n", " ytid = genius_id_to_youtube_id[gid]\n", " audio_features_map[ytid] = m\n", "\n", "feature_bounds = {\n", " \"loudness\": [-32, -4],\n", " \"spectral_centroid\": [1750, 5000],\n", " \"spectral_flatness\": [0.02, 0.3],\n", " \"crest_factor\": [1.0, 3],\n", " #\"bass\" : [0.1, 0.5],\n", " #\"mid\" : [0.4, 1.0],\n", " # \"high\" : [0.15, 1.25],\n", " \"stereo_width\" : [0.1, 0.4]\n", "}" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['UldJ9AnalEY',\n", " 'sIul-FKmPCw',\n", " '35wHGGrGQVU',\n", " 'DEJm1igan2Y',\n", " 'ZrGWefub8RU',\n", " 'b-2qwp6cK3Q',\n", " 'MOimfP5NinY',\n", " '1jTFhXhJLnE',\n", " 'IHL_wb0D-0E',\n", " 'j-wpN-lQ8N8']" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "list(audio_features_map.keys())[:10]" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 3572640/3572640 [00:10<00:00, 337601.55it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2745248\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "# now filter based on audio features bounds\n", "from tqdm import tqdm\n", "filtered_ear_scores_ids_by_features = set()\n", "for ytid in tqdm(filtered_ear_scores_ids):\n", " feats = audio_features_map.get(ytid)\n", " if feats is None:\n", " continue\n", " passed = True\n", " for feat, (low, high) in feature_bounds.items():\n", " val = feats.get(feat)\n", " if val is None:\n", " passed = False\n", " break\n", " if not (low <= val <= high):\n", " passed = False\n", " break\n", " if passed:\n", " filtered_ear_scores_ids_by_features.add(ytid)\n", "\n", "print(len(filtered_ear_scores_ids_by_features))\n" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "711,944\n" ] } ], "source": [ "# find the intersection of ids_set and filtered_ear_scores_ids\n", "intersection = ids_set.intersection(filtered_ear_scores_ids)\n", "print(f\"{len(intersection):,}\")\n", "\n", "# filter ids_set\n", "ids_set = intersection\n" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "dict_keys(['diffusion_mix_fix'])\n", "2745248\n", "['753717bd-d5fc-41a6-8f02-92f82196b785', '3e77b2XCHTk', '-P6noUz6nSE', '4be892ee-c0af-4f42-8171-bd58caf82cce', 'XgxYhhtdZC4', '7296d44e-e00e-49ce-856b-7365b0dfa2ca', '5c3ae886-d715-4c6d-9cb3-83fcece24989', 'JBBWJb87W5s', '71d5dd7b-26e1-4677-bcd5-010e187ab591', 'EbnI-wqSzz0']\n" ] } ], "source": [ "data_cut = \"t8\"\n", "#output_filepath = f\"/app2/suno/data/diffusion_mix/dac_vae_tuned_25hz/info_tr_{data_cut}.json\" # cutoff below 20 and above 26\n", "output_filepath = f\"/app2/suno/data/diffusion/dac_vae_tuned_25hz/info_tr_{data_cut}.json\"\n", "\n", "output_dict = {\n", " \"diffusion_mix_fix\": list(filtered_ear_scores_ids_by_features)\n", "}\n", "print(output_dict.keys())\n", "# save out the passed tracksaa\n", "with open(output_filepath, 'w') as f:\n", " json.dump(output_dict, f)\n", "\n", "print(len(output_dict[\"diffusion_mix_fix\"]))\n", "print(output_dict[\"diffusion_mix_fix\"][:10])\n" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "9680870\n" ] } ], "source": [ "# load the train ids \n", "train_metas = read_jsonl(\"/app2/suno/data/diffusion/dac_vae_tuned_25hz/metas_tr_extended_apr30.jsonl\")\n", "print(len(train_metas))\n", "total_train_metas = len(train_metas)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total_filtered_train_metas: 3157078\n", "3157078/9680870 (32.61%)\n", "total_filtered_train_metas_hours: 26309.0\n" ] } ], "source": [ "# now filter based on the ids_set\n", "\n", "filtered_train_metas = []\n", "\n", "for meta in tqdm(train_metas):\n", " if meta[\"id\"] in filtered_ear_scores_ids_by_features:\n", " filtered_train_metas.append(meta)\n", "\n", "total_filtered_train_metas = len(filtered_train_metas)\n", "print(f\"total_filtered_train_metas: {total_filtered_train_metas}\")\n", "print(f\"{total_filtered_train_metas}/{total_train_metas} ({total_filtered_train_metas/total_train_metas*100:.2f}%)\")\n", "\n", "# each meta is 30 seconds, lets convert to hours\n", "total_filtered_train_metas_hours = total_filtered_train_metas * 30 / 3600\n", "print(f\"total_filtered_train_metas_hours: {total_filtered_train_metas_hours:0.1f}\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.9" } }, "nbformat": 4, "nbformat_minor": 2 }