{ "cells": [ { "cell_type": "code", "execution_count": 9, "id": "5b9214bb", "metadata": {}, "outputs": [], "source": [ "import os\n", "import polars as pl\n", "from tqdm import tqdm" ] }, { "cell_type": "code", "execution_count": null, "id": "deae075e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Processing /app2/suno/data/christian/metadata/metas_v9_tr.parquet\n", "1409888\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "1409888it [00:51, 27246.82it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Processing /app2/suno/data/christian/metadata/metas_v9_val.parquet\n" ] }, { "ename": "FileNotFoundError", "evalue": "No such file or directory (os error 2): /app2/suno/data/christian/metadata/metas_v9_val.parquet\n\nThis error occurred with the following context stack:\n\t[1] 'parquet scan'\n\t[2] 'sink'\n", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[11], line 10\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m parquet_filepath \u001b[38;5;129;01min\u001b[39;00m parquet_filepaths:\n\u001b[1;32m 9\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mProcessing \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mparquet_filepath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 10\u001b[0m df \u001b[38;5;241m=\u001b[39m \u001b[43mpl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_parquet\u001b[49m\u001b[43m(\u001b[49m\u001b[43mparquet_filepath\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 11\u001b[0m \u001b[38;5;66;03m# build out a stem index\u001b[39;00m\n\u001b[1;32m 12\u001b[0m \u001b[38;5;66;03m# find all rows where stems is not null\u001b[39;00m\n\u001b[1;32m 13\u001b[0m stems_df \u001b[38;5;241m=\u001b[39m df\u001b[38;5;241m.\u001b[39mfilter(pl\u001b[38;5;241m.\u001b[39mcol(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstems\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39mis_not_null())\n", "File \u001b[0;32m~/miniconda3/envs/suno_env/lib/python3.10/site-packages/polars/_utils/deprecation.py:119\u001b[0m, in \u001b[0;36mdeprecate_renamed_parameter..decorate..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 114\u001b[0m \u001b[38;5;129m@wraps\u001b[39m(function)\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs: P\u001b[38;5;241m.\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: P\u001b[38;5;241m.\u001b[39mkwargs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m T:\n\u001b[1;32m 116\u001b[0m _rename_keyword_argument(\n\u001b[1;32m 117\u001b[0m old_name, new_name, kwargs, function\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__qualname__\u001b[39m, version\n\u001b[1;32m 118\u001b[0m )\n\u001b[0;32m--> 119\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunction\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/miniconda3/envs/suno_env/lib/python3.10/site-packages/polars/_utils/deprecation.py:119\u001b[0m, in \u001b[0;36mdeprecate_renamed_parameter..decorate..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 114\u001b[0m \u001b[38;5;129m@wraps\u001b[39m(function)\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs: P\u001b[38;5;241m.\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: P\u001b[38;5;241m.\u001b[39mkwargs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m T:\n\u001b[1;32m 116\u001b[0m _rename_keyword_argument(\n\u001b[1;32m 117\u001b[0m old_name, new_name, kwargs, function\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__qualname__\u001b[39m, version\n\u001b[1;32m 118\u001b[0m )\n\u001b[0;32m--> 119\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunction\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/miniconda3/envs/suno_env/lib/python3.10/site-packages/polars/io/parquet/functions.py:252\u001b[0m, in \u001b[0;36mread_parquet\u001b[0;34m(source, columns, n_rows, row_index_name, row_index_offset, parallel, use_statistics, hive_partitioning, glob, schema, hive_schema, try_parse_hive_dates, rechunk, low_memory, storage_options, credential_provider, retries, use_pyarrow, pyarrow_options, memory_map, include_file_paths, allow_missing_columns)\u001b[0m\n\u001b[1;32m 249\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 250\u001b[0m lf \u001b[38;5;241m=\u001b[39m lf\u001b[38;5;241m.\u001b[39mselect(columns)\n\u001b[0;32m--> 252\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mlf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/miniconda3/envs/suno_env/lib/python3.10/site-packages/polars/_utils/deprecation.py:93\u001b[0m, in \u001b[0;36mdeprecate_streaming_parameter..decorate..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 89\u001b[0m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mengine\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124min-memory\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 91\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstreaming\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m---> 93\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunction\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/miniconda3/envs/suno_env/lib/python3.10/site-packages/polars/lazyframe/frame.py:2188\u001b[0m, in \u001b[0;36mLazyFrame.collect\u001b[0;34m(self, type_coercion, _type_check, predicate_pushdown, projection_pushdown, simplify_expression, slice_pushdown, comm_subplan_elim, comm_subexpr_elim, cluster_with_columns, collapse_joins, no_optimization, engine, background, _check_order, _eager, **_kwargs)\u001b[0m\n\u001b[1;32m 2186\u001b[0m \u001b[38;5;66;03m# Only for testing purposes\u001b[39;00m\n\u001b[1;32m 2187\u001b[0m callback \u001b[38;5;241m=\u001b[39m _kwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpost_opt_callback\u001b[39m\u001b[38;5;124m\"\u001b[39m, callback)\n\u001b[0;32m-> 2188\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m wrap_df(\u001b[43mldf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollect\u001b[49m\u001b[43m(\u001b[49m\u001b[43mengine\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallback\u001b[49m\u001b[43m)\u001b[49m)\n", "\u001b[0;31mFileNotFoundError\u001b[0m: No such file or directory (os error 2): /app2/suno/data/christian/metadata/metas_v9_val.parquet\n\nThis error occurred with the following context stack:\n\t[1] 'parquet scan'\n\t[2] 'sink'\n" ] } ], "source": [ "parquet_filepaths = [\n", " \"/app2/suno/data/christian/metadata/metas_v9_tr.parquet\",\n", "]\n", "\n", "stem_map = {}\n", "\n", "for parquet_filepath in parquet_filepaths:\n", " print(f\"Processing {parquet_filepath}\")\n", " df = pl.read_parquet(parquet_filepath)\n", " # build out a stem index\n", " # find all rows where stems is not null\n", " stems_df = df.filter(pl.col(\"stems\").is_not_null())\n", " print(len(stems_df))\n", "\n", " # iterate over all the rows in the stems_df\n", " for idx, row in enumerate(tqdm(stems_df.iter_rows(named=True))):\n", " # get the main id\n", " meta_id = row[\"id\"]\n", " stems = row[\"stems\"]\n", " stem_map[meta_id] = stems\n", "\n", "print(len(stem_map))" ] }, { "cell_type": "code", "execution_count": 14, "id": "dc0061e7", "metadata": {}, "outputs": [], "source": [ "# Remove any inner keys from stem_map[key] where the value is None\n", "for k, v in stem_map.items():\n", " # v is expected to be a dict; remove any key in v whose value is None\n", " if isinstance(v, dict):\n", " keys_to_remove = [ik for ik, iv in v.items() if iv is None]\n", " for ik in keys_to_remove:\n", " del v[ik]" ] }, { "cell_type": "code", "execution_count": 15, "id": "cbf42205", "metadata": {}, "outputs": [], "source": [ "# write this metadata to a json file\n", "import json\n", "with open(\"/app2/suno/data/christian/metadata/stems_metadata_v9.json\", \"w\") as f:\n", " json.dump(stem_map, f)\n" ] }, { "cell_type": "code", "execution_count": 8, "id": "e84b038e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 383/1409734 [00:54<55:31:38, 7.05it/s]\n" ] } ], "source": [ "# load the stem map\n", "import json\n", "import os\n", "from tqdm import tqdm\n", "stem_map = json.load(open(\"/app2/suno/data/christian/metadata/stems_metadata_v9.json\"))\n" ] }, { "cell_type": "code", "execution_count": 15, "id": "d1b1e4b9", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 575/1409734 [04:45<194:16:53, 2.01it/s]\n", "100%|██████████| 1409734/1409734 [01:46<00:00, 13294.55it/s]\n" ] } ], "source": [ "# lets do a special one just for trimmed vocals\n", "trimmed_vocals_stem_map = {}\n", "\n", "trimmed_vocals_path = \"/app2/suno/data/sft_stems_12_output_v11_vox_trimmed/\"\n", "\n", "from joblib import Parallel, delayed\n", "\n", "def process_meta_id(meta_id, stems):\n", " for stem_name, stem_path in stems.items():\n", " if stem_name == \"Vocals\" and \"/app2/suno/data/sft_stems_12_output_v11\" in stem_path:\n", " stem_filename = os.path.basename(stem_path)\n", " stem_root, stem_ext = os.path.splitext(stem_filename)\n", " trimmed_stem_filename = f\"{stem_root}_trimmed{stem_ext}\"\n", " trimmed_stem_path = os.path.join(trimmed_vocals_path, trimmed_stem_filename)\n", " #if os.path.exists(trimmed_stem_path):\n", " return (meta_id, trimmed_stem_path)\n", " return None\n", "\n", "results = Parallel(n_jobs=-1, backend=\"threading\")(\n", " delayed(process_meta_id)(meta_id, stems)\n", " for meta_id, stems in tqdm(stem_map.items())\n", ")\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 17, "id": "7619af99", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "901438\n" ] } ], "source": [ "# Only keep results that are not None\n", "filtered_results = [res for res in results if res is not None]\n", "\n", "# Now unpack\n", "trimmed_vocals_stem_map = {meta_id: trimmed_path for meta_id, trimmed_path in filtered_results}\n", "\n", "print(len(trimmed_vocals_stem_map))" ] }, { "cell_type": "code", "execution_count": null, "id": "11ad37a0", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 901437/901437 [11:03<00:00, 1357.99it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "901437\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "# check if the filepaths exist, # if not, remove from the stem_map, \n", "filtered_trimmed_vocals_stem_map = {}\n", "for meta_id, trimmed_path in tqdm(trimmed_vocals_stem_map.items()):\n", " if os.path.exists(trimmed_path):\n", " filtered_trimmed_vocals_stem_map[meta_id] = trimmed_path\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 24, "id": "43259f10", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "479620\n" ] } ], "source": [ "\n", "print(len(filtered_trimmed_vocals_stem_map))\n", "\n", "# save this to a json file\n", "import json\n", "with open(\"/app2/suno/data/christian/metadata/trimmed_vocals_stem_map_v9.json\", \"w\") as f:\n", " json.dump(filtered_trimmed_vocals_stem_map, f)\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 14, "id": "9e18b97e", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'Vocals': '/app2/suno/data/cover_stems/O9tswMhdmJ4/Vocals.opus',\n", " 'Drums': '/app2/suno/data/cover_stems/O9tswMhdmJ4/Drums.opus',\n", " 'Guitar': '/app2/suno/data/cover_stems/O9tswMhdmJ4/Guitar.opus',\n", " 'Percussion': '/app2/suno/data/cover_stems/O9tswMhdmJ4/Percussion.opus',\n", " 'FX': '/app2/suno/data/cover_stems/O9tswMhdmJ4/FX.opus'}" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "stem_map[list(stem_map.keys())[0]]" ] }, { "cell_type": "code", "execution_count": 37, "id": "fd7edf82", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loaded /home/christian/code/christian/metadata/v4/genius_audio_production_features_v2.csv: 2090009 rows\n", "Loaded /home/christian/code/christian/metadata/v4/discogs_subset_audio_production_features_v2.csv: 2916657 rows\n", "Loaded /home/christian/code/christian/metadata/v4/imslp_audio_production_features_v2.csv: 278620 rows\n", "Combined dataframe has 5285286 rows\n", "After removing duplicates, dataframe has 4895862 rows\n" ] } ], "source": [ "# audio production metadata\n", "import pandas as pd\n", "\n", "genius_filepath = \"/home/christian/code/christian/metadata/v4/genius_audio_production_features_v2.csv\"\n", "discogs_filepath = \"/home/christian/code/christian/metadata/v4/discogs_subset_audio_production_features_v2.csv\"\n", "imslp_filepath = \"/home/christian/code/christian/metadata/v4/imslp_audio_production_features_v2.csv\"\n", "\n", "# Read the dataframes\n", "dfs = []\n", "for filepath in [genius_filepath, discogs_filepath, imslp_filepath]:\n", " df = pd.read_csv(filepath)\n", " dfs.append(df)\n", " print(f\"Loaded {filepath}: {len(df)} rows\")\n", "\n", "# Merge into one dataframe\n", "combined_df = pd.concat(dfs, ignore_index=True)\n", "print(f\"Combined dataframe has {len(combined_df)} rows\")\n", "\n", "# remove any rows where the id is duplicated\n", "combined_df = combined_df.drop_duplicates(subset=\"id\")\n", "print(f\"After removing duplicates, dataframe has {len(combined_df)} rows\")\n", "\n", "# save this into one csv file\n", "combined_df.to_csv(\"/home/christian/code/christian/metadata/v4/genius_discogs_imslp_audio_production_features_v2.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "f61b05dc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of unique ids: 4895862\n", "Rows where the ids are copied (appear more than once):\n", " id spectral_centroid bass mid high \\\n", "1391927 ---2ayLSzvo 3371.313721 0.354725 0.421632 0.517771 \n", "3427491 ---2ayLSzvo 3371.313721 0.354725 0.421632 0.517771 \n", "4765668 ---a4A0MqbI 1732.193481 0.210528 0.823635 0.222957 \n", "1451815 ---a4A0MqbI 1732.193481 0.210528 0.823635 0.222957 \n", "345652 ---jlZoyPGQ 3579.635498 0.294456 0.848636 0.862382 \n", "2248930 ---jlZoyPGQ 3579.635498 0.294456 0.848636 0.862382 \n", "473364 --00o6iIWg8 2717.190186 0.444563 0.878516 0.622754 \n", "3634036 --00o6iIWg8 2717.190186 0.444563 0.878516 0.622754 \n", "3379338 --0KrtoupMI 3084.461182 0.259746 0.892174 0.813724 \n", "116049 --0KrtoupMI 3084.461182 0.259746 0.892174 0.813724 \n", "\n", " crest_factor stereo_width spectral_flatness silence_percentage \\\n", "1391927 1.766134 0.072733 0.087815 0.286205 \n", "3427491 1.766134 0.072733 0.087815 0.286205 \n", "4765668 1.987843 0.190238 0.006272 3.400309 \n", "1451815 1.987843 0.190238 0.006272 3.400309 \n", "345652 1.348450 0.188308 0.089433 1.279911 \n", "2248930 1.348450 0.188308 0.089433 1.279911 \n", "473364 1.491839 0.060057 0.086763 0.000000 \n", "3634036 1.491839 0.060057 0.086763 0.000000 \n", "3379338 1.379445 0.225887 0.060117 0.649773 \n", "116049 1.379445 0.225887 0.060117 0.649773 \n", "\n", " loudness \n", "1391927 -12.400481 \n", "3427491 -12.400481 \n", "4765668 -14.203429 \n", "1451815 -14.203429 \n", "345652 -6.430933 \n", "2248930 -6.430933 \n", "473364 -11.564868 \n", "3634036 -11.564868 \n", "3379338 -7.431763 \n", "116049 -7.431763 \n" ] } ], "source": [] }, { "cell_type": "code", "execution_count": 28, "id": "59f8804d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idspectral_centroidbassmidhighcrest_factorstereo_widthspectral_flatnesssilence_percentageloudness
0OzrkFekIbxE3132.6977540.3035810.6495340.6012011.3907180.0920430.1331380.041667-8.416751
1hxftunYgwm82779.5632320.4022060.5995640.4983021.3296930.1013600.1080090.083333-8.857994
2jvt3i_RN-E02948.6259770.3287960.5962930.5649281.4656660.1234060.1008330.000000-7.800515
3QbuhWGSmp203343.8144530.3892720.6407920.6016471.3494170.0959010.1182552.031603-7.785441
4lfFwVsK_pJI3250.3439940.3157960.6762090.7548031.4089120.0569170.1053690.000000-7.879766
\n", "
" ], "text/plain": [ " id spectral_centroid bass mid high crest_factor \\\n", "0 OzrkFekIbxE 3132.697754 0.303581 0.649534 0.601201 1.390718 \n", "1 hxftunYgwm8 2779.563232 0.402206 0.599564 0.498302 1.329693 \n", "2 jvt3i_RN-E0 2948.625977 0.328796 0.596293 0.564928 1.465666 \n", "3 QbuhWGSmp20 3343.814453 0.389272 0.640792 0.601647 1.349417 \n", "4 lfFwVsK_pJI 3250.343994 0.315796 0.676209 0.754803 1.408912 \n", "\n", " stereo_width spectral_flatness silence_percentage loudness \n", "0 0.092043 0.133138 0.041667 -8.416751 \n", "1 0.101360 0.108009 0.083333 -8.857994 \n", "2 0.123406 0.100833 0.000000 -7.800515 \n", "3 0.095901 0.118255 2.031603 -7.785441 \n", "4 0.056917 0.105369 0.000000 -7.879766 " ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head()" ] }, { "cell_type": "code", "execution_count": null, "id": "8363c9e3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 32, "id": "8679a002", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "5280941\n", "\n", "Filtering Results:\n", "Total unfiltered entries: 5,280,941\n", "Total filtered entries: 2,528,997\n", "Total audio stats filtered entries: 2,110,817\n", "Total filtered duration: 151166.16 hours\n", "\n", "Filtering breakdown:\n", " Duration filtered: 146,125 entries\n", " Audio stats filtered: 2,110,817 entries\n", " Loudness filtered: 272,032 entries\n", " Silence filtered: 222,970 entries\n", " CER filtered: 0 entries\n", "Wrote 2,528,997 filtered metas to /home/christian/code/christian/metadata/diffusion_pretrain/filtered_metas_v0.jsonl\n" ] } ], "source": [ "from suno_utils.utils.text import read_jsonl, write_jsonl\n", "\n", "output_filepath = \"/home/christian/code/christian/metadata/diffusion_pretrain/unfiltered_metas_v0.jsonl\"\n", "output_metas = read_jsonl(output_filepath)\n", "print(len(output_metas))\n", "\n", "# ------------------------------------------------------------\n", "# filter the metas\n", "# ------------------------------------------------------------\n", "# now apply the filtering here\n", "# filter on\n", "# 1. duration (must be between 30 and 480 seconds)\n", "# 2. audio stats\n", "# - loudness (must be between -20 and -6)\n", "# - silence_percentage (must be less than 4%)\n", "# 3. if we have alignments, then we ensure CER is less than 0.8\n", "\n", "filtered_metas = []\n", "total_duration_filtered = 0.0\n", "\n", "# Track filtering statistics\n", "duration_filtered_count = 0\n", "loudness_filtered_count = 0\n", "silence_filtered_count = 0\n", "cer_filtered_count = 0\n", "audio_stats_filtered_count = 0\n", "\n", "for meta in tqdm(output_metas):\n", " # Duration filter\n", " if meta[\"duration_s\"] < 30 or meta[\"duration_s\"] > 480:\n", " duration_filtered_count += 1\n", " continue\n", "\n", " # check if we have audio_stats, if we dont, skip\n", " if \"audio_stats\" not in meta:\n", " audio_stats_filtered_count += 1\n", " continue\n", "\n", " # Loudness filter\n", " if (\n", " meta[\"audio_stats\"][\"loudness\"] < -20\n", " or meta[\"audio_stats\"][\"loudness\"] > -6\n", " ):\n", " loudness_filtered_count += 1\n", " continue\n", "\n", " # Silence filter\n", " if meta[\"audio_stats\"][\"silence_percentage\"] > 4:\n", " silence_filtered_count += 1\n", " continue\n", "\n", " # CER filter (if alignments exist)\n", " if meta.get(\"text_aligned\"):\n", " if meta[\"cer\"] > 0.8:\n", " cer_filtered_count += 1\n", " continue\n", "\n", " # Track duration for passed entries\n", " total_duration_filtered += meta[\"duration_s\"]\n", " filtered_metas.append(meta)\n", "\n", "# write the current filtered metas to\n", "out_filepath = \"/home/christian/code/christian/metadata/diffusion_pretrain/filtered_metas_v0.jsonl\"\n", "write_jsonl(filtered_metas, out_filepath)\n", "\n", "print(f\"\\nFiltering Results:\")\n", "print(f\"Total unfiltered entries: {len(output_metas):,}\")\n", "print(f\"Total filtered entries: {len(filtered_metas):,}\")\n", "print(f\"Total audio stats filtered entries: {audio_stats_filtered_count:,}\")\n", "#print(f\"Total unfiltered duration: {total_duration_unfiltered/3600:.2f} hours\")\n", "print(f\"Total filtered duration: {total_duration_filtered/3600:.2f} hours\")\n", "#print(\n", "# f\"Filtered percentage: {(total_duration_filtered/total_duration_unfiltered*100):.1f}%\"\n", "#)\n", "print(f\"\\nFiltering breakdown:\")\n", "print(f\" Duration filtered: {duration_filtered_count:,} entries\")\n", "print(f\" Audio stats filtered: {audio_stats_filtered_count:,} entries\")\n", "print(f\" Loudness filtered: {loudness_filtered_count:,} entries\")\n", "print(f\" Silence filtered: {silence_filtered_count:,} entries\")\n", "print(f\" CER filtered: {cer_filtered_count:,} entries\")\n", "print(f\"Wrote {len(filtered_metas):,} filtered metas to {out_filepath}\")" ] }, { "cell_type": "code", "execution_count": 18, "id": "26a8b891", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Shuffling 4,061,232 filtered metas...\n", "Train size: 4,020,620 (99.0%)\n", "Validation size: 40,612 (1.0%)\n", "\n", "Duration Summary:\n", "Train duration: 248736.29 hours (99.0%)\n", "Validation duration: 2510.63 hours (1.0%)\n", "\n", "Writing train metas to: /app2/suno/data/diffusion/v1/metas_diff_v1_tr.jsonl\n", "Writing validation metas to: /app2/suno/data/diffusion/v1/metas_diff_v1_val.jsonl\n", "\n", "✅ Successfully split and saved:\n", " Train: 4,020,620 entries (248736.29 hours)\n", " Validation: 40,612 entries (2510.63 hours)\n", " Total: 4,061,232 entries (251246.93 hours)\n" ] } ], "source": [ "# Split filtered_metas into train and validation sets\n", "import random\n", "import os\n", "from suno_utils.utils.text import write_jsonl\n", "\n", "# Set random seed for reproducibility\n", "random.seed(42)\n", "\n", "# Shuffle the filtered_metas list\n", "print(f\"Shuffling {len(filtered_metas):,} filtered metas...\")\n", "shuffled_metas = filtered_metas.copy()\n", "random.shuffle(shuffled_metas)\n", "\n", "# Calculate split sizes (1% for validation)\n", "val_size = int(len(shuffled_metas) * 0.01)\n", "train_size = len(shuffled_metas) - val_size\n", "\n", "print(f\"Train size: {train_size:,} ({train_size/len(shuffled_metas)*100:.1f}%)\")\n", "print(f\"Validation size: {val_size:,} ({val_size/len(shuffled_metas)*100:.1f}%)\")\n", "\n", "# Split the data\n", "train_metas = shuffled_metas[:train_size]\n", "val_metas = shuffled_metas[train_size:]\n", "\n", "# Calculate durations for each split\n", "train_duration = sum(meta[\"duration_s\"] for meta in train_metas)\n", "val_duration = sum(meta[\"duration_s\"] for meta in val_metas)\n", "total_duration = train_duration + val_duration\n", "\n", "print(f\"\\nDuration Summary:\")\n", "print(f\"Train duration: {train_duration/3600:.2f} hours ({train_duration/total_duration*100:.1f}%)\")\n", "print(f\"Validation duration: {val_duration/3600:.2f} hours ({val_duration/total_duration*100:.1f}%)\")\n", "\n", "# Define output directory and filenames\n", "output_dir = \"/app2/suno/data/diffusion/v1/\"\n", "train_filename = \"metas_diff_v1_tr.jsonl\"\n", "val_filename = \"metas_diff_v1_val.jsonl\"\n", "\n", "# Ensure output directory exists\n", "os.makedirs(output_dir, exist_ok=True)\n", "\n", "# Write train and validation files\n", "train_filepath = os.path.join(output_dir, train_filename)\n", "val_filepath = os.path.join(output_dir, val_filename)\n", "\n", "print(f\"\\nWriting train metas to: {train_filepath}\")\n", "write_jsonl(train_metas, train_filepath)\n", "\n", "print(f\"Writing validation metas to: {val_filepath}\")\n", "write_jsonl(val_metas, val_filepath)\n", "\n", "print(f\"\\n✅ Successfully split and saved:\")\n", "print(f\" Train: {len(train_metas):,} entries ({train_duration/3600:.2f} hours)\")\n", "print(f\" Validation: {len(val_metas):,} entries ({val_duration/3600:.2f} hours)\")\n", "print(f\" Total: {len(filtered_metas):,} entries ({total_duration/3600:.2f} hours)\")\n" ] }, { "cell_type": "code", "execution_count": 15, "id": "52c277df", "metadata": {}, "outputs": [], "source": [ "from typing import List, Dict, Tuple, Optional, Any\n", "\n", "def _build_artist_vox_mappings(\n", " metas: List[Dict],\n", ") -> Tuple[Dict[str, List[int]], Dict[str, Optional[Dict[str, Any]]]]:\n", " \"\"\"\n", " Build mappings from artist IDs to metadata indices and vox stem paths with duration.\n", "\n", " Args:\n", " metas: List of metadata dictionaries\n", "\n", " Returns:\n", " Tuple of (artist_id_to_meta_idx, artist_id_to_vox_paths)\n", " - artist_id_to_meta_idx: Maps artist_id to list of meta indices\n", " - artist_id_to_vox_paths: Maps artist_id to dict with 'path' and 'duration_s' (or None)\n", " \"\"\"\n", " # Build mapping of artist_ids to meta indices\n", " artist_id_to_meta_idx = {}\n", " for idx, meta in enumerate(metas):\n", " if \"artists\" in meta and meta[\"artists\"] is not None:\n", " if len(meta[\"artists\"]) == 1: # only do if there is a single artist\n", " for artist_id in meta[\"artists\"]:\n", " if artist_id not in artist_id_to_meta_idx:\n", " artist_id_to_meta_idx[artist_id] = []\n", " artist_id_to_meta_idx[artist_id].append(idx)\n", "\n", " # Build mapping of artist_ids to vox stem paths with duration\n", " artist_id_to_vox_paths = {}\n", " for artist_id, meta_indices in artist_id_to_meta_idx.items():\n", " for meta_idx in meta_indices:\n", " meta = metas[meta_idx]\n", " if meta.get(\"vox_stem\") is not None:\n", " if artist_id not in artist_id_to_vox_paths:\n", " artist_id_to_vox_paths[artist_id] = []\n", " artist_id_to_vox_paths[artist_id].append(meta[\"vox_stem\"])\n", "\n", " return artist_id_to_vox_paths" ] }, { "cell_type": "code", "execution_count": 2, "id": "3e1176a2", "metadata": {}, "outputs": [ { "ename": "NameError", "evalue": "name 'filtered_metas' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[2], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m artist_id_to_vox_paths \u001b[38;5;241m=\u001b[39m _build_artist_vox_mappings(\u001b[43mfiltered_metas\u001b[49m)\n", "\u001b[0;31mNameError\u001b[0m: name 'filtered_metas' is not defined" ] } ], "source": [ "artist_id_to_vox_paths = _build_artist_vox_mappings(filtered_metas)" ] }, { "cell_type": "code", "execution_count": null, "id": "ebc68a95", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "4061232\n" ] }, { "ename": "NameError", "evalue": "name '_build_artist_vox_mappings' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[2], line 5\u001b[0m\n\u001b[1;32m 3\u001b[0m filtered_metas \u001b[38;5;241m=\u001b[39m read_jsonl(filtered_metas_filepath)\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;28mlen\u001b[39m(filtered_metas))\n\u001b[0;32m----> 5\u001b[0m artist_id_to_vox_paths \u001b[38;5;241m=\u001b[39m \u001b[43m_build_artist_vox_mappings\u001b[49m(filtered_metas)\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;28mlen\u001b[39m(artist_id_to_vox_paths))\n", "\u001b[0;31mNameError\u001b[0m: name '_build_artist_vox_mappings' is not defined" ] } ], "source": [ "from suno_utils.utils.text import read_jsonl\n", "filtered_metas_filepath = \"/home/christian/code/christian/metadata/diffusion_pretrain/filtered_metas_v0.jsonl\"\n", "filtered_metas = read_jsonl(filtered_metas_filepath)\n", "print(len(filtered_metas))\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 17, "id": "54b129de", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "41837\n", "Total number of vox_paths: 372,570\n" ] } ], "source": [ "artist_id_to_vox_paths = _build_artist_vox_mappings(filtered_metas)\n", "print(len(artist_id_to_vox_paths))\n", "# total number of vox_paths\n", "total_vox_paths = sum(len(vox_paths) for vox_paths in artist_id_to_vox_paths.values())\n", "print(f\"Total number of vox_paths: {total_vox_paths:,}\")\n" ] }, { "cell_type": "code", "execution_count": 14, "id": "d0263ad4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['x5hm5dg3']\n" ] } ], "source": [ "for meta in filtered_metas:\n", " if \"artists\" in meta:\n", " print(meta[\"artists\"])\n", " break\n" ] }, { "cell_type": "code", "execution_count": null, "id": "5298c19b", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.9" } }, "nbformat": 4, "nbformat_minor": 5 }