{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "d94790de", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.text import read_jsonl, write_jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "60f1ba15", "metadata": {}, "outputs": [], "source": [ "import polars as pl\n", "\n", "# dataset sources\n", "# discogs_subset, 2.9M\n", "# genius, 2.1M\n", "# imslp, 200k\n", "# deezer, 700k\n", "# podcast,\n", "# sfx, 5M\n", "\n", "# Define all dataset parquet filepaths in a dictionary for easier loading and management\n", "parquet_filepaths = {\n", " #\"discogs_subset\": \"/app2/suno/data/christian/metadata/raw_discogs_subset_metas.parquet\",\n", " #\"genius\": \"/app2/suno/data/christian/metadata/raw_genius_metas.parquet\",\n", " \"genius\": \"/app2/suno/data/christian/metadata/genius_hq_metas_plus.parquet\",\n", " #\"imslp\": \"/app2/suno/data/christian/metadata/raw_imslp_metas.parquet\",\n", " #\"deezer\": \"/app2/suno/data/christian/metadata/raw_deezer_metas.parquet\",\n", " #\"sfx\": \"/app2/suno/data/christian/metadata/combined_v3_w_extreme_metas_v0_aligned.parquet\"\n", "}\n", "\n", "# Load the parquet files into Polars DataFrames and keep them in a dictionary\n", "dfs = {}\n", "for key, path in parquet_filepaths.items():\n", " try:\n", " dfs[key] = pl.read_parquet(path)\n", " print(f\"Loaded {key} ({dfs[key].shape[0]:,} rows)\")\n", " except Exception as e:\n", " print(f\"Error loading {key} from {path}: {e}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "7b0d4e7f", "metadata": {}, "outputs": [], "source": [ "# create a special map for genius that maps id to original id\n", "genius_df = dfs.get(\"genius\")\n", "genius_id_map = {}\n", "for row in genius_df.iter_rows(named=True):\n", " genius_id_map[row[\"id\"]] = row[\"original_id\"]\n" ] }, { "cell_type": "code", "execution_count": null, "id": "8a1321a8", "metadata": {}, "outputs": [], "source": [ "# If you want the discogs_subset DataFrame specifically:\n", "genius_df = dfs.get(\"genius\")" ] }, { "cell_type": "code", "execution_count": null, "id": "a4bd06f9", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.text import read_jsonl\n", "\n", "version = \"5\"\n", "\n", "# load some alignemnts info (h5 alignments)\n", "genius_alignments_filepath = (\n", " f\"/home/tony/Work/tony/hoot/tmp/genius_hq_alignments_h5_t480_v{version}.jsonl\"\n", ")\n", "discogs_alignments_filepath = (\n", " f\"/home/tony/Work/tony/hoot/tmp/discogs_hq_alignments_h5_t480_v{version}.jsonl\"\n", ")\n", "deezer_alignments_filepath = (\n", " f\"/home/tony/Work/tony/hoot/tmp/deezer_hq_alignments_h5_t480_v{version}.jsonl\"\n", ")\n", "\n", "genius_alignments = read_jsonl(genius_alignments_filepath, progress=False)\n", "print(len(genius_alignments))\n", "discogs_alignments = read_jsonl(discogs_alignments_filepath, progress=False)\n", "print(len(discogs_alignments))\n", "deezer_alignments = read_jsonl(deezer_alignments_filepath, progress=False)\n", "print(len(deezer_alignments))\n", "\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "16cd53a1", "metadata": {}, "outputs": [], "source": [ "\n", "def build_alignment_map(data):\n", " result = {}\n", " for k, v, cer in data:\n", " meta = v[0]\n", " lines, starts, ends = meta[\"line_text\"], meta[\"line_start_s\"], meta[\"line_end_s\"]\n", "\n", " line_entries = []\n", " for text, start, end in zip(lines, starts, ends):\n", " if start is None or end is None:\n", " continue\n", " line_entries.append((\n", " start,\n", " end,\n", " text,\n", " ))\n", "\n", " result[k] = {\n", " \"lines\": line_entries,\n", " \"cer\": cer,\n", " \"text\": meta.get(\"text\"),\n", " \"start_s\": meta.get(\"start_s\"),\n", " \"end_s\": meta.get(\"end_s\"),\n", " \"vocal_start_s\": meta.get(\"vocal_start_s\"),\n", " \"vocal_end_s\": meta.get(\"vocal_end_s\")\n", " }\n", " return result\n", "\n", "\n", "# usage\n", "genius_alignments_map = build_alignment_map(genius_alignments)\n", "discogs_alignments_map = build_alignment_map(discogs_alignments)\n", "deezer_alignments_map = build_alignment_map(deezer_alignments)\n", "\n", "# adjust the genius_alignments_map to have the original_id as the key\n", "#genius_alignments_map = {genius_id_map[k]: v for k, v in genius_alignments_map.items()}" ] }, { "cell_type": "code", "execution_count": null, "id": "697638d7", "metadata": {}, "outputs": [], "source": [ "# adjust the genius_alignments_map to have the original_id as the key\n", "genius_alignments_map = {genius_id_map[k]: v for k, v in genius_alignments_map.items()}\n" ] }, { "cell_type": "code", "execution_count": null, "id": "d05fd7d5", "metadata": {}, "outputs": [], "source": [ "# improve data for diffusion\n", "# augment with pitch, test how slow\n", "# ensure only one artist id is present\n", "# audio loader, async\n", "# evaluation for vox diffusion benchmark\n", "# train on speaker embed as seprate experiment\n", "# how to make speech to vox task?\n", "\n", "\n", "\n", "# semantic\n", "# rvq, hart, \n", "# quantized, \n", "# use codec, 4 codebook rvq as semantic, 16khz stereo audio, train on 4 codebooks\n", "# (similar to vq-vae on images)\n", "\n", "# scale up gpt, memorization, then try data reweighting\n", "\n", "# data for codec, sfx, stems, learnablility" ] }, { "cell_type": "code", "execution_count": null, "id": "4fc207f9", "metadata": {}, "outputs": [], "source": [ "\n", "# extract sets of IDs\n", "genius_ids = set(genius_alignments_map.keys())\n", "discogs_ids = set(discogs_alignments_map.keys())\n", "deezer_ids = set(deezer_alignments_map.keys())\n", "\n", "# pairwise overlaps\n", "genius_discogs_overlap = genius_ids & discogs_ids\n", "genius_deezer_overlap = genius_ids & deezer_ids\n", "discogs_deezer_overlap = discogs_ids & deezer_ids\n", "\n", "# triple overlap\n", "all_three_overlap = genius_ids & discogs_ids & deezer_ids\n", "\n", "# summary counts\n", "print(f\"Genius total: {len(genius_ids)}\")\n", "print(f\"Discogs total: {len(discogs_ids)}\")\n", "print(f\"Deezer total: {len(deezer_ids)}\")\n", "print()\n", "print(f\"Genius ∩ Discogs: {len(genius_discogs_overlap)}\")\n", "print(f\"Genius ∩ Deezer: {len(genius_deezer_overlap)}\")\n", "print(f\"Discogs ∩ Deezer: {len(discogs_deezer_overlap)}\")\n", "print(f\"All three overlap: {len(all_three_overlap)}\")\n", "\n", "# optionally inspect example IDs\n", "print(\"\\nExample overlaps:\")\n", "print(\"Genius ∩ Discogs:\", list(genius_discogs_overlap)[:10])\n", "print(\"All three:\", list(all_three_overlap)[:10])\n", "\n", "# also print the total number of unique ids across all three\n", "print(f\"Total unique ids across all three: {len(genius_ids | discogs_ids | deezer_ids):,}\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "a41197b0", "metadata": {}, "outputs": [], "source": [ "meta_id = \"Ew8T9MRCJuU\"\n", "\n", "# reverse genius id map\n", "\n", "\n", "# Use only Polars for retrieving row as dict (avoiding .to_pandas(), so no pyarrow needed)\n", "genius_df = dfs.get(\"genius\").filter(pl.col(\"id\") == meta_id)\n", "genius_meta = genius_df.row(0) if genius_df.height > 0 else None\n", "if genius_meta is not None:\n", " genius_meta = dict(zip(genius_df.columns, genius_meta))\n", "print(genius_meta)\n", "\n", "# get from discogs \n", "discogs_df = dfs.get(\"discogs_subset\").filter(pl.col(\"id\") == meta_id)\n", "discogs_meta = discogs_df.row(0) if discogs_df.height > 0 else None\n", "if discogs_meta is not None:\n", " discogs_meta = dict(zip(discogs_df.columns, discogs_meta))\n", "print(discogs_meta)\n", "\n", "# get from deezer\n", "deezer_df = dfs.get(\"deezer\").filter(pl.col(\"id\") == meta_id)\n", "deezer_meta = deezer_df.row(0) if deezer_df.height > 0 else None\n", "if deezer_meta is not None:\n", " deezer_meta = dict(zip(deezer_df.columns, deezer_meta))\n", "print(deezer_meta)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "5ed3e66d", "metadata": {}, "outputs": [], "source": [ "parquet_filepath = \"/app2/suno/data/christian/metadata/metas_v9_tr.parquet\"\n", "df = pl.read_parquet(parquet_filepath)" ] }, { "cell_type": "code", "execution_count": null, "id": "624e29ea", "metadata": {}, "outputs": [], "source": [ "df" ] }, { "cell_type": "code", "execution_count": null, "id": "54164073", "metadata": {}, "outputs": [], "source": [ "# Filter the DataFrame `df` to only include rows where at least one of\n", "# \"text\", \"text_aligned\", \"stems\", or \"tags\" is not null, AND \"weight\" is not null\n", "df_filtered = df.filter(\n", " (\n", " pl.col(\"text\").is_not_null() |\n", " pl.col(\"text_aligned\").is_not_null() |\n", " pl.col(\"stems\").is_not_null() |\n", " pl.col(\"tags\").is_not_null()\n", " )\n", ")\n", "print(df_filtered.height)" ] }, { "cell_type": "code", "execution_count": null, "id": "b7f3730c", "metadata": {}, "outputs": [], "source": [ "# get all rows that start with \"podcast\"\n", "podcast_metas = df.filter(pl.col(\"id\").str.starts_with(\"podcast\"))\n", "\n", "print(len(podcast_metas))" ] }, { "cell_type": "code", "execution_count": null, "id": "90cc9c1d", "metadata": {}, "outputs": [], "source": [ "# count the number of rows that have text_aligned and print\n", "#print(df_filtered.filter(pl.col(\"text_aligned\").is_not_null()).height) # about 3.2M\n", "\n", "# count the number of rows that have text and print\n", "#print(df_filtered.filter(pl.col(\"text\").is_not_null()).height) # about 7.6M\n", "\n", "# can we print the first 10 rows of df_filtered with text_aligned\n", "#df_filtered.filter(pl.col(\"text\").is_not_null()).head(10)" ] }, { "cell_type": "code", "execution_count": null, "id": "79cae7b5", "metadata": {}, "outputs": [], "source": [ "# load audio quality scores\n", "discogs_subset_ear_scores = pl.read_csv(\"/home/christian/code/christian/metadata/organized/ear/discogs_subset_ear_scores.csv\")\n", "genius_ear_scores = pl.read_csv(\"/home/christian/code/christian/metadata/organized/ear/genius_ear_scores.csv\")\n", "imslp_ear_scores = pl.read_csv(\"/home/christian/code/christian/metadata/organized/ear/imslp_ear_scores.csv\")\n", "\n", "# convert the column \"mean_score\" to \"ear_score\" in all the dataframes\n", "discogs_subset_ear_scores = discogs_subset_ear_scores.rename({\"mean_score\": \"ear_score\"})\n", "genius_ear_scores = genius_ear_scores.rename({\"mean_score\": \"ear_score\"})\n", "#imslp_ear_scores = imslp_ear_scores.rename({\"mean_score\": \"ear_score\"})\n", "\n", "# merge the df_filtered with the ear scores\n", "df_filtered = df_filtered.join(discogs_subset_ear_scores, on=\"id\", how=\"left\")\n", "df_filtered = df_filtered.join(genius_ear_scores, on=\"id\", how=\"left\")\n", "#df_filtered = df_filtered.join(imslp_ear_scores, on=\"id\", how=\"left\")" ] }, { "cell_type": "code", "execution_count": null, "id": "aae1fe7e", "metadata": {}, "outputs": [], "source": [ "# iterate over the rows in the df\n", "from tqdm import tqdm\n", "\n", "new_metas = []\n", "missing_alignments = []\n", "with_alignments = []\n", "\n", "# i want to reset weights to 1 for all rows\n", "\n", "# this is just going to repair the metas alignment issues\n", "# i guess only use text when we have alignments? \n", "for i, row in tqdm(enumerate(df_filtered.iter_rows(named=True)), total=df_filtered.height, desc=\"Processing rows\"):\n", "\n", " weight = row.get(\"weight\", 1)\n", " if weight is None:\n", " weight = 1\n", " \n", " if weight < 1:\n", " continue\n", "\n", " # check if this has text_aligned\n", " has_text_aligned = row[\"text_aligned\"] is not None\n", " have_genius_aligned = False\n", " have_discogs_aligned = False\n", " have_deezer_aligned = False\n", " have_podcast_aligned = True if \"podcast\" in row[\"id\"] else False\n", " genius_cer = None\n", " discogs_cer = None\n", " deezer_cer = None\n", " # check if the id is in the alignments map\n", " if row[\"id\"] in genius_alignments_map:\n", " #print(row[\"id\"])\n", " #print(row)\n", " #print(genius_alignments_map[row[\"id\"]])\n", " have_genius_aligned = True \n", " genius_cer = genius_alignments_map[row[\"id\"]][\"cer\"]\n", " genius_text = genius_alignments_map[row[\"id\"]][\"text\"]\n", " genius_alignments = genius_alignments_map[row[\"id\"]][\"lines\"]\n", " elif row[\"id\"] in discogs_alignments_map:\n", " #print(row[\"id\"])\n", " #print(row)\n", " #print(discogs_alignments_map[row[\"id\"]])\n", " have_discogs_aligned = True\n", " discogs_cer = discogs_alignments_map[row[\"id\"]][\"cer\"]\n", " discogs_text = discogs_alignments_map[row[\"id\"]][\"text\"]\n", " discogs_alignments = discogs_alignments_map[row[\"id\"]][\"lines\"]\n", " elif row[\"id\"] in deezer_alignments_map:\n", " #print(row[\"id\"])\n", " #print(row)\n", " #print(deezer_alignments_map[row[\"id\"]])\n", " have_deezer_aligned = True\n", " deezer_cer = deezer_alignments_map[row[\"id\"]][\"cer\"]\n", " deezer_text = deezer_alignments_map[row[\"id\"]][\"text\"]\n", " deezer_alignments = deezer_alignments_map[row[\"id\"]][\"lines\"]\n", "\n", " # has alignments\n", " new_meta = row\n", "\n", " # set the weight to 1\n", " new_meta[\"weight\"] = 1\n", "\n", " # only use text_aligned if it comes from genius, discogs, or deezer\n", " # otherwise we null the text, and text_aligned\n", " # also we check if the CER is lower than 0.8\n", " if have_genius_aligned:\n", " if genius_cer < 0.8:\n", " new_meta[\"text\"] = genius_text\n", " new_meta[\"text_aligned\"] = genius_alignments\n", " else:\n", " new_meta[\"text\"] = \"\"\n", " new_meta[\"text_aligned\"] = []\n", " elif have_discogs_aligned:\n", " if discogs_cer < 0.8:\n", " new_meta[\"text\"] = discogs_text\n", " new_meta[\"text_aligned\"] = discogs_alignments\n", " else:\n", " new_meta[\"text\"] = \"\"\n", " new_meta[\"text_aligned\"] = []\n", " elif have_deezer_aligned:\n", " if deezer_cer < 0.8:\n", " new_meta[\"text\"] = deezer_text\n", " new_meta[\"text_aligned\"] = deezer_alignments\n", " else:\n", " new_meta[\"text\"] = \"\"\n", " new_meta[\"text_aligned\"] = []\n", " elif have_podcast_aligned:\n", " pass # keep the text as is\n", " else:\n", " new_meta[\"text\"] = \"\"\n", " new_meta[\"text_aligned\"] = []\n", "\n", " new_metas.append(new_meta)\n", "\n", "\n", "print(len(missing_alignments))\n", "print(len(with_alignments))" ] }, { "cell_type": "code", "execution_count": null, "id": "03309ba8", "metadata": {}, "outputs": [], "source": [ "# count the number of rows with text, and text_aligned in df_filtered (as a DataFrame)\n", "num_with_text = (df_filtered[\"text\"] != \"\").sum()\n", "# Use list comprehension to count the length of 'text_aligned' if it's a list or tuple\n", "num_with_text_aligned = sum(\n", " isinstance(x, (list, tuple)) and len(x) > 0\n", " for x in df_filtered[\"text_aligned\"]\n", ")\n", "print(f\"Number of rows with text: {num_with_text}\")\n", "print(f\"Number of rows with text_aligned: {num_with_text_aligned}\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "0f88f6e4", "metadata": {}, "outputs": [], "source": [ "# count the number of rows with text, and text_aligned\n", "# in new_metas\n", "num_with_text = sum(1 for meta in new_metas if meta[\"text\"] != \"\")\n", "num_with_text_aligned = sum(1 for meta in new_metas if meta[\"text_aligned\"] != [])\n", "print(f\"Number of rows with text: {num_with_text}\")\n", "print(f\"Number of rows with text_aligned: {num_with_text_aligned}\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "9998a0bf", "metadata": {}, "outputs": [], "source": [ "# find a list of metas where id starts with \"podcast\"\n", "podcast_metas = [meta for meta in new_metas if meta[\"id\"].startswith(\"podcast\")]\n", "# print the first 10\n", "for podcast_meta in podcast_metas:\n", " print(podcast_meta)" ] }, { "cell_type": "code", "execution_count": null, "id": "ebb9362e", "metadata": {}, "outputs": [], "source": [ "# ear score distribution\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "# get all the ear scores from the new_metas\n", "ear_scores = [meta[\"ear_score\"] for meta in new_metas if meta[\"ear_score\"] is not None]\n", "# plot the distribution of the ear scores\n", "plt.figure(figsize=(10, 5))\n", "plt.hist(ear_scores, bins=250, alpha=0.5, label=\"Ear scores\")\n", "# bottom 10%\n", "plt.axvline(np.percentile(ear_scores, 5), color='red', linestyle='--', label=\"Bottom 10%\")\n", "plt.legend()\n", "plt.show()\n" ] }, { "cell_type": "code", "execution_count": null, "id": "469da303", "metadata": {}, "outputs": [], "source": [ "genius_cers = [a[\"cer\"][\"genius\"] for a in missing_alignments if a[\"cer\"][\"genius\"] is not None]\n", "discogs_cers = [a[\"cer\"][\"discogs\"] for a in missing_alignments if a[\"cer\"][\"discogs\"] is not None]\n", "deezer_cers = [a[\"cer\"][\"deezer\"] for a in missing_alignments if a[\"cer\"][\"deezer\"] is not None]\n", "\n", "import matplotlib.pyplot as plt\n", "# plot this on the same plot\n", "plt.figure(figsize=(10, 5))\n", "plt.title(\"CERs of missing alignments\")\n", "plt.hist(genius_cers, bins=250, alpha=0.5, label=\"Genius\")\n", "plt.hist(discogs_cers, bins=250, alpha=0.5, label=\"Discogs\")\n", "plt.hist(deezer_cers, bins=250, alpha=0.5, label=\"Deezer\")\n", "plt.legend()\n", "plt.show()\n" ] }, { "cell_type": "code", "execution_count": null, "id": "d585afdd", "metadata": {}, "outputs": [], "source": [ "meta_id = \"kkrUGbvXf1Y\"\n", "#find the row in the df_filtered that has this id\n", "row = df_filtered.filter(pl.col(\"id\") == meta_id)\n", "print(row[\"text\"])\n", "print(row[\"text_aligned\"])\n", "\n", "# get the alignments for this row\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "a76e6fce", "metadata": {}, "outputs": [], "source": [ "# Select only rows that have at least one of \"text\", \"text_aligned\", \"stems\", or \"tags\" not null,\n", "# AND must have \"weight\" not null -- ensure correct operator precedence!\n", "df_filtered = [\n", " m for m in metas\n", " if (\n", " (m.get(\"text\") is not None) or\n", " (m.get(\"text_aligned\") is not None) or\n", " (m.get(\"stems\") is not None) or\n", " (m.get(\"tags\") is not None)\n", " ) and (m.get(\"weight\") is not None)\n", "]\n", "print(len(df_filtered))" ] }, { "cell_type": "code", "execution_count": null, "id": "87fe520b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8c593505", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "96145177", "metadata": {}, "outputs": [], "source": [ "# Split df_filtered into train and val sets with 1% in val, randomly distributed\n", "\n", "import random\n", "\n", "val_frac = 0.01\n", "random.seed(42)\n", "df_shuffled = df_filtered[:]\n", "random.shuffle(df_shuffled)\n", "num_val = int(len(df_shuffled) * val_frac)\n", "df_val = df_shuffled[:num_val]\n", "df_train = df_shuffled[num_val:]\n", "\n", "print(\"train:\", len(df_train), \"val:\", len(df_val))" ] }, { "cell_type": "code", "execution_count": null, "id": "a8f4f7b7", "metadata": {}, "outputs": [], "source": [ "# Split new_metas into train and val sets with 1% in val, randomly distributed\n", "\n", "import random\n", "\n", "val_frac = 0.01\n", "random.seed(42)\n", "df_shuffled = new_metas[:]\n", "random.shuffle(df_shuffled)\n", "num_val = int(len(df_shuffled) * val_frac)\n", "df_val = df_shuffled[:num_val]\n", "df_train = df_shuffled[num_val:]\n", "\n", "print(\"train:\", len(df_train), \"val:\", len(df_val))" ] }, { "cell_type": "code", "execution_count": null, "id": "81ec6972", "metadata": {}, "outputs": [], "source": [ "# save to new jsonl files\n", "from suno_utils.utils.text import write_jsonl\n", "train_filepath = \"/app2/suno/data/diffusion/v1/metas_diff_v0_tr.jsonl\"\n", "val_filepath = \"/app2/suno/data/diffusion/v1/metas_diff_v0_val.jsonl\"\n", "write_jsonl(df_train, train_filepath)\n", "write_jsonl(df_val, val_filepath)" ] }, { "cell_type": "code", "execution_count": null, "id": "b86af9ed", "metadata": {}, "outputs": [], "source": [ "df_train.head()" ] }, { "cell_type": "code", "execution_count": null, "id": "a2934787", "metadata": {}, "outputs": [], "source": [ "# get all the unique meta keys across all metas\n", "unique_keys = set()\n", "for m in metas:\n", " unique_keys.update(m.keys())\n", "print(\"Unique keys:\", unique_keys)\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "c75638b9", "metadata": {}, "outputs": [], "source": [ "# count the audio types in metas\n", "audio_type_counts = {}\n", "source_counts = {}\n", "for meta in metas:\n", " audio_type = meta.get(\"audio_type\", None) # fine if missing\n", " audio_type_counts[audio_type] = audio_type_counts.get(audio_type, 0) + 1\n", " source = meta.get(\"source\", None)\n", " source_counts[source] = source_counts.get(source, 0) + 1\n" ] }, { "cell_type": "code", "execution_count": null, "id": "04a0f07f", "metadata": {}, "outputs": [], "source": [ "print(\"Audio type counts:\", audio_type_counts)\n", "print(\"Source counts:\", source_counts)" ] }, { "cell_type": "code", "execution_count": null, "id": "5a90b593", "metadata": {}, "outputs": [], "source": [ "# iterate through all the metas, look at the s3 filepath, if it doesnt have that, look at local_filepath\n", "# remove the filename and just look at the dirnames; find how many are unique\n", "\n", "import os\n", "from tqdm import tqdm\n", "\n", "unique_dirs = set()\n", "for meta in tqdm(metas, desc=\"Processing metas\", total=len(metas), mininterval=3.0, miniters=100000):\n", " filepath = meta.get('s3_filepath') or meta.get('local_filepath')\n", " if filepath:\n", " dirpath = os.path.dirname(filepath)\n", " unique_dirs.add(dirpath)\n", "\n", "print(f\"Number of unique directories: {len(unique_dirs)}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "f71482f2", "metadata": {}, "outputs": [], "source": [ "filtered_unique_dirs = set()\n", "for unique_dir in unique_dirs:\n", " if \"extreme_music\" not in unique_dir:\n", " filtered_unique_dirs.add(unique_dir)\n", "\n", "print(f\"Number of unique directories: {len(filtered_unique_dirs)}\")\n", "# remove the filename and just look at the dirnames find how many art unique\n" ] }, { "cell_type": "code", "execution_count": null, "id": "e6571d31", "metadata": {}, "outputs": [], "source": [ "filtered_unique_dirs" ] }, { "cell_type": "code", "execution_count": null, "id": "adebc662", "metadata": {}, "outputs": [], "source": [ "list(unique_dirs)[100]" ] }, { "cell_type": "code", "execution_count": null, "id": "7b6c4436", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.9" } }, "nbformat": 4, "nbformat_minor": 5 }