{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "ec50ae42", "metadata": {}, "outputs": [], "source": [ "# Datasets\n", "\n", "# Discogs (+subset)\n", "# Covers\n", "# Genius\n", "# Youtube music\n", "# Deezer\n", "# Pond5\n", "# Imslp\n", "# Karaoke stems\n", "# Musdb stems" ] }, { "cell_type": "code", "execution_count": null, "id": "4da60164", "metadata": {}, "outputs": [], "source": [ "# # TODO\n", "\n", "# playlist data\n", "# ditto embed\n", "# filter data for duplicates\n", "# filter data for really bad quality\n", "# filter data for incorrect lyrics/metas" ] }, { "cell_type": "code", "execution_count": null, "id": "4c753cad", "metadata": {}, "outputs": [], "source": [ "# v4 used:\n", "# (\"youtube_music\", \"v1\", (1, 4204), 1, 1, \"default\"),\n", "# (\"genius_hq\", \"v1\", (1, 4302), 1, 1, \"default\"),\n", "# (\"jamendo\", \"v1\", (1, 112), 1, 1, \"default\"),\n", "# (\"imslp\", \"v1\", (1, 558), 1, 1, \"default\"),\n", "# (\"pond5_music\", \"v2\", (1, 4138), 1, 1, \"default\"),\n", "# (\"deezer\", \"v2\", (1, 1538), 1, 1, \"default\"),\n", "# (\"ytm_tagged\", \"v2\", (1, 5545), 1, 1, \"default\"),\n", "# (\"discogs\", \"v3\", (1, 106_715), 1, 1, \"default\"),\n", "# (\"discogs_covers\", \"v3\", (1, 3_689), 1, 1, \"covers\")," ] }, { "cell_type": "code", "execution_count": null, "id": "e1a51dc7", "metadata": {}, "outputs": [], "source": [ "# 2 metas per dataset in folder v4 (either can be used for embed)\n", "# metas_raw.jsonl\n", "# - dataset specific columns\n", "# - order randomized unless groups for covers or playlists\n", "# required: id, dataset, s3_filepath, duration\n", "# optional: tags, text, title, artists, views, cover_info, playlist_info, ...\n", "# metas_v0.jsonl\n", "# - versioned processed file for use\n", "# - standardized columns\n", "# - anonimized info\n", "# - same order and length as metas_raw\n", "# required: id, dataset, s3_filepath, duration_s\n", "# optional: \n", "# tags, text, lang, artists, cover_parent_id, is_reliable, \n", "# stem_bundle_id, stem_type, playlist_ids, text_lines" ] }, { "cell_type": "code", "execution_count": null, "id": "ce37e54e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "96d89d67", "metadata": {}, "source": [ "## Make raw v5 datasets" ] }, { "cell_type": "code", "execution_count": 74, "id": "1fe86389", "metadata": {}, "outputs": [], "source": [ "import random\n", "import tqdm\n", "import numpy as np\n", "from suno_utils.audio import Audio\n", "from suno_utils.utils.text import read_jsonl, write_jsonl" ] }, { "cell_type": "markdown", "id": "b728c801", "metadata": {}, "source": [ "### Discogs & Covers" ] }, { "cell_type": "code", "execution_count": 204, "id": "3a34a73b", "metadata": {}, "outputs": [], "source": [ "# Mistakes:\n", "# Discogs:\n", "# ikkZzxyZ0bo - has lyrics but track is instrumental" ] }, { "cell_type": "code", "execution_count": 267, "id": "ba6ba230", "metadata": {}, "outputs": [], "source": [ "# s3://suno-data/shared/nfdg/manifest_v2.jsonl\n", "metas_raw = read_jsonl(\"/app/suno/tmp/andromeda_dicogs_manifest_v2.jsonl\")\n", "metas_raw_map = {m[\"id\"]: m for m in metas_raw}\n", "# s3://suno-data/datasets/harvest/ytm2/artist_songs_2_flat.jsonl\n", "metas_songs = read_jsonl(\"/app/suno/tmp/artist_songs_2_flat.jsonl\")\n", "# s3://suno-data/datasets/harvest/ytm2/cover_search.jsonl\n", "metas_covers = read_jsonl(\"/app/suno/tmp/cover_search.jsonl\")\n", "metas_covers = [m for m in metas_covers if len(m[\"covers\"]) > 0]\n", "# s3://suno-data/datasets/harvest/ytm2/ytm_tagged_songs_1207.jsonl\n", "metas_tagged = read_jsonl(\"/app/suno/tmp/ytm_tagged_songs_1207.jsonl\")\n", "metas_tagged_map = {m[\"id\"]: m for m in metas_tagged}\n", "# s3://suno-data/datasets/harvest/playlists/cleaned_playlists.jsonl\n", "metas_playlists = read_jsonl(\"/app/suno/tmp/cleaned_playlists.jsonl\")\n", "random.seed(6006)\n", "random.shuffle(metas_playlists)\n", "# TODO: use titles for cleaning or tags?\n", " \n", "# songs is a subset of raw\n", "# tagged is a subset of songs and has lots of extra info\n", "# covers parent is a subset of songs\n", "# covers child is a subset of raw\n", "\n", "# old:\n", "# metas_discogs = read_jsonl(\"/app/suno/data/chirp_v5/tmp_metas/discogs_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "b864a11d", "metadata": {}, "outputs": [], "source": [ "# look at some playlist titles" ] }, { "cell_type": "code", "execution_count": 73, "id": "46d1a4c5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "124,166,165 playlist metas\n" ] } ], "source": [ "print(f\"{len(metas_playlists):,} playlist metas\")" ] }, { "cell_type": "code", "execution_count": 71, "id": "6327418a", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 28%|█████████▉ | 35334331/124166165 [12:45<15:15, 96990.08it/s]IOPub message rate exceeded.\n", "The notebook server will temporarily stop sending output\n", "to the client in order to avoid crashing it.\n", "To change this limit, set the config variable\n", "`--NotebookApp.iopub_msg_rate_limit`.\n", "\n", "Current values:\n", "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n", "NotebookApp.rate_limit_window=3.0 (secs)\n", "\n" ] } ], "source": [ "from collections import defaultdict, Counter\n", "metas_playlists_map = defaultdict(list)\n", "for m in tqdm.tqdm(metas_playlists):\n", " if (\n", " len(m[\"tracks\"]) < 5 or \n", " len(m[\"tracks\"]) > 50 or\n", " m.get(\"views\", 0) is None or\n", " m.get(\"views\", 0) < 25\n", " ):\n", " continue\n", " # each track can only be listed in max 10 playlists\n", " track_ids = [_id for _id in m[\"tracks\"] if len(metas_playlists_map[_id]) < 10]\n", " if len(track_ids) < 5:\n", " continue\n", " for _id in track_ids:\n", " metas_playlists_map[_id].append(m[\"id\"])" ] }, { "cell_type": "code", "execution_count": 74, "id": "615fb407", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "50,404,900 tracks have playlist_id label\n", "11,246,325 playlists retained\n" ] } ], "source": [ "print(f\"{len(metas_playlists_map):,} tracks have playlist_id label\")\n", "pls = set()\n", "for v in metas_playlists_map.values():\n", " pls |= set(v)\n", "print(f\"{len(pls):,} playlists retained\")" ] }, { "cell_type": "code", "execution_count": 310, "id": "4325bd34", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "62,436,190 raw metas rows\n", "54,150,971 main song rows\n", "10,800,943 tagged rows\n", "972,920 cover parent rows\n", "17,581,647 cover child rows\n" ] } ], "source": [ "print(f\"{len(metas_raw):,} raw metas rows\")\n", "print(f\"{len(metas_songs):,} main song rows\")\n", "print(f\"{len(metas_tagged):,} tagged rows\")\n", "print(f\"{len(metas_covers):,} cover parent rows\")\n", "print(f\"{sum(len(m['covers']) for m in metas_covers):,} cover child rows\")" ] }, { "cell_type": "code", "execution_count": null, "id": "214892af", "metadata": {}, "outputs": [], "source": [ "# metas_raw\n", "# s3_filepath, keywords, lyrics_text as lyrics, lengthSeconds as duration_s, viewCount as views\n", "# metas_songs\n", "# title, artists, album\n", "# tagged_songs\n", "# bpm, discogs, hooktheory, musicbrainz_album, musicbrainz_track, rym_genres,sos_genre, ultimate_guitar" ] }, { "cell_type": "code", "execution_count": null, "id": "c1ceb587", "metadata": {}, "outputs": [], "source": [ "s1 = set([m[\"id\"] for m in metas_raw])\n", "s2 = set([m[\"videoId\"] for m in metas_songs])\n", "s3 = set([m[\"id\"] for m in metas_tagged])\n", "s4 = set([m[\"videoId\"] for m in metas_covers])\n", "s5 = set()\n", "for m in metas_covers:\n", " for mm in m[\"covers\"]:\n", " s5.add(mm[\"id\"])\n", "allowed_discogs_ids = s1 & s2\n", "# print(f\"{len(s1 & s2):,}\")" ] }, { "cell_type": "code", "execution_count": 322, "id": "944a9058", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████| 54150971/54150971 [10:04<00:00, 89559.70it/s]\n" ] } ], "source": [ "raw_discogs_metas = []\n", "for m in tqdm.tqdm(metas_songs):\n", " if m[\"videoId\"] not in metas_raw_map:\n", " continue\n", " new_m = {\n", " \"id\": m[\"videoId\"],\n", " \"title\": m[\"title\"],\n", " \"artists\": m[\"artists\"],\n", " \"album\": m[\"album\"],\n", " }\n", " mm = metas_raw_map[m[\"videoId\"]]\n", " new_m[\"s3_filepath\"] = \"s3://suno-data/\" + mm[\"audio\"]\n", " new_m[\"keywords\"] = mm[\"keywords\"]\n", " if mm.get(\"lyrics_text\") is not None:\n", " new_m[\"lyrics\"] = mm[\"lyrics_text\"]\n", " new_m[\"duration_s\"] = mm[\"lengthSeconds\"]\n", " new_m[\"views\"] = mm[\"viewCount\"]\n", " if m[\"videoId\"] in metas_playlists_map:\n", " new_m[\"playlist_ids\"] = metas_playlists_map[m[\"videoId\"]]\n", " if m[\"videoId\"] in metas_tagged_map:\n", " mmm = metas_tagged_map[m[\"videoId\"]]\n", " for k in [\n", " \"bpm\", \"discogs\", \"hooktheory\", \"musicbrainz_album\", \"musicbrainz_track\", \n", " \"rym_genres\", \"sos_genre\", \"ultimate_guitar\"\n", " ]:\n", " if k in mmm:\n", " new_m[k] = mmm[k]\n", " raw_discogs_metas.append(new_m)\n", "random.seed(6006)\n", "random.shuffle(raw_discogs_metas)" ] }, { "cell_type": "code", "execution_count": 220, "id": "34406400", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "52,220,969 total rows\n", "6,679,604 with lyrics\n", "5,952,453 with discogs\n", "8,695 with bpm\n", "13,350 with hooktheory\n", "4,850,664 with musicbrainz_album\n", "2,610,922 with rym_genres\n", "521,013 with sos_genre\n", "164,992 with ultimate_guitar\n", "8,981,181 with playlist_ids\n" ] } ], "source": [ "metadata_counts = {\n", " \"lyrics\": 0,\n", " \"discogs\": 0,\n", " \"bpm\": 0,\n", " \"hooktheory\": 0,\n", " \"musicbrainz_album\": 0,\n", " \"rym_genres\": 0,\n", " \"sos_genre\": 0,\n", " \"ultimate_guitar\": 0,\n", " \"playlist_ids\": 0,\n", "}\n", "for m in raw_discogs_metas:\n", " for key in metadata_counts.keys():\n", " if key in m:\n", " metadata_counts[key] += 1\n", "print(f\"{len(raw_discogs_metas):,} total rows\")\n", "for key, count in metadata_counts.items():\n", " print(f\"{count:,} with {key}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "fa4b0745", "metadata": {}, "outputs": [], "source": [ "# aim for ~200k hours of juicy subset (~3million tracks)\n", "raw_discogs_subset_metas = []\n", "for m in tqdm.tqdm(raw_discogs_metas):\n", " if (\n", " len(m.get(\"sos_genre\", [])) >= 2 or\n", " len(m.get(\"rym_genres\", [])) >= 5 or\n", " \"hooktheory\" in m or \n", " \"ultimate_guitar\" in m or\n", " (\"lyrics\" in m and m.get(\"views\", 0) >= 100)\n", " ) and m.get(\"views\", 0) >= 1_000:\n", " raw_discogs_subset_metas.append(m)\n", "print(f\"{len(raw_discogs_subset_metas):,}\")" ] }, { "cell_type": "code", "execution_count": 127, "id": "d80e0a08", "metadata": {}, "outputs": [], "source": [ "write_jsonl(raw_discogs_subset_metas, \"/app/suno/tmp/raw_discogs_subset_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 128, "id": "61dd710f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/raw_discogs_subset_metas.jsonl to s3://suno-data/datasets/bundles/v4/discogs_subset/metas_raw.jsonl\n" ] } ], "source": [ "# !aws s3 cp /app/suno/tmp/raw_discogs_subset_metas.jsonl s3://suno-data/datasets/bundles/v4/discogs_subset/metas_raw.jsonl" ] }, { "cell_type": "code", "execution_count": 338, "id": "246923c6", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████| 972920/972920 [00:47<00:00, 20443.32it/s]\n" ] } ], "source": [ "raw_covers_metas = []\n", "for m in tqdm.tqdm(metas_covers):\n", " if m[\"videoId\"] not in metas_raw_map:\n", " continue\n", " new_m = {\n", " \"id\": m[\"videoId\"],\n", " \"title\": m[\"title\"],\n", " \"artists\": m[\"artists\"],\n", " \"album\": m[\"album\"],\n", " }\n", " mm = metas_raw_map[m[\"videoId\"]]\n", " new_m[\"s3_filepath\"] = \"s3://suno-data/\" + mm[\"audio\"]\n", " new_m[\"keywords\"] = mm[\"keywords\"]\n", " if mm.get(\"lyrics_text\") is not None:\n", " new_m[\"lyrics\"] = mm[\"lyrics_text\"]\n", " new_m[\"duration_s\"] = mm[\"lengthSeconds\"]\n", " new_m[\"views\"] = mm[\"viewCount\"]\n", " if m[\"videoId\"] in metas_tagged_map:\n", " mmm = metas_tagged_map[m[\"videoId\"]]\n", " for k in [\n", " \"bpm\", \"discogs\", \"hooktheory\", \"musicbrainz_album\", \"musicbrainz_track\", \n", " \"rym_genres\", \"sos_genre\", \"ultimate_guitar\"\n", " ]:\n", " if k in mmm:\n", " new_m[k] = mmm[k]\n", " covers = [\n", " {\n", " \"id\": mm[\"id\"],\n", " \"duration_s\": mm[\"duration\"],\n", " \"s3_filepath\": \"s3://suno-data/\" + metas_raw_map[mm[\"id\"]][\"audio\"],\n", " \"title\": mm[\"title\"],\n", " \"views\": mm[\"views\"],\n", " } for mm in m[\"covers\"] if mm[\"id\"] in metas_raw_map\n", " ]\n", " if len(covers) == 0:\n", " continue\n", " new_m[\"covers\"] = covers\n", " raw_covers_metas.append(new_m)\n", "# don't shuffle to not lose structure of parent/child" ] }, { "cell_type": "code", "execution_count": 340, "id": "b5614c9d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "908,582 total rows\n", "15,780,354 with covers\n", "561,501 with lyrics\n", "433,217 with discogs\n", "5,431 with bpm\n", "10,599 with hooktheory\n", "426,075 with musicbrainz_album\n", "351,776 with rym_genres\n", "117,730 with sos_genre\n", "98,321 with ultimate_guitar\n" ] } ], "source": [ "metadata_counts = {\n", " \"covers\": 0,\n", " \"lyrics\": 0,\n", " \"discogs\": 0,\n", " \"bpm\": 0,\n", " \"hooktheory\": 0,\n", " \"musicbrainz_album\": 0,\n", " \"rym_genres\": 0,\n", " \"sos_genre\": 0,\n", " \"ultimate_guitar\": 0,\n", "}\n", "for m in raw_covers_metas:\n", " for key in metadata_counts.keys():\n", " if key in m:\n", " if key == \"covers\":\n", " metadata_counts[key] += len(m[key])\n", " else:\n", " metadata_counts[key] += 1\n", "print(f\"{len(raw_covers_metas):,} total rows\")\n", "for key, count in metadata_counts.items():\n", " print(f\"{count:,} with {key}\")" ] }, { "cell_type": "code", "execution_count": 342, "id": "04b1416e", "metadata": {}, "outputs": [], "source": [ "write_jsonl(raw_covers_metas, \"/app/suno/tmp/raw_covers_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "1c218d80", "metadata": {}, "outputs": [], "source": [ "# !aws s3 cp /app/suno/tmp/raw_covers_metas.jsonl s3://suno-data/datasets/bundles/v4/covers/metas_raw.jsonl" ] }, { "cell_type": "markdown", "id": "a06558b9", "metadata": {}, "source": [ "### Genius" ] }, { "cell_type": "code", "execution_count": 44, "id": "a0f3a3cf", "metadata": {}, "outputs": [], "source": [ "# s3://suno-data/datasets/bundles/v1/genius_hq/metas_plus_artists.jsonl\n", "metas_raw = read_jsonl(\"/app/suno/tmp/metas_genius_plus_artists.jsonl\")\n", "# s3://suno-data/datasets/metadata/chirp_v4/genius_hq_v5_plus.jsonl\n", "# metas_extra = read_jsonl(\"/app/suno/data/chirp_v4/metadata/genius_hq_v5_plus.jsonl\")\n", "llm_tags = read_jsonl(\"/home/minz/llm_tagging/data/genius_llm_metas_r3_enhanced.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "4dab7399", "metadata": {}, "outputs": [], "source": [ "# It includes 481,715 lines\n", "# 385,428 songs have local information (rest of them only have global tags and captions)\n", "# Use enhanced_lyrics to get the local tags, local captions, and the lyrics\n", "# Use global_tags to get the global tags. I made it as a list so you can shuffle or dropout easily. (Later I want to handle this better, so we can specify a certain tag is for vocal or instruments)\n", "# Use global_captions to get the global captions. I also made this as a list so you can shuffle or dropout.\n", "# The key does not exist when it does not have that information (I filtered out no information available)\n", "# There's no duplicated tags anymore" ] }, { "cell_type": "code", "execution_count": 65, "id": "a6f4bbc0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "481,715\n" ] } ], "source": [ "llm_tag_map = {}\n", "for m in llm_tags:\n", " _id = m[\"s3_filepath\"].split(\"/\")[-1].split(\".\")[0]\n", " llm_tag_map[_id] = {}\n", " if \"enhanced_lyrics\" in m:\n", " llm_tag_map[_id][\"lyrics\"] = m[\"enhanced_lyrics\"]\n", " tags = []\n", " if \"global_tags\" in m:\n", " tags.extend(m[\"global_tags\"])\n", " if \"global_captions\" in m:\n", " tags.extend(m[\"global_captions\"])\n", " if len(tags) > 0:\n", " llm_tag_map[_id][\"tags\"] = tags\n", "print(f\"{len(llm_tag_map):,}\")" ] }, { "cell_type": "code", "execution_count": 144, "id": "51b4d06a", "metadata": {}, "outputs": [], "source": [ "import re\n", "\n", "def _clean_song_header(s):\n", " s = re.sub(r\"^\\s*\\[.*\\\".*\\\".*\\]\\s*\", \"\", s) # song/artist name at start\n", " return s\n", "\n", "def _clean_lyrics(s):\n", " s = _clean_song_header(s)\n", " s = re.sub(r\"\\[(.*)\\:.*\\]\", \"[\\\\1]\", s) # artist name in section tag\n", " s = re.sub(r\"\\((.*)\\:.*\\)\", \"[\\\\1]\", s) # artist name in section tag\n", " return s\n", "\n", "# s = \"\"\"\n", "# [Testo di \"Abisso\"]\\n\\n[]aasdas\n", "# [Intro: blabla]\n", "# [Intro]\n", "# [Intro; asda]\n", "# \"\"\"\n", "# print(clean_lyrics(s))" ] }, { "cell_type": "code", "execution_count": 173, "id": "63594afa", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████| 2090009/2090009 [00:46<00:00, 44747.25it/s]\n" ] } ], "source": [ "raw_genius_metas = []\n", "for m in tqdm.tqdm(metas_raw):\n", " lyrics = m[\"lyrics\"]\n", " lyrics = _clean_lyrics(lyrics)\n", " new_m = {\n", " \"id\": m[\"original_id\"],\n", " \"s3_filepath\": m[\"audio_filepath\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " \"genius_tags\": [s.replace(\" Genius\", \"\") for s in m.get(\"tags_text\", [])],\n", " \"lyrics\": lyrics,\n", " \"lang\": m[\"lang\"],\n", " \"artists\": m[\"artists\"],\n", " \"views\": m[\"youtube_views\"],\n", " \"genius_views\": m[\"genius_views\"],\n", " \"genius_slug\": m[\"genius_slug\"],\n", " \"youtube_title\": m[\"youtube_title\"],\n", " \"youtube_channel_id\": m[\"youtube_channel_id\"],\n", " }\n", " _id = m[\"original_id\"]\n", " if _id in llm_tag_map and \"lyrics\" in llm_tag_map[_id]:\n", " new_m[\"lyrics\"] = _clean_song_header(llm_tag_map[_id][\"lyrics\"])\n", " if _id in llm_tag_map and \"tags\" in llm_tag_map[_id]:\n", " new_m[\"tags\"] = llm_tag_map[_id][\"tags\"]\n", " raw_genius_metas.append(new_m)\n", "random.seed(6006)\n", "random.shuffle(raw_genius_metas)" ] }, { "cell_type": "code", "execution_count": 174, "id": "648e9eee", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2,090,009 tracks\n", "481,711 tracks with tags\n", "126,615 hours total\n" ] } ], "source": [ "print(f\"{len(raw_genius_metas):,} tracks\")\n", "print(f\"{len([m for m in raw_genius_metas if 'tags' in m]):,} tracks with tags\")\n", "print(f\"{round(sum(m['duration_s'] for m in raw_genius_metas)/60/60):,} hours total\")" ] }, { "cell_type": "code", "execution_count": 175, "id": "bf6e38a5", "metadata": {}, "outputs": [], "source": [ "# TODO: artists are anonymized" ] }, { "cell_type": "code", "execution_count": 181, "id": "5d5ad5af", "metadata": {}, "outputs": [], "source": [ "write_jsonl(raw_genius_metas, \"/app/suno/tmp/raw_genius_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 182, "id": "66806ca3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/raw_genius_metas.jsonl to s3://suno-data/datasets/bundles/v4/genius/metas_raw.jsonl\n" ] } ], "source": [ "!aws s3 cp /app/suno/tmp/raw_genius_metas.jsonl s3://suno-data/datasets/bundles/v4/genius/metas_raw.jsonl" ] }, { "cell_type": "markdown", "id": "e2bbbc3b", "metadata": {}, "source": [ "### Youtube Music" ] }, { "cell_type": "code", "execution_count": null, "id": "87b04384", "metadata": {}, "outputs": [], "source": [ "# s3://suno-data/datasets/bundles/v1/youtube_music/metas.jsonl\n", "metas_raw = read_jsonl(\"/app/suno/tmp/metas_youtube_music.jsonl\")\n", "# s3://suno-data/datasets/metadata/chirp_v4/youtube_music.jsonl\n", "metas_extra = read_jsonl(\"/app/suno/data/chirp_v4/metadata/youtube_music.jsonl\")\n", "metas_extra_map = {m[\"id\"]: m for m in metas_extra}" ] }, { "cell_type": "code", "execution_count": 56, "id": "c1dfa9f1", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████| 2101712/2101712 [00:04<00:00, 486215.51it/s]\n" ] } ], "source": [ "raw_youtube_music_metas = []\n", "for m in tqdm.tqdm(metas_raw):\n", " if m[\"id\"] not in metas_extra_map:\n", " continue\n", " mm = metas_extra_map[m[\"id\"]]\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"s3_filepath\": m[\"s3_filepath\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " \"views\": m[\"view_count\"],\n", " \"title\": m[\"title\"],\n", " \"artists\": m[\"artists\"],\n", " \"tags\": mm[\"private_tags\"],\n", " }\n", " if \"text\" in mm:\n", " new_m[\"text\"] = mm[\"text\"]\n", " new_m[\"lang\"] = mm[\"lang\"]\n", " if \"genre_guess\" in m:\n", " new_m[\"genres\"] = m[\"genre_guess\"]\n", " if \"mood_guess\" in m:\n", " new_m[\"moods\"] = m[\"mood_guess\"]\n", " if \"album\" in mm:\n", " new_m[\"album\"] = mm[\"album\"]\n", " raw_youtube_music_metas.append(new_m)\n", "random.seed(6006)\n", "random.shuffle(raw_youtube_music_metas)" ] }, { "cell_type": "code", "execution_count": 61, "id": "2e62ac5a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2,013,545 tracks\n", "138,373 hours\n", "43,833 hours with lyrics\n" ] } ], "source": [ "print(f\"{len(raw_youtube_music_metas):,} tracks\")\n", "print(f\"{round(sum(m['duration_s'] for m in raw_youtube_music_metas)/60/60):,} hours\")\n", "print(f\"{round(sum(m['duration_s'] for m in raw_youtube_music_metas if 'text' in m)/60/60):,} hours with lyrics\")" ] }, { "cell_type": "code", "execution_count": 62, "id": "a7ccc98f", "metadata": {}, "outputs": [], "source": [ "write_jsonl(raw_youtube_music_metas, \"/app/suno/tmp/raw_youtube_music_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 63, "id": "b48e05ee", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/raw_youtube_music_metas.jsonl to s3://suno-data/datasets/bundles/v4/youtube_music/metas_raw.jsonl\n" ] } ], "source": [ "# !aws s3 cp /app/suno/tmp/raw_youtube_music_metas.jsonl s3://suno-data/datasets/bundles/v4/youtube_music/metas_raw.jsonl" ] }, { "cell_type": "markdown", "id": "7378c255", "metadata": {}, "source": [ "### Deezer" ] }, { "cell_type": "code", "execution_count": 119, "id": "fcc8dde8", "metadata": {}, "outputs": [], "source": [ "# s3://suno-data/datasets/bundles/v2/deezer/metas_plus.jsonl\n", "metas_raw = read_jsonl(\"/app/suno/tmp/metas_deezer.jsonl\")\n", "# s3://suno-data/datasets/metadata/chirp_v4/deezer.jsonl\n", "# metas_extra = read_jsonl(\"/app/suno/data/chirp_v4/metadata/deezer.jsonl\")" ] }, { "cell_type": "code", "execution_count": 122, "id": "2e26b82c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████| 768515/768515 [00:01<00:00, 676941.64it/s]\n" ] } ], "source": [ "raw_deezer_metas = []\n", "for m in tqdm.tqdm(metas_raw):\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"s3_filepath\": m[\"s3_filepath\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " \"views\": m[\"views\"],\n", " \"text_lines\": m[\"lyrics_lines\"],\n", " \"lang\": m[\"lang_guess\"],\n", " \"title\": m[\"youtube_title\"],\n", " \"artist\": m[\"youtube_artist\"],\n", " }\n", " raw_deezer_metas.append(new_m)\n", "random.seed(6006)\n", "random.shuffle(raw_deezer_metas)" ] }, { "cell_type": "code", "execution_count": 123, "id": "3ca15898", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "768,515 tracks\n", "42,975 hours\n" ] } ], "source": [ "print(f\"{len(raw_deezer_metas):,} tracks\")\n", "print(f\"{round(sum(m['duration_s'] for m in raw_deezer_metas)/60/60):,} hours\")" ] }, { "cell_type": "code", "execution_count": 124, "id": "db2297e0", "metadata": {}, "outputs": [], "source": [ "write_jsonl(raw_deezer_metas, \"/app/suno/tmp/raw_deezer_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 125, "id": "a8f5af1e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/raw_deezer_metas.jsonl to s3://suno-data/datasets/bundles/v4/deezer/metas_raw.jsonl\n" ] } ], "source": [ "# !aws s3 cp /app/suno/tmp/raw_deezer_metas.jsonl s3://suno-data/datasets/bundles/v4/deezer/metas_raw.jsonl" ] }, { "cell_type": "markdown", "id": "751915de", "metadata": {}, "source": [ "### Pond5" ] }, { "cell_type": "code", "execution_count": 83, "id": "3684c669", "metadata": {}, "outputs": [], "source": [ "# s3://suno-data/datasets/bundles/v2/pond5_music/metas.jsonl\n", "metas_raw = read_jsonl(\"/app/suno/tmp/metas_pond5.jsonl\")\n", "# s3://suno-data/datasets/metadata/chirp_v4/pond5_music.jsonl\n", "metas_extra = read_jsonl(\"/app/suno/data/chirp_v4/metadata/pond5_music.jsonl\")\n", "tags_map = {m[\"id\"]: m[\"tags\"] for m in metas_extra}" ] }, { "cell_type": "code", "execution_count": 98, "id": "a893f055", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████| 2068743/2068743 [00:15<00:00, 135611.17it/s]\n" ] } ], "source": [ "raw_pond5_metas = []\n", "for m in tqdm.tqdm(metas_raw):\n", " if m[\"id\"] not in tags_map:\n", " continue\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"s3_filepath\": m[\"s3_filepath\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " \"tags\": tags_map[m[\"id\"]],\n", " }\n", " raw_pond5_metas.append(new_m)\n", "random.seed(6006)\n", "random.shuffle(raw_pond5_metas)" ] }, { "cell_type": "code", "execution_count": 115, "id": "e3b5ee94", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2,068,743 tracks\n", "64,000 hours\n" ] } ], "source": [ "print(f\"{len(raw_pond5_metas):,} tracks\")\n", "print(f\"{round(sum(m['duration_s'] for m in raw_pond5_metas)/60/60):,} hours\")" ] }, { "cell_type": "code", "execution_count": 99, "id": "f483bc6e", "metadata": {}, "outputs": [], "source": [ "write_jsonl(raw_pond5_metas, \"/app/suno/tmp/raw_pond5_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 100, "id": "5a101b64", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/raw_pond5_metas.jsonl to s3://suno-data/datasets/bundles/v4/pond5/metas_raw.jsonl\n" ] } ], "source": [ "# !aws s3 cp /app/suno/tmp/raw_pond5_metas.jsonl s3://suno-data/datasets/bundles/v4/pond5/metas_raw.jsonl" ] }, { "cell_type": "markdown", "id": "10ad7879", "metadata": {}, "source": [ "## Imslp" ] }, { "cell_type": "code", "execution_count": 103, "id": "c0396339", "metadata": {}, "outputs": [], "source": [ "# s3://suno-data/datasets/bundles/v1/imslp/metas.jsonl\n", "metas_raw = read_jsonl(\"/app/suno/tmp/metas_imslp.jsonl\")\n", "# s3://suno-data/datasets/metadata/chirp_v4/pond5_music.jsonl\n", "metas_extra = read_jsonl(\"/app/suno/data/chirp_v4/metadata/imslp.jsonl\")\n", "metas_extra_map = {m[\"id\"]: m for m in metas_extra}" ] }, { "cell_type": "code", "execution_count": 113, "id": "1dbb6240", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████| 278620/278620 [00:00<00:00, 417620.94it/s]\n" ] } ], "source": [ "raw_imslp_metas = []\n", "for m in tqdm.tqdm(metas_raw):\n", " if m[\"id\"] not in metas_extra_map:\n", " continue\n", " mm = metas_extra_map[m[\"id\"]]\n", " new_m = {\n", " \"id\": m[\"s3_filepath\"].split(\"/\")[-1].split(\".\")[0],\n", " \"original_id\": m[\"original_id\"],\n", " \"s3_filepath\": m[\"s3_filepath\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " \"composer\": m[\"composer\"],\n", " \"recording_category\": m[\"recording_category\"],\n", " \"tags\": mm[\"private_tags\"],\n", " \"tags_redacted\": mm[\"tags\"],\n", " }\n", " for k in [\"title\", \"instruments\", \"genre\"]:\n", " if k in m:\n", " new_m[k] = m[k]\n", " raw_imslp_metas.append(new_m)\n", "random.seed(6006)\n", "random.shuffle(raw_imslp_metas)" ] }, { "cell_type": "code", "execution_count": 114, "id": "b02e59fe", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "274,275 tracks\n", "21,921 hours\n" ] } ], "source": [ "print(f\"{len(raw_imslp_metas):,} tracks\")\n", "print(f\"{round(sum(m['duration_s'] for m in raw_imslp_metas)/60/60):,} hours\")" ] }, { "cell_type": "code", "execution_count": 117, "id": "b7e05077", "metadata": {}, "outputs": [], "source": [ "write_jsonl(raw_imslp_metas, \"/app/suno/tmp/raw_imslp_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 118, "id": "d646b149", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/raw_imslp_metas.jsonl to s3://suno-data/datasets/bundles/v4/imslp/metas_raw.jsonl\n" ] } ], "source": [ "# !aws s3 cp /app/suno/tmp/raw_imslp_metas.jsonl s3://suno-data/datasets/bundles/v4/imslp/metas_raw.jsonl" ] }, { "cell_type": "markdown", "id": "f486cb7b", "metadata": {}, "source": [ "### Karaoke stems" ] }, { "cell_type": "code", "execution_count": 2, "id": "7c5c9f6c", "metadata": {}, "outputs": [], "source": [ "# s3://suno-data/datasets/bundles/v3/karaoke_stems/metas.jsonl\n", "metas_raw = read_jsonl(\"/app/suno/tmp/metas_karaoke.jsonl\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "27393ee1", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'id': '9980_main',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/full_audio/9980.mp3',\n", " 'duration_s': 46.0,\n", " 'is_main': True}" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "metas_raw[0]" ] }, { "cell_type": "code", "execution_count": 4, "id": "75b9feeb", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'id': '9980_Acoustic_Guitar',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9980/9980_Acoustic_Guitar.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Acoustic Guitar',\n", " 'is_main': False}" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "metas_raw[5]" ] }, { "cell_type": "code", "execution_count": null, "id": "7cda9bd7", "metadata": {}, "outputs": [], "source": [ "# check how many have lead vocals" ] }, { "cell_type": "code", "execution_count": 22, "id": "02e380ff", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[{'id': '9980_main',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/full_audio/9980.mp3',\n", " 'duration_s': 46.0,\n", " 'is_main': True},\n", " {'id': '9980_Drum_Kit',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9980/9980_Drum_Kit.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Drum Kit',\n", " 'is_main': False},\n", " {'id': '9980_Percussion',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9980/9980_Percussion.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Percussion',\n", " 'is_main': False},\n", " {'id': '9980_Noise_effects',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9980/9980_Noise_effects.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Noise effects',\n", " 'is_main': False},\n", " {'id': '9980_Bass',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9980/9980_Bass.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Bass',\n", " 'is_main': False},\n", " {'id': '9980_Acoustic_Guitar',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9980/9980_Acoustic_Guitar.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Acoustic Guitar',\n", " 'is_main': False},\n", " {'id': '9980_Lead_Electric_Guitar',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9980/9980_Lead_Electric_Guitar.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Lead Electric Guitar',\n", " 'is_main': False},\n", " {'id': '9980_Arr._Electric_Guitar',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9980/9980_Arr._Electric_Guitar.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Arr. Electric Guitar',\n", " 'is_main': False},\n", " {'id': '9980_Synth_Pad',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9980/9980_Synth_Pad.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Synth Pad',\n", " 'is_main': False},\n", " {'id': '9980_Synth_Strings',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9980/9980_Synth_Strings.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Synth Strings',\n", " 'is_main': False},\n", " {'id': '9980_Synth_Voice',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9980/9980_Synth_Voice.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Synth Voice',\n", " 'is_main': False},\n", " {'id': '9980_Synth_Keys',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9980/9980_Synth_Keys.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Synth Keys',\n", " 'is_main': False},\n", " {'id': '9980_Backing_Vocals',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9980/9980_Backing_Vocals.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Backing Vocals',\n", " 'is_main': False},\n", " {'id': '9980_Lead_Vocal',\n", " 'bundle_id': '9980',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9980/9980_Lead_Vocal.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Lead Vocal',\n", " 'is_main': False},\n", " {'id': '9981_main',\n", " 'bundle_id': '9981',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/full_audio/9981.mp3',\n", " 'duration_s': 46.0,\n", " 'is_main': True},\n", " {'id': '9981_Drum_Kit',\n", " 'bundle_id': '9981',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9981/9981_Drum_Kit.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Drum Kit',\n", " 'is_main': False},\n", " {'id': '9981_Tambourine',\n", " 'bundle_id': '9981',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9981/9981_Tambourine.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Tambourine',\n", " 'is_main': False},\n", " {'id': '9981_Bass',\n", " 'bundle_id': '9981',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9981/9981_Bass.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Bass',\n", " 'is_main': False},\n", " {'id': '9981_Rhythm_Acoustic_Guitar',\n", " 'bundle_id': '9981',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9981/9981_Rhythm_Acoustic_Guitar.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Rhythm Acoustic Guitar',\n", " 'is_main': False},\n", " {'id': '9981_Rhythm_Acoustic_Guitar',\n", " 'bundle_id': '9981',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9981/9981_Rhythm_Acoustic_Guitar.mp3',\n", " 'duration_s': 46.0,\n", " 'stem_type': 'Rhythm Acoustic Guitar',\n", " 'is_main': False}]" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "metas_raw[:20]" ] }, { "cell_type": "code", "execution_count": 44, "id": "ded3648d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "64,498 bundles with vocals\n" ] } ], "source": [ "bundle_info = {}\n", "for m in metas_raw:\n", " if \"full_audio\" in m[\"s3_filepath\"]:\n", " continue\n", " _id = m[\"bundle_id\"]\n", " if _id not in bundle_info:\n", " bundle_info[_id] = {}\n", " bundle_info[_id][\"has_vocals\"] = False\n", " bundle_info[_id][\"stems\"] = []\n", " if m[\"stem_type\"] == \"Lead Vocal\":\n", " bundle_info[_id][\"has_vocals\"] = True\n", " bundle_info[_id][\"stems\"].append({k: v for k, v in m.items()})\n", "bundle_info = {\n", " k: v for k, v in bundle_info.items() if v[\"has_vocals\"] and len(v[\"stems\"]) >= 5\n", "}\n", "print(f\"{len(bundle_info):,} bundles with vocals\")" ] }, { "cell_type": "code", "execution_count": 37, "id": "c871076f", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['9980', '9981']" ] }, "execution_count": 37, "metadata": {}, "output_type": "execute_result" } ], "source": [ "list(bundle_info.keys())[:2]" ] }, { "cell_type": "code", "execution_count": 51, "id": "857a80ff", "metadata": {}, "outputs": [], "source": [ "# bundle_info[\"9980\"]" ] }, { "cell_type": "code", "execution_count": 24, "id": "bf5e8965", "metadata": {}, "outputs": [], "source": [ "# audios = []\n", "# for m in metas_raw:\n", "# if m[\"bundle_id\"] == \"9980\" and \"full_audio\" not in m[\"s3_filepath\"]:\n", "# audios.append(Audio.from_s3(m[\"s3_filepath\"], sample_rate=48_000, n_channels=2))\n", "# for a in audios:\n", "# a.play()" ] }, { "cell_type": "code", "execution_count": 25, "id": "c64380ec", "metadata": {}, "outputs": [], "source": [ "# array_float = None\n", "# for a in audios:\n", "# if array_float is None:\n", "# array_float = a.array_float.copy()\n", "# else:\n", "# array_float += a.array_float.copy()\n", "# print(\"max:\", round(array_float.max(), 2))\n", "# print(\"min:\", round(array_float.min(), 2))\n", "# array_float = array_float / np.abs(array_float).max()\n", "# Audio.from_array_float(array_float, sample_rate=audios[0].sample_rate).play()" ] }, { "cell_type": "code", "execution_count": 121, "id": "908a5974", "metadata": {}, "outputs": [], "source": [ "import os\n", "import numpy as np\n", "import tqdm\n", "from joblib import Parallel, delayed\n", "import tempfile\n", "from suno_utils.utils.s3 import _upload_s3_file\n", "\n", "def _upload_to_s3(audio, to_s3_path):\n", " with tempfile.TemporaryDirectory() as td:\n", " fp = os.path.join(td, \"audio.mp3\")\n", " audio.to_hq_mp3(fp)\n", " _upload_s3_file(fp, to_s3_path)\n", "\n", "def _maybe_peak_norm(arr):\n", " max_val = np.abs(arr).max()\n", " if max_val > 1:\n", " arr /= max_val\n", " return arr\n", "\n", "def process_bundle(bundle_item):\n", " _id, v = bundle_item\n", " try:\n", " # Load all stems first to check lengths\n", " stems_data = []\n", " for m in v[\"stems\"]:\n", " arr = Audio.from_s3(m[\"s3_filepath\"], sample_rate=48_000, n_channels=2).array_float\n", " stems_data.append({\n", " \"array\": arr,\n", " \"stem_type\": m[\"stem_type\"]\n", " })\n", " \n", " # Check length differences\n", " lengths = [len(stem[\"array\"]) for stem in stems_data]\n", " max_diff_samples = int(0.01 * 48_000) # 10ms at 48kHz\n", " if max(lengths) - min(lengths) > max_diff_samples:\n", " raise ValueError(f\"Skipping bundle {k}: stems length difference > 10ms\")\n", " \n", " # Truncate to shortest length\n", " min_length = min(lengths)\n", " for stem in stems_data:\n", " stem[\"array\"] = stem[\"array\"][:min_length]\n", " \n", " # Process stems\n", " arr_full = None\n", " arr_lead_vocals = None\n", " arr_no_vocals = None\n", " \n", " for stem in stems_data:\n", " arr = stem[\"array\"]\n", " if arr_full is None:\n", " arr_full = arr.copy()\n", " else:\n", " arr_full += arr.copy()\n", " \n", " if stem[\"stem_type\"] == \"Lead Vocal\":\n", " arr_lead_vocals = arr.copy()\n", " elif \"vocal\" not in stem[\"stem_type\"].lower():\n", " if arr_no_vocals is None:\n", " arr_no_vocals = arr.copy()\n", " else:\n", " arr_no_vocals += arr.copy()\n", " \n", " # Normalize\n", " arr_full = _maybe_peak_norm(arr_full)\n", " arr_lead_vocals = _maybe_peak_norm(arr_lead_vocals)\n", " arr_no_vocals = _maybe_peak_norm(arr_no_vocals)\n", " \n", " # Convert to Audio objects\n", " audio_full = Audio.from_array_float(arr_full, sample_rate=48_000)\n", " audio_lead_vocals = Audio.from_array_float(arr_lead_vocals, sample_rate=48_000)\n", " audio_no_vocals = Audio.from_array_float(arr_no_vocals, sample_rate=48_000)\n", " \n", " # convert to mp3 and upload to s3\n", " s3_prefix = \"s3://suno-data/datasets/harvest/karaoke_versions/stems/generated\"\n", " s3_filepath_full = os.path.join(s3_prefix, f\"{_id}_full.mp3\")\n", " s3_filepath_lead_vocals = os.path.join(s3_prefix, f\"{_id}_lead_vocals.mp3\")\n", " s3_filepath_no_vocals = os.path.join(s3_prefix, f\"{_id}_no_vocals.mp3\")\n", " _upload_to_s3(audio_full, s3_filepath_full)\n", " _upload_to_s3(audio_lead_vocals, s3_filepath_lead_vocals)\n", " _upload_to_s3(audio_no_vocals, s3_filepath_no_vocals)\n", " return {\n", " \"bundle_id\": _id,\n", " \"bundle_info\": {\n", " \"id\": _id,\n", " \"duration_s\": round(audio_full.duration_s, 2),\n", " \"s3_filepath_full\": s3_filepath_full,\n", " \"s3_filepath_lead_vocals\": s3_filepath_lead_vocals,\n", " \"s3_filepath_no_vocals\": s3_filepath_no_vocals,\n", " },\n", " \"success\": True\n", " }\n", " \n", " except Exception as e:\n", " print(f\"Error processing bundle {k}: {str(e)}\")\n", " return {\n", " \"bundle_id\": _id,\n", " \"success\": False,\n", " \"error\": str(e)\n", " }" ] }, { "cell_type": "code", "execution_count": 122, "id": "02b7bac5", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 2.12 s, sys: 852 ms, total: 2.97 s\n", "Wall time: 11.2 s\n" ] } ], "source": [ "%%time\n", "test_items = list(bundle_info.items())[:50]\n", "out = Parallel(n_jobs=16, prefer=\"threads\")(delayed(process_bundle)(e) for e in test_items)" ] }, { "cell_type": "code", "execution_count": 126, "id": "11d1998b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2/2 success.\n" ] } ], "source": [ "print(f\"{len([e for e in out if e['success']])}/{len(test_items)} success.\")" ] }, { "cell_type": "code", "execution_count": 130, "id": "f6baadd3", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[{'id': '9980_full',\n", " 'bundel_id': '9980',\n", " 'duration_s': 46.03,\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/generated/9980_full.mp3'},\n", " {'id': '9980_lead_vocals',\n", " 'bundel_id': '9980',\n", " 'duration_s': 46.03,\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/generated/9980_lead_vocals.mp3'},\n", " {'id': '9980_no_vocals',\n", " 'bundel_id': '9980',\n", " 'duration_s': 46.03,\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/generated/9980_no_vocals.mp3'},\n", " {'id': '9981_full',\n", " 'bundel_id': '9981',\n", " 'duration_s': 46.03,\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/generated/9981_full.mp3'},\n", " {'id': '9981_lead_vocals',\n", " 'bundel_id': '9981',\n", " 'duration_s': 46.03,\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/generated/9981_lead_vocals.mp3'},\n", " {'id': '9981_no_vocals',\n", " 'bundel_id': '9981',\n", " 'duration_s': 46.03,\n", " 's3_filepath': 's3://suno-data/datasets/harvest/karaoke_versions/stems/generated/9981_no_vocals.mp3'}]" ] }, "execution_count": 130, "metadata": {}, "output_type": "execute_result" } ], "source": [ "processed_metas = []\n", "for e in out:\n", " if not e[\"success\"]:\n", " continue\n", " m = e[\"bundle_info\"]\n", " bundel_id = m[\"id\"]\n", " processed_metas.append({\n", " \"id\": f\"{bundel_id}_full\",\n", " \"bundel_id\": bundel_id,\n", " \"duration_s\": m[\"duration_s\"],\n", " \"s3_filepath\": m[\"s3_filepath_full\"],\n", " })\n", " processed_metas.append({\n", " \"id\": f\"{bundel_id}_lead_vocals\",\n", " \"bundel_id\": bundel_id,\n", " \"duration_s\": m[\"duration_s\"],\n", " \"s3_filepath\": m[\"s3_filepath_lead_vocals\"],\n", " })\n", " processed_metas.append({\n", " \"id\": f\"{bundel_id}_no_vocals\",\n", " \"bundel_id\": bundel_id,\n", " \"duration_s\": m[\"duration_s\"],\n", " \"s3_filepath\": m[\"s3_filepath_no_vocals\"],\n", " })\n", "processed_metas" ] }, { "cell_type": "code", "execution_count": 106, "id": "719fb83a", "metadata": {}, "outputs": [], "source": [ "# print(round(len(bundle_info) / 50 / 60 / 60 * 55.0, 1), \"hours total runtime\")\n", "# ~20h total (TODO: move to modal)" ] }, { "cell_type": "code", "execution_count": 103, "id": "fc4add18", "metadata": {}, "outputs": [], "source": [ "# s3://suno-data/datasets/harvest/karaoke_versions/stems/full_audio/9980.mp3\n", "# s3://suno-data/datasets/harvest/karaoke_versions/stems/audio/9980/9980_Drum_Kit.mp3\n", "# !aws s3 ls s3://suno-data/datasets/harvest/karaoke_versions/stems/generated/" ] }, { "cell_type": "code", "execution_count": 107, "id": "935c44b7", "metadata": {}, "outputs": [], "source": [ "# d = \"s3://suno-data/datasets/harvest/karaoke_versions/stems/generated/\"\n", "# _id = \"9981\"\n", "# Audio.from_s3(d + f\"{_id}_full.mp3\", sample_rate=48_000, n_channels=2).play()\n", "# Audio.from_s3(d + f\"{_id}_lead_vocals.mp3\", sample_rate=48_000, n_channels=2).play()\n", "# Audio.from_s3(d + f\"{_id}_no_vocals.mp3\", sample_rate=48_000, n_channels=2).play()" ] }, { "cell_type": "code", "execution_count": 136, "id": "0f58f765", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████| 645/645 [17:01:21<00:00, 95.01s/it]\n" ] } ], "source": [ "import funcy\n", "queue_items = list(bundle_info.items())\n", "processed_metas = []\n", "for items_chunk in tqdm.tqdm(funcy.chunks(100, queue_items), total=int(np.ceil(len(queue_items)/100))):\n", " out = Parallel(n_jobs=16, prefer=\"threads\")(delayed(process_bundle)(e) for e in items_chunk)\n", " n_fail = len([e for e in out if not e['success']])\n", " if n_fail > 0:\n", " print(f\"{n_fail}/{len(items_chunk)} failed.\")\n", " for e in out:\n", " if not e[\"success\"]:\n", " continue\n", " m = e[\"bundle_info\"]\n", " bundel_id = m[\"id\"]\n", " processed_metas.append({\n", " \"id\": f\"{bundel_id}_full\",\n", " \"bundel_id\": bundel_id,\n", " \"duration_s\": m[\"duration_s\"],\n", " \"s3_filepath\": m[\"s3_filepath_full\"],\n", " })\n", " processed_metas.append({\n", " \"id\": f\"{bundel_id}_lead_vocals\",\n", " \"bundel_id\": bundel_id,\n", " \"duration_s\": m[\"duration_s\"],\n", " \"s3_filepath\": m[\"s3_filepath_lead_vocals\"],\n", " })\n", " processed_metas.append({\n", " \"id\": f\"{bundel_id}_no_vocals\",\n", " \"bundel_id\": bundel_id,\n", " \"duration_s\": m[\"duration_s\"],\n", " \"s3_filepath\": m[\"s3_filepath_no_vocals\"],\n", " })\n", "# if len(processed_metas) >= 200:\n", "# break" ] }, { "cell_type": "code", "execution_count": 137, "id": "2c4d633e", "metadata": {}, "outputs": [], "source": [ "write_jsonl(processed_metas, \"/app/suno/tmp/raw_karaoke_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 141, "id": "c2f2cad8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/raw_karaoke_metas.jsonl to s3://suno-data/datasets/bundles/v4/karaoke_stems/metas_raw.jsonl\n" ] } ], "source": [ "!aws s3 cp /app/suno/tmp/raw_karaoke_metas.jsonl s3://suno-data/datasets/bundles/v4/karaoke_stems/metas_raw.jsonl" ] }, { "cell_type": "markdown", "id": "ced208dc", "metadata": {}, "source": [ "### MusDB stems" ] }, { "cell_type": "code", "execution_count": null, "id": "aae22730", "metadata": {}, "outputs": [], "source": [ "import os\n", "import numpy as np\n", "import tqdm\n", "import tempfile\n", "from suno_utils.utils.s3 import _upload_s3_file\n", "\n", "def _upload_to_s3(audio, to_s3_path):\n", " with tempfile.TemporaryDirectory() as td:\n", " fp = os.path.join(td, \"audio.mp3\")\n", " audio.to_hq_mp3(fp)\n", " _upload_s3_file(fp, to_s3_path)\n", "\n", "def _maybe_peak_norm(arr):\n", " max_val = np.abs(arr).max()\n", " if max_val > 1:\n", " arr /= max_val\n", " return arr" ] }, { "cell_type": "code", "execution_count": null, "id": "767717d3", "metadata": {}, "outputs": [], "source": [ "import re\n", "from suno_utils.utils.text import normalize_whitespace\n", "d = \"/app/suno/data/musdb\"\n", "metas_info = []\n", "seen_ids = set()\n", "dl = os.listdir(d)\n", "for idx, dd in tqdm.tqdm(enumerate(dl), total=len(dl)):\n", " arr_full = Audio.from_file(os.path.join(d, dd, \"mixture.wav\"), sample_rate=48_000).array_float\n", " arr_vocals = Audio.from_file(os.path.join(d, dd, \"vocals.wav\"), sample_rate=48_000).array_float\n", " arr_no_vocals = (\n", " Audio.from_file(os.path.join(d, dd, \"drums.wav\"), sample_rate=48_000).array_float +\n", " Audio.from_file(os.path.join(d, dd, \"bass.wav\"), sample_rate=48_000).array_float +\n", " Audio.from_file(os.path.join(d, dd, \"other.wav\"), sample_rate=48_000).array_float\n", " )\n", " arr_full = _maybe_peak_norm(arr_full)\n", " arr_vocals = _maybe_peak_norm(arr_vocals)\n", " arr_no_vocals = _maybe_peak_norm(arr_no_vocals)\n", " assert arr_full.shape == arr_vocals.shape == arr_no_vocals.shape\n", " \n", " audio_full = Audio.from_array_float(arr_full, sample_rate=48_000)\n", " audio_vocals = Audio.from_array_float(arr_vocals, sample_rate=48_000)\n", " audio_no_vocals = Audio.from_array_float(arr_no_vocals, sample_rate=48_000)\n", " s3_prefix = \"s3://suno-data/datasets/harvest/musdb/generated\"\n", " bundle_id = normalize_whitespace(\n", " \"\".join([s for c in dd if ord(s:=c.lower()) < 128])\n", " ).replace(\" \", \"_\")#[:16]\n", " assert bundle_id not in seen_ids\n", " seen_ids.add(bundle_id)\n", " s3_filepath_full = os.path.join(s3_prefix, f\"{bundle_id}__full.mp3\")\n", " s3_filepath_vocals = os.path.join(s3_prefix, f\"{bundle_id}__vocals.mp3\")\n", " s3_filepath_no_vocals = os.path.join(s3_prefix, f\"{bundle_id}__no_vocals.mp3\")\n", " _upload_to_s3(audio_full, s3_filepath_full)\n", " _upload_to_s3(audio_vocals, s3_filepath_vocals)\n", " _upload_to_s3(audio_no_vocals, s3_filepath_no_vocals)\n", " metas_info.append({\n", " \"id\": f\"{bundle_id}__full\",\n", " \"bundle_id\": bundle_id,\n", " \"duration_s\": round(audio_full.duration_s, 2),\n", " \"s3_filepath\": s3_filepath_full,\n", " })\n", " metas_info.append({\n", " \"id\": f\"{bundle_id}__vocals\",\n", " \"bundle_id\": bundle_id,\n", " \"duration_s\": round(audio_vocals.duration_s, 2),\n", " \"s3_filepath\": s3_filepath_vocals,\n", " })\n", " metas_info.append({\n", " \"id\": f\"{bundle_id}__no_vocals\",\n", " \"bundle_id\": bundle_id,\n", " \"duration_s\": round(audio_no_vocals.duration_s, 2),\n", " \"s3_filepath\": s3_filepath_no_vocals,\n", " })\n", "# if len(metas_info) > 5:\n", "# break" ] }, { "cell_type": "code", "execution_count": null, "id": "80381b9f", "metadata": {}, "outputs": [], "source": [ "# Audio.from_s3(metas_info[0][\"s3_filepath\"], sample_rate=48_000, n_channels=2).play()\n", "# Audio.from_s3(metas_info[1][\"s3_filepath\"], sample_rate=48_000, n_channels=2).play()\n", "# Audio.from_s3(metas_info[2][\"s3_filepath\"], sample_rate=48_000, n_channels=2).play()\n", "# Audio.from_s3(metas_info[3][\"s3_filepath\"], sample_rate=48_000, n_channels=2).play()" ] }, { "cell_type": "code", "execution_count": null, "id": "fe4180e8", "metadata": {}, "outputs": [], "source": [ "write_jsonl(metas_info, \"/app/suno/tmp/raw_musdb_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "3a9b93ce", "metadata": {}, "outputs": [], "source": [ "# !aws s3 cp /app/suno/tmp/raw_musdb_metas.jsonl s3://suno-data/datasets/bundles/v4/musdb_stems/metas_raw.jsonl" ] }, { "cell_type": "markdown", "id": "9bc0fe3f", "metadata": {}, "source": [ "### Podcasts" ] }, { "cell_type": "code", "execution_count": null, "id": "34aa9168", "metadata": {}, "outputs": [], "source": [ "# make download list\n", "from suno_utils.utils.podcasts import load_podcast_db\n", "load_f = funcy.partial(load_podcast_db, english_only=False, anchor_only=False)\n", "podcast_df = read_from_s3(\"s3://suno-data/datasets/harvest/podcasts/podcastindex_feeds.db\", read_f=load_f)\n", "podcast_df[\"tags\"] = (podcast_df[\"category1\"] + \", \" + podcast_df[\"category2\"]).str.strip(\" ,\")\n", "assert(podcast_df[\"id\"].nunique() == podcast_df.shape[0])" ] }, { "cell_type": "code", "execution_count": 235, "id": "8929a93b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "4,109,623\n", "4,109,623\n", "2,832,597\n", "709,521\n", "701,958\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_153869/2981800771.py:15: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n", " df = df.groupby(\"language\").apply(lambda g: g.sample(n=min(len(g), s_max[g.name]), random_state=42)).reset_index(drop=True)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "328,454\n" ] } ], "source": [ "print(f\"{podcast_df.shape[0]:,}\")\n", "df = podcast_df[~podcast_df[\"newest_item_pubdate\"].isnull()]\n", "print(f\"{df.shape[0]:,}\")\n", "df = df[df[\"newest_item_pubdate\"].astype(str).str[:4].astype(int) >= 2020]\n", "print(f\"{df.shape[0]:,}\")\n", "df = df[df[\"episode_count\"].fillna(0) >= 20]\n", "print(f\"{df.shape[0]:,}\")\n", "s = df[\"language\"].value_counts()\n", "s = s[s>=100]\n", "df = df[df[\"language\"].isin(set(s.index))]\n", "print(f\"{df.shape[0]:,}\")\n", "n_samples = lambda x: int(round((min(x, max(1, x / 1_000) ** 0.8) * 1_000) / 10) * 10)\n", "s_max = s.map(n_samples)\n", "# 300k->100k, 10k->10k\n", "df = df.groupby(\"language\").apply(lambda g: g.sample(n=min(len(g), s_max[g.name]), random_state=42)).reset_index(drop=True)\n", "df = df.sample(frac=1.0, random_state=42).reset_index(drop=True)\n", "print(f\"{df.shape[0]:,}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "7106f782", "metadata": {}, "outputs": [], "source": [ "# TODO: filter more by athor, tags etc?\n", "# TODO: for each of the above URLs collect max 5 random episodes post 2015" ] }, { "cell_type": "code", "execution_count": 253, "id": "8db81d06", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import feedparser\n", "import random\n", "import datetime\n", "import concurrent.futures\n", "from tqdm import tqdm\n", "import time\n", "import socket\n", "from urllib.request import urlopen\n", "\n", "def parse_rss(url):\n", " \"\"\"\n", " Parse an RSS feed URL and extract podcast and episode information.\n", " \n", " Args:\n", " url (str): The RSS feed URL\n", " \n", " Returns:\n", " dict: A dictionary containing podcast metadata and selected episodes\n", " \"\"\"\n", " try:\n", " # Set socket timeout to 5 seconds\n", " original_timeout = socket.getdefaulttimeout()\n", " socket.setdefaulttimeout(5)\n", " \n", " # Parse the RSS feed with timeout\n", " feed = feedparser.parse(url)\n", " \n", " # Reset timeout to original value\n", " socket.setdefaulttimeout(original_timeout)\n", " \n", " # If the feed is empty or invalid, return None\n", " if not feed or not feed.get('feed') or not feed.get('entries'):\n", " return {\n", " 'url': url,\n", " 'status': 'error',\n", " 'message': 'Invalid or empty feed'\n", " }\n", " \n", " # Extract podcast metadata\n", " podcast_data = {\n", " 'url': url,\n", " 'status': 'success',\n", " 'title': feed.feed.get('title', 'Unknown'),\n", " 'author': feed.feed.get('author', feed.feed.get('itunes_author', 'Unknown')),\n", " 'description': feed.feed.get('description', feed.feed.get('subtitle', 'No description')),\n", " 'link': feed.feed.get('link', ''),\n", " 'image': feed.feed.get('image', {}).get('href', \n", " feed.feed.get('itunes_image', {}).get('href', '')),\n", " 'total_episodes': len(feed.entries),\n", " }\n", " \n", " # Filter episodes after 2012\n", " post_2012_episodes = []\n", " for entry in feed.entries:\n", " # Try to get published date\n", " pub_date = entry.get('published_parsed')\n", " if pub_date:\n", " # Convert to datetime object\n", " date = datetime.datetime(*pub_date[:6])\n", " if date.year > 2012:\n", " # Get audio information\n", " audio_url = \"\"\n", " audio_type = \"\"\n", " audio_length = 0\n", " \n", " # Try to get enclosure info (contains the audio file)\n", " if 'enclosures' in entry and entry.enclosures:\n", " for enclosure in entry.enclosures:\n", " if enclosure.get('type', '').startswith('audio/'):\n", " audio_url = enclosure.get('url', '')\n", " audio_type = enclosure.get('type', '')\n", " audio_length = enclosure.get('length', 0)\n", " break\n", " \n", " # If no enclosure found, try alternate fields that might contain audio URL\n", " if not audio_url and 'links' in entry:\n", " for link in entry.links:\n", " if link.get('type', '').startswith('audio/'):\n", " audio_url = link.get('href', '')\n", " audio_type = link.get('type', '')\n", " break\n", " \n", " # Create episode dict with enhanced information\n", " episode = {\n", " 'title': entry.get('title', 'Unknown Title'),\n", " 'published': entry.get('published', 'Unknown Date'),\n", " 'published_date': date.strftime('%Y-%m-%d'),\n", " 'duration': entry.get('itunes_duration', 'Unknown Duration'),\n", " 'link': entry.get('link', ''),\n", " 'description': entry.get('description', entry.get('subtitle', 'No description')),\n", " 'audio_url': audio_url,\n", " 'audio_type': audio_type,\n", " 'audio_length_bytes': audio_length,\n", " 'guid': entry.get('id', entry.get('guid', '')),\n", " 'itunes_episode': entry.get('itunes_episode', ''),\n", " 'itunes_season': entry.get('itunes_season', ''),\n", " 'itunes_explicit': entry.get('itunes_explicit', 'Unknown'),\n", " 'image': entry.get('image', {}).get('href', \n", " entry.get('itunes_image', {}).get('href', ''))\n", " }\n", " post_2012_episodes.append(episode)\n", " \n", " # Randomly select 3 episodes (or all if less than 3)\n", " selected_episodes = []\n", " if post_2012_episodes:\n", " if len(post_2012_episodes) <= 3:\n", " selected_episodes = post_2012_episodes\n", " else:\n", " selected_episodes = random.sample(post_2012_episodes, 3)\n", " \n", " podcast_data['selected_episodes'] = selected_episodes\n", " podcast_data['post_2012_episode_count'] = len(post_2012_episodes)\n", " \n", " return podcast_data\n", " \n", " except Exception as e:\n", " return {\n", " 'url': url,\n", " 'status': 'error',\n", " 'message': str(e)\n", " }\n", "\n", "def process_chunk(urls):\n", " \"\"\"\n", " Process a chunk of URLs using multithreading.\n", " \n", " Args:\n", " urls (list): List of URLs to process\n", " \n", " Returns:\n", " list: List of processed results\n", " \"\"\"\n", " results = []\n", " # Use ThreadPoolExecutor for parallel processing\n", " with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:\n", " # Submit tasks and collect futures\n", " futures = {executor.submit(parse_rss, url): url for url in urls}\n", " \n", " # Process completed futures as they complete\n", " for future in concurrent.futures.as_completed(futures):\n", " try:\n", " result = future.result(timeout=10) # Additional timeout as safety\n", " results.append(result)\n", " except concurrent.futures.TimeoutError:\n", " url = futures[future]\n", " results.append({\n", " 'url': url,\n", " 'status': 'error',\n", " 'message': 'Processing timed out after 10 seconds'\n", " })\n", " except Exception as e:\n", " url = futures[future]\n", " results.append({\n", " 'url': url,\n", " 'status': 'error',\n", " 'message': f'Processing error: {str(e)}'\n", " })\n", " \n", " return results\n", "\n", "def collect_podcast_data(df, chunk_size=10):\n", " \"\"\"\n", " Collect podcast data from RSS feeds in a DataFrame.\n", " \n", " Args:\n", " df (pandas.DataFrame): DataFrame containing podcast RSS URLs\n", " chunk_size (int): Size of chunks to process\n", " \n", " Returns:\n", " list: List of dictionaries containing podcast data\n", " \"\"\"\n", " all_results = []\n", " \n", " # Get total number of chunks\n", " total_chunks = (len(df) + chunk_size - 1) // chunk_size\n", " \n", " # Process in chunks with tqdm progress bar\n", " for i in tqdm(range(0, len(df), chunk_size), total=total_chunks, desc=\"Processing RSS feeds\"):\n", " # Get chunk of URLs\n", " chunk_df = df.iloc[i:i+chunk_size]\n", " chunk_urls = chunk_df['url'].tolist()\n", " \n", " # Process chunk\n", " chunk_results = process_chunk(chunk_urls)\n", " all_results.extend(chunk_results)\n", " \n", " # Add a small delay to be nice to servers\n", " time.sleep(0.5)\n", " \n", " return all_results\n", "\n", "def quick_check(url):\n", " \"\"\"\n", " Quickly check a single podcast RSS feed.\n", " \n", " Args:\n", " url (str): The RSS feed URL\n", " \n", " Returns:\n", " dict: Dictionary containing podcast data\n", " \"\"\"\n", " print(f\"Checking podcast: {url}\")\n", " result = parse_rss(url)\n", " \n", " # Print some basic info\n", " if result['status'] == 'success':\n", " print(f\"Title: {result['title']}\")\n", " print(f\"Author: {result['author']}\")\n", " print(f\"Total episodes: {result['total_episodes']}\")\n", " print(f\"Post-2012 episodes: {result['post_2012_episode_count']}\")\n", " print(\"\\nSelected episodes:\")\n", " for i, episode in enumerate(result['selected_episodes'], 1):\n", " print(f\"\\n{i}. {episode['title']}\")\n", " print(f\" Published: {episode['published']}\")\n", " print(f\" Duration: {episode['duration']}\")\n", " print(f\" Audio URL: {episode['audio_url']}\")\n", " print(f\" Audio type: {episode['audio_type']}\")\n", " print(f\" Audio size: {episode['audio_length_bytes']} bytes\")\n", " if episode['image']:\n", " print(f\" Episode image: {episode['image']}\")\n", " else:\n", " print(f\"Error: {result['message']}\")\n", " \n", " return result" ] }, { "cell_type": "code", "execution_count": 259, "id": "de985f72", "metadata": {}, "outputs": [], "source": [ "# brain_science_podcast = quick_check(\"https://brainsciencepodcast.libsyn.com/rss\")" ] }, { "cell_type": "code", "execution_count": null, "id": "8209a989", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Processing RSS feeds: 40%|████████▎ | 1306/3285 [5:13:17<7:28:32, 13.60s/it]" ] } ], "source": [ "results = collect_podcast_data(df, chunk_size=100)" ] }, { "cell_type": "code", "execution_count": 317, "id": "fedf3b05", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "288,123/328,454 successfull collections\n", "778,558 total episodes\n" ] } ], "source": [ "def time_to_seconds(time_str):\n", " try:\n", " if time_str.count(\":\") == 2:\n", " h, m, s = map(int, time_str.split(':'))\n", " elif time_str.count(\":\") == 1:\n", " h = 0\n", " m, s = map(int, time_str.split(':'))\n", " elif time_str.count(\":\") == 0:\n", " h, m = 0, 0\n", " s = int(time_str)\n", " else:\n", " raise NotImplementedError(time_str)\n", " except:\n", " return None\n", " return h * 3600 + m * 60 + s\n", "\n", "print(f\"{len([e for e in results if e['status'] == 'success']):,}/{df.shape[0]:,} successfull collections\")\n", "n = 0\n", "flat_results = []\n", "seen_urls = set()\n", "for e, _id in zip(results, df[\"id\"].tolist()):\n", " e[\"id\"] = _id\n", " if e['status'] != 'success':\n", " continue\n", " for ee in e[\"selected_episodes\"]:\n", " if ee[\"audio_url\"] in seen_urls:\n", " continue\n", " duration_s = time_to_seconds(ee[\"duration\"])\n", " if duration_s is None or duration_s >= 2*60*60 or duration_s <= 2*60:\n", " continue\n", " n += 1\n", " flat_results.append({\n", " \"parent_id\": str(_id),\n", " \"duration_s\": duration_s,\n", " \"url\": ee[\"audio_url\"],\n", " })\n", " seen_urls.add(ee[\"audio_url\"])\n", "print(f\"{n:,} total episodes\")" ] }, { "cell_type": "code", "execution_count": 323, "id": "1e3278c1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "466,282 hours\n" ] } ], "source": [ "print(f\"{sum([m['duration_s'] for m in flat_results])/60/60:,.0f} hours\")" ] }, { "cell_type": "code", "execution_count": 318, "id": "152ec0a8", "metadata": {}, "outputs": [], "source": [ "write_jsonl(results, \"/app/suno/tmp/podcast_sample.jsonl\")\n", "write_jsonl(flat_results, \"/app/suno/tmp/podcast_sample_episodes.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "a530eb0f", "metadata": {}, "outputs": [], "source": [ "# TODO: do some quality checks" ] }, { "cell_type": "code", "execution_count": null, "id": "5567df20", "metadata": {}, "outputs": [], "source": [ "# TODO: entire below sequence (filtered to what we could actually download and randomize into single good info file)\n", "# also move everything to s3" ] }, { "cell_type": "code", "execution_count": 319, "id": "b885fba5", "metadata": {}, "outputs": [], "source": [ "# raw_youtube_music_metas = []\n", "# for m in tqdm.tqdm(metas_raw):\n", "# if m[\"id\"] not in metas_extra_map:\n", "# continue\n", "# mm = metas_extra_map[m[\"id\"]]\n", "# new_m = {\n", "# \"id\": m[\"id\"],\n", "# \"s3_filepath\": m[\"s3_filepath\"],\n", "# \"duration_s\": m[\"duration_s\"],\n", "# \"views\": m[\"view_count\"],\n", "# \"title\": m[\"title\"],\n", "# \"artists\": m[\"artists\"],\n", "# \"tags\": mm[\"private_tags\"],\n", "# }\n", "# if \"text\" in mm:\n", "# new_m[\"text\"] = mm[\"text\"]\n", "# new_m[\"lang\"] = mm[\"lang\"]\n", "# if \"genre_guess\" in m:\n", "# new_m[\"genres\"] = m[\"genre_guess\"]\n", "# if \"mood_guess\" in m:\n", "# new_m[\"moods\"] = m[\"mood_guess\"]\n", "# if \"album\" in mm:\n", "# new_m[\"album\"] = mm[\"album\"]\n", "# raw_youtube_music_metas.append(new_m)\n", "# random.seed(6006)\n", "# random.shuffle(raw_youtube_music_metas)" ] }, { "cell_type": "code", "execution_count": 320, "id": "e8f21477", "metadata": {}, "outputs": [], "source": [ "# print(f\"{len(raw_youtube_music_metas):,} tracks\")\n", "# print(f\"{round(sum(m['duration_s'] for m in raw_youtube_music_metas)/60/60):,} hours\")\n", "# print(f\"{round(sum(m['duration_s'] for m in raw_youtube_music_metas if 'text' in m)/60/60):,} hours with lyrics\")" ] }, { "cell_type": "code", "execution_count": 321, "id": "2a25fda0", "metadata": {}, "outputs": [], "source": [ "# write_jsonl(raw_youtube_music_metas, \"/app/suno/tmp/raw_youtube_music_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 322, "id": "b0758c40", "metadata": {}, "outputs": [], "source": [ "# !aws s3 cp /app/suno/tmp/raw_youtube_music_metas.jsonl s3://suno-data/datasets/bundles/v4/youtube_music/metas_raw.jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "acdb9401", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6c250194", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "29c40fed", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "3ccd6955", "metadata": {}, "source": [ "## Make filtered datasets" ] }, { "cell_type": "code", "execution_count": 1, "id": "cfa12e5c", "metadata": {}, "outputs": [], "source": [ "import random\n", "import funcy\n", "import tqdm\n", "from suno_utils.utils.text import read_jsonl, write_jsonl" ] }, { "cell_type": "code", "execution_count": 2, "id": "29e5ff77", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n" ] } ], "source": [ "import fasttext\n", "from suno_utils.harvest.youtube.language_classify import get_text_lang_p\n", "fasttext_lang_model = fasttext.load_model(\"/app/suno/models/lid.176.bin\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "313ef3d7", "metadata": {}, "outputs": [], "source": [ "import re\n", "\n", "def _space_repl(m):\n", " s = m.group()\n", " n_newline = s.count(\"\\n\")\n", " if n_newline >= 2:\n", " return \"\\n\\n\"\n", " elif n_newline == 1:\n", " return \"\\n\"\n", " return \" \"\n", "\n", "\n", "def _simplify_whitespace(text, retain_newlines=True):\n", " \"\"\"simplify while respecting up to 2 newlines\"\"\"\n", " if retain_newlines:\n", " text = re.sub(r\"\\s+\", _space_repl, text).strip()\n", " else:\n", " text = re.sub(r\"\\s+\", \" \", text).strip()\n", " return text" ] }, { "cell_type": "code", "execution_count": 4, "id": "93f4834f", "metadata": {}, "outputs": [], "source": [ "import hashlib\n", "import random\n", "\n", "def simple_hash_artist_name(artist_name: str, seed: int = 0) -> str:\n", " # Normalize the artist name (case insensitive)\n", " normalized_name = artist_name.strip().lower()\n", " \n", " # Combine the name and seed into a single string\n", " seed_string = f\"{normalized_name}{seed}\"\n", " \n", " # Use a hash function (SHA256 for simplicity)\n", " hash_object = hashlib.sha256(seed_string.encode('utf-8'))\n", " \n", " # Convert the hash into a base36-like string (numbers and lowercase letters)\n", " hash_digest = hash_object.hexdigest()\n", " \n", " # Use only the first few characters for simplicity (e.g., 8 characters)\n", " shortened_hash = int(hash_digest, 16) % (36**8)\n", " \n", " # Convert the integer into a base36-like string\n", " base36 = \"0123456789abcdefghijklmnopqrstuvwxyz\"\n", " result = \"\"\n", " while shortened_hash > 0:\n", " shortened_hash, remainder = divmod(shortened_hash, 36)\n", " result = base36[remainder] + result\n", " \n", " # Pad with leading zeros if result is too short\n", " return result.zfill(8)" ] }, { "cell_type": "code", "execution_count": null, "id": "40916fbf", "metadata": {}, "outputs": [], "source": [ "# Datasets\n", "\n", "# Discogs (+subset)\n", "# Discogs covers\n", "# Genius\n", "# Youtube music\n", "# Deezer\n", "# Pond5\n", "# Imslp\n", "# Karaoke stems\n", "# Musdb stems" ] }, { "cell_type": "markdown", "id": "da28691b", "metadata": {}, "source": [ "### Discogs (+subset)" ] }, { "cell_type": "code", "execution_count": 5, "id": "6537148b", "metadata": {}, "outputs": [], "source": [ "# s3://suno-data/datasets/bundles/v4/discogs/metas_raw.jsonl\n", "metas_raw = read_jsonl(\"/app/suno/tmp/raw_discogs_metas.jsonl\")\n", "metas_subset_raw = read_jsonl(\"/app/suno/tmp/raw_discogs_subset_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 141, "id": "e3172d1a", "metadata": {}, "outputs": [], "source": [ "# filter discogs & musicbrainz with Pat&Victor set\n", "with open(\"/home/georg/notebooks/bk_alpaca/dataprep/filtered_tags.txt\") as f:\n", " allowed_tags = set(f.read().strip().split(\"\\n\"))\n", "\n", "def _process_meta(m):\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"s3_filepath\": m[\"s3_filepath\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " }\n", " if \"playlist_ids\" in m:\n", " new_m[\"playlist_ids\"] = m[\"playlist_ids\"]\n", " artists = None\n", " if \"artists\" in m:\n", " # technically this contains things like 'nature sounds' but we probably don't care\n", " artists = list(set([\n", " mm[\"name\"].strip() for mm in m[\"artists\"] if len(mm[\"name\"].strip()) > 0\n", " ]))\n", " release_year = None\n", " discogs_genres = []\n", " if \"discogs\" in m:\n", "# artists = list(set([\n", "# mm[\"name\"].strip() for mm in m[\"discogs\"][\"artists\"] if len(mm[\"name\"].strip()) > 0\n", "# ]))\n", " discogs_genres += m[\"discogs\"].get(\"genres\", []) + m[\"discogs\"].get(\"styles\", [])\n", " tmp_release_year = m[\"discogs\"].get(\"released\", \"\")\n", " if tmp_release_year is not None:\n", " tmp_release_year = re.search(r\"\\b[0-9]{4}\\b\", tmp_release_year)\n", " if tmp_release_year and (1900 <= int(tmp_release_year.group()) <= 2050):\n", " release_year = tmp_release_year.group()\n", " musicbrainz_genres = []\n", " for ss in [\"musicbrainz_album\", \"musicbrainz_track\"]:\n", " if ss not in m:\n", " continue\n", " tmp_release_year = m[ss].get(\"year\", \"\")\n", " if tmp_release_year is not None:\n", " tmp_release_year = re.search(r\"\\b[0-9]{4}\\b\", tmp_release_year)\n", " if tmp_release_year and (1900 <= int(tmp_release_year.group()) <= 2050):\n", " release_year = tmp_release_year.group()\n", " for sss in [\n", " \"release_group_tags\",\n", " \"release_group_genres\",\n", " \"artist_tags\",\n", " \"artist_genres\",\n", " \"release_genres\",\n", " \"release_tags\",\n", " \"recording_instruments\",\n", " ]:\n", " musicbrainz_genres.extend(m[ss].get(sss, []))\n", " discogs_genres = list(set(discogs_genres) & allowed_tags)\n", " musicbrainz_genres = list(set(musicbrainz_genres) & allowed_tags)\n", " sos_genre = m.get(\"sos_genre\", [])\n", " rym_genres = m.get(\"rym_genres\", [])\n", " # rym has 2228 genres (capitalized), 214 descriptors (lowercased)\n", " if isinstance(rym_genres, str):\n", " # formatting bug\n", " rym_genres = re.sub(r\"([a-z][a-z])([A-Z])\", \"\\\\1, \\\\2\", rym_genres)\n", " if not isinstance(rym_genres, list):\n", " rym_genres = [s.strip() for s in rym_genres.split(\",\") if len(s.strip()) > 0]\n", " tags = (\n", " discogs_genres + musicbrainz_genres + sos_genre + rym_genres\n", " )\n", " if release_year is not None:\n", " tags.append(release_year)\n", " if \"bpm\" in m:\n", " tags.append(f\"{m['bpm']} BPM\")\n", " if \"hooktheory\" in m:\n", " tags.extend(m[\"hooktheory\"][\"keys\"])\n", " tags.append(f\"HTK:{','.join(m['hooktheory']['keys'])}\")\n", " tags.append(f\"HTCC:{int(round(m['hooktheory']['Chord Complexity']))}\")\n", " tags.append(f\"HTMC:{int(round(m['hooktheory']['Melodic Complexity']))}\")\n", " tags.append(f\"HTCMT:{int(round(m['hooktheory']['Chord-Melody Tension']))}\")\n", " tags.append(f\"HTCPN:{int(round(m['hooktheory']['Chord Progression Novelty']))}\")\n", " tags.append(f\"HTCBM:{int(round(m['hooktheory']['Chord Bass Melody']))}\")\n", " tags = sorted(list(set(tags)))\n", " if len(tags) > 0:\n", " new_m[\"tags\"] = tags\n", " if \"ultimate_guitar\" in m:\n", " new_m[\"text\"] = _simplify_whitespace(m[\"ultimate_guitar\"][\"aligned_chords_lyrics\"])\n", " elif \"lyrics\" in m:\n", " new_m[\"text\"] = _simplify_whitespace(m[\"lyrics\"])\n", " if \"text\" in new_m:\n", " plaintext = _simplify_whitespace(re.sub(r\"\\[.*?\\]\", \" \", new_m[\"text\"]), retain_newlines=False)\n", " new_m[\"lang\"] = list(get_text_lang_p(fasttext_lang_model, plaintext, top_k=1).keys())[0]\n", " if artists is not None and len(artists) > 0:\n", " new_m[\"artists\"] = sorted([simple_hash_artist_name(s) for s in artists])\n", " return new_m" ] }, { "cell_type": "code", "execution_count": null, "id": "9fbec7d6", "metadata": {}, "outputs": [], "source": [ "metas_clean = []\n", "for m in tqdm.tqdm(metas_raw):\n", " metas_clean.append(_process_meta(m))\n", " \n", "metas_subset_clean = []\n", "for m in tqdm.tqdm(metas_subset_raw):\n", " metas_subset_clean.append(_process_meta(m))" ] }, { "cell_type": "code", "execution_count": 8, "id": "dc0a50d6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['artists', 'duration_s', 'id', 'lang', 's3_filepath', 'tags', 'text']\n" ] } ], "source": [ "assert len(metas_raw) == len(metas_clean)\n", "assert metas_raw[0][\"id\"] == metas_clean[0][\"id\"]\n", "assert len(metas_clean) == len(set([m[\"id\"] for m in metas_clean]))\n", "assert (\n", " \"id\" in metas_clean[0] and\n", " \"duration_s\" in metas_clean[0] and\n", " \"s3_filepath\" in metas_clean[0]\n", ")\n", "print(list(sorted(set(funcy.flatten([list(m.keys()) for m in metas_clean[:10_000]])))))\n", "# ['artists', 'duration_s', 'id', 'lang', 'playlist_ids', 's3_filepath', 'tags', 'text']" ] }, { "cell_type": "code", "execution_count": 16, "id": "8a99ef0f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['artists', 'duration_s', 'id', 'lang', 's3_filepath', 'tags', 'text']\n" ] } ], "source": [ "assert len(metas_subset_raw) == len(metas_subset_clean)\n", "assert metas_subset_clean[0][\"id\"] == metas_subset_clean[0][\"id\"]\n", "assert len(metas_subset_clean) == len(set([m[\"id\"] for m in metas_subset_clean]))\n", "assert (\n", " \"id\" in metas_subset_clean[0] and\n", " \"duration_s\" in metas_subset_clean[0] and\n", " \"s3_filepath\" in metas_subset_clean[0]\n", ")\n", "print(list(sorted(set(funcy.flatten([list(m.keys()) for m in metas_subset_clean[:10_000]])))))\n", "# ['artists', 'duration_s', 'id', 'lang', 'playlist_ids', s3_filepath', 'tags', 'text']" ] }, { "cell_type": "code", "execution_count": 164, "id": "fa268f0e", "metadata": {}, "outputs": [], "source": [ "# l = []\n", "# for m in tqdm.tqdm(metas_clean):\n", "# if \"tags\" in m and \"bagpipes\" in ';'.join(m[\"tags\"]).lower():\n", "# l.append(m)\n", "# len(l)" ] }, { "cell_type": "code", "execution_count": 215, "id": "0a3e60fc", "metadata": {}, "outputs": [], "source": [ "# for m in tqdm.tqdm(metas_raw):\n", "# # for m in tqdm.tqdm(metas_clean):\n", "# if m[\"id\"] == \"g-cGbdbs7k0\":\n", "# print(\"found\")\n", "# break" ] }, { "cell_type": "code", "execution_count": null, "id": "162375f1", "metadata": {}, "outputs": [], "source": [ "# # check for known songs/artists and how well they are labeled\n", "# \"8AHCfZTRGiI\" - johnny cash\n", "# \"p1JPKLa-Ofc\" - beyonce\n", "# \"Dkk9gvTmCXY\", \"e-ORhEE9VVg\" - taylor swift\n", "# \"uelHwf8o7_U\" - eminem\n", "# \"60ItHLz5WEA\", \"1-xGerv5FOk\" - alan walker\n", "# \"xFrGuyw1V8s\", \"XEjLoHdbVeE\" - abba" ] }, { "cell_type": "code", "execution_count": null, "id": "c772954b", "metadata": {}, "outputs": [], "source": [ "# TODOS:\n", "# filtering based on various stuff\n", "# merge all this metadata into genius/ytm/deezer\n", "# minz local lyrics" ] }, { "cell_type": "code", "execution_count": 217, "id": "b75b310f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/clean_discogs_v0_metas.jsonl to s3://suno-data/datasets/bundles/v4/discogs/metas_v0.jsonl\n" ] } ], "source": [ "write_jsonl(metas_clean, \"/app/suno/tmp/clean_discogs_v0_metas.jsonl\")\n", "# !aws s3 cp /app/suno/tmp/clean_discogs_v0_metas.jsonl s3://suno-data/datasets/bundles/v4/discogs/metas_v0.jsonl" ] }, { "cell_type": "code", "execution_count": 18, "id": "71628dd5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/clean_discogs_subset_v0_metas.jsonl to s3://suno-data/datasets/bundles/v4/discogs_subset/metas_v0.jsonl\n" ] } ], "source": [ "write_jsonl(metas_subset_clean, \"/app/suno/tmp/clean_discogs_subset_v0_metas.jsonl\")\n", "# !aws s3 cp /app/suno/tmp/clean_discogs_subset_v0_metas.jsonl s3://suno-data/datasets/bundles/v4/discogs_subset/metas_v0.jsonl" ] }, { "cell_type": "code", "execution_count": 43, "id": "dbda172d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████| 52220969/52220969 [00:16<00:00, 3179925.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "0 found\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "l = []\n", "for m in tqdm.tqdm(metas_clean):\n", " if \"glockenspiel\" in \" \".join(m.get(\"tags\", [])).lower():\n", " l.append(m)\n", "print(f\"{len(l):,} found\")" ] }, { "cell_type": "code", "execution_count": 38, "id": "07c75aef", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'id': 'ppD6jf_tcXs',\n", " 's3_filepath': 's3://suno-data/shared/nfdg/ppD6jf_tcXs/ppD6jf_tcXs/audio.webm',\n", " 'duration_s': 323,\n", " 'tags': ['Vinahouse', 'Việt EDM'],\n", " 'artists': ['7c0j1s99']}" ] }, "execution_count": 38, "metadata": {}, "output_type": "execute_result" } ], "source": [ "random.choice(l)" ] }, { "cell_type": "markdown", "id": "ef89a132", "metadata": {}, "source": [ "### Covers" ] }, { "cell_type": "code", "execution_count": 138, "id": "421ebff4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "908,582 real tracks\n", "15,780,354 covers\n" ] } ], "source": [ "metas_raw = read_jsonl(\"/app/suno/tmp/raw_covers_metas.jsonl\")\n", "# ids_discogs = read_jsonl(\n", "# \"/app/suno/tmp/raw_discogs_metas.jsonl\", allowed_keys=[\"id\", \"title\"], progress=True\n", "# )\n", "print(f\"{len(metas_raw):,} real tracks\")\n", "print(f\"{sum([len(m['covers']) for m in metas_raw]):,} covers\")" ] }, { "cell_type": "code", "execution_count": 139, "id": "113f9c3b", "metadata": {}, "outputs": [], "source": [ "# existing_discogs_ids = set([m[\"id\"] for m in ids_discogs])\n", "# print(len(existing_discogs_ids))\n", "# existing_ids = set([m[\"id\"] for m in metas_raw])\n", "# print(len(existing_ids))" ] }, { "cell_type": "code", "execution_count": 142, "id": "54c01104", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████| 908582/908582 [03:18<00:00, 4578.55it/s]\n" ] } ], "source": [ "metas_clean = []\n", "seen_ids = set() # make sure covers only map to a single true song (for simplicity)\n", "for m in tqdm.tqdm(metas_raw):\n", " new_m = _process_meta(m)\n", " new_mms = []\n", " for mm in m[\"covers\"]:\n", " if mm[\"id\"] in seen_ids:\n", " continue\n", "# id_seems_real = mm[\"id\"] in existing_discogs_ids and mm[\"id\"] not in existing_ids\n", " name_seems_match = (\n", " len(m[\"title\"]) >= 10 and len(mm[\"title\"]) >= 10 and \n", " len(set(m[\"title\"].lower()) & set(mm[\"title\"].lower())) >= 8 and\n", " re.search(r\"cover|remix|version\", mm[\"title\"].lower()) and\n", " not re.search(r\"cover|remix|version\", m[\"title\"].lower())\n", " )\n", "# is_reliable = id_seems_real or not name_seems_match\n", " if not name_seems_match:\n", " continue\n", " new_mm = {\n", " \"id\": mm[\"id\"],\n", " \"duration_s\": mm[\"duration_s\"],\n", " \"s3_filepath\": mm[\"s3_filepath\"],\n", " \"parent_id\": m[\"id\"],\n", " }\n", " new_mms.append(new_mm)\n", " seen_ids.add(mm[\"id\"])\n", " if len(new_mms) > 0:\n", " metas_clean.append(new_m)\n", " metas_clean.extend(new_mms[:10]) # use max 10\n", " \n", "# 402,918 real tracks\n", "# 1,466,195 covers" ] }, { "cell_type": "code", "execution_count": 143, "id": "25196fb2", "metadata": {}, "outputs": [], "source": [ "# TODO: this is very aggressive filtering, making this a tiny set. relax the checks" ] }, { "cell_type": "code", "execution_count": 144, "id": "5b4b547c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "402,918 real tracks\n", "1,466,195 covers\n" ] } ], "source": [ "n_real = len([m for m in metas_clean if \"parent_id\" not in m])\n", "print(f\"{n_real:,} real tracks\")\n", "print(f\"{len(metas_clean)-n_real:,} covers\")" ] }, { "cell_type": "code", "execution_count": 145, "id": "64b296af", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['artists', 'duration_s', 'id', 'lang', 'parent_id', 's3_filepath', 'tags', 'text']\n" ] } ], "source": [ "assert len(metas_raw) < len(metas_clean)\n", "# assert metas_raw[0][\"id\"] == metas_clean[0][\"id\"]\n", "# assert len(metas_clean) == len(set([m[\"id\"] for m in metas_clean])) # song can be cover of multiple??\n", "assert (\n", " \"id\" in metas_clean[0] and\n", " \"duration_s\" in metas_clean[0] and\n", " \"s3_filepath\" in metas_clean[0]\n", ")\n", "print(list(sorted(set(funcy.flatten([list(m.keys()) for m in metas_clean[:10_000]])))))\n", "# ['artists', 'duration_s', id', 'lang', 'parent_id', 's3_filepath', 'tags', 'text']" ] }, { "cell_type": "code", "execution_count": 147, "id": "870ca678", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/clean_covers_v0_metas.jsonl to s3://suno-data/datasets/bundles/v4/covers/metas_v0.jsonl\n" ] } ], "source": [ "write_jsonl(metas_clean, \"/app/suno/tmp/clean_covers_v0_metas.jsonl\")\n", "# !aws s3 cp /app/suno/tmp/clean_covers_v0_metas.jsonl s3://suno-data/datasets/bundles/v4/covers/metas_v0.jsonl" ] }, { "cell_type": "markdown", "id": "e22ea37c", "metadata": {}, "source": [ "### Genius" ] }, { "cell_type": "code", "execution_count": 89, "id": "1880feb8", "metadata": {}, "outputs": [], "source": [ "metas_raw = read_jsonl(\"/app/suno/tmp/raw_genius_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 110, "id": "a39d7617", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████| 2090009/2090009 [00:03<00:00, 649137.83it/s]\n" ] } ], "source": [ "metas_clean = []\n", "for m in tqdm.tqdm(metas_raw):\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " \"s3_filepath\": m[\"s3_filepath\"],\n", " \"text\": m[\"lyrics\"],\n", " \"lang\": m[\"lang\"],\n", " }\n", " tags = m.get(\"genius_tags\", []) + m.get(\"tags\", [])\n", " if len(tags) > 0\n", " new_m[\"tags\"] = list(set(tags))\n", " metas_clean.append(new_m)" ] }, { "cell_type": "code", "execution_count": 111, "id": "419cc05e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['duration_s', 'id', 'lang', 's3_filepath', 'tags', 'text']\n" ] } ], "source": [ "assert len(metas_raw) == len(metas_clean)\n", "assert metas_raw[0][\"id\"] == metas_clean[0][\"id\"]\n", "assert len(metas_clean) == len(set([m[\"id\"] for m in metas_clean]))\n", "assert (\n", " \"id\" in metas_clean[0] and\n", " \"duration_s\" in metas_clean[0] and\n", " \"s3_filepath\" in metas_clean[0]\n", ")\n", "print(list(sorted(set(funcy.flatten([list(m.keys()) for m in metas_clean[:10_000]])))))\n", "# 'duration_s', 'id', 'lang', 's3_filepath', 'tags', 'text'" ] }, { "cell_type": "code", "execution_count": 193, "id": "04af8380", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/clean_genius_v0_metas.jsonl to s3://suno-data/datasets/bundles/v4/genius/metas_v0.jsonl\n" ] } ], "source": [ "write_jsonl(metas_clean, \"/app/suno/tmp/clean_genius_v0_metas.jsonl\")\n", "# !aws s3 cp /app/suno/tmp/clean_genius_v0_metas.jsonl s3://suno-data/datasets/bundles/v4/genius/metas_v0.jsonl" ] }, { "cell_type": "markdown", "id": "96d49958", "metadata": {}, "source": [ "### Youtube music" ] }, { "cell_type": "code", "execution_count": 113, "id": "6fc5e067", "metadata": {}, "outputs": [], "source": [ "metas_raw = read_jsonl(\"/app/suno/tmp/raw_youtube_music_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 115, "id": "032d9005", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████| 2013545/2013545 [00:02<00:00, 781821.38it/s]\n" ] } ], "source": [ "metas_clean = []\n", "for m in tqdm.tqdm(metas_raw):\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " \"s3_filepath\": m[\"s3_filepath\"],\n", " }\n", " if \"text\" in m:\n", " new_m[\"text\"] = m[\"text\"]\n", " new_m[\"lang\"] = m[\"lang\"]\n", " metas_clean.append(new_m)" ] }, { "cell_type": "code", "execution_count": 116, "id": "f2a72434", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['duration_s', 'id', 'lang', 's3_filepath', 'text']\n" ] } ], "source": [ "assert len(metas_raw) == len(metas_clean)\n", "assert metas_raw[0][\"id\"] == metas_clean[0][\"id\"]\n", "assert len(metas_clean) == len(set([m[\"id\"] for m in metas_clean]))\n", "assert (\n", " \"id\" in metas_clean[0] and\n", " \"duration_s\" in metas_clean[0] and\n", " \"s3_filepath\" in metas_clean[0]\n", ")\n", "print(list(sorted(set(funcy.flatten([list(m.keys()) for m in metas_clean[:10_000]])))))\n", "# 'duration_s', 'id', 'lang', 's3_filepath', 'text'" ] }, { "cell_type": "code", "execution_count": 117, "id": "54726e6e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/clean_youtube_music_v0_metas.jsonl to s3://suno-data/datasets/bundles/v4/youtube_music/metas_v0.jsonl\n" ] } ], "source": [ "write_jsonl(metas_clean, \"/app/suno/tmp/clean_youtube_music_v0_metas.jsonl\")\n", "# !aws s3 cp /app/suno/tmp/clean_youtube_music_v0_metas.jsonl s3://suno-data/datasets/bundles/v4/youtube_music/metas_v0.jsonl" ] }, { "cell_type": "markdown", "id": "e70a27df", "metadata": {}, "source": [ "### Deezer" ] }, { "cell_type": "code", "execution_count": 118, "id": "e7620e84", "metadata": {}, "outputs": [], "source": [ "metas_raw = read_jsonl(\"/app/suno/tmp/raw_deezer_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 121, "id": "14d75153", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████| 768515/768515 [00:01<00:00, 672985.85it/s]\n" ] } ], "source": [ "metas_clean = []\n", "for m in tqdm.tqdm(metas_raw):\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " \"s3_filepath\": m[\"s3_filepath\"],\n", " \"text_lines\": m[\"text_lines\"],\n", " \"lang\": m[\"lang\"],\n", " }\n", " metas_clean.append(new_m)" ] }, { "cell_type": "code", "execution_count": 122, "id": "1108efc5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['duration_s', 'id', 'lang', 's3_filepath', 'text_lines']\n" ] } ], "source": [ "assert len(metas_raw) == len(metas_clean)\n", "assert metas_raw[0][\"id\"] == metas_clean[0][\"id\"]\n", "assert len(metas_clean) == len(set([m[\"id\"] for m in metas_clean]))\n", "assert (\n", " \"id\" in metas_clean[0] and\n", " \"duration_s\" in metas_clean[0] and\n", " \"s3_filepath\" in metas_clean[0]\n", ")\n", "print(list(sorted(set(funcy.flatten([list(m.keys()) for m in metas_clean[:10_000]])))))\n", "# 'duration_s', 'id', 'lang', 's3_filepath', 'text_lines'" ] }, { "cell_type": "code", "execution_count": 123, "id": "a2aac45f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/clean_deezer_v0_metas.jsonl to s3://suno-data/datasets/bundles/v4/deezer/metas_v0.jsonl\n" ] } ], "source": [ "write_jsonl(metas_clean, \"/app/suno/tmp/clean_deezer_v0_metas.jsonl\")\n", "# !aws s3 cp /app/suno/tmp/clean_deezer_v0_metas.jsonl s3://suno-data/datasets/bundles/v4/deezer/metas_v0.jsonl" ] }, { "cell_type": "markdown", "id": "f68ed9cf", "metadata": {}, "source": [ "### Pond5" ] }, { "cell_type": "code", "execution_count": 14, "id": "a96ea55c", "metadata": {}, "outputs": [], "source": [ "metas_raw = read_jsonl(\"/app/suno/tmp/raw_pond5_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "81349a46", "metadata": {}, "outputs": [], "source": [ "# TODO: do some basic filtering with is_blocked\n", "# TODO: filter some tags" ] }, { "cell_type": "code", "execution_count": null, "id": "0596c62f", "metadata": {}, "outputs": [], "source": [ "# filterable patterns\n", "# - Pond5\n", "# - collection\n", "# - https://\n", "# - www.\n", "# - any long strings that are repetitive\n", "\n", "# Note: hard to filter when descriptions only partly weird like 'other versions available'" ] }, { "cell_type": "code", "execution_count": 38, "id": "d5b84b3d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "68,531,502\n", "3,261,788\n" ] } ], "source": [ "all_tags = []\n", "for m in metas_raw:\n", " all_tags.extend(m.get(\"tags\", []))\n", "print(f\"{len(all_tags):,}\")\n", "print(f\"{len(set(all_tags)):,}\")" ] }, { "cell_type": "code", "execution_count": 70, "id": "e75ff070", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "113,195\n" ] } ], "source": [ "import pandas as pd\n", "vc = pd.Series(all_tags).value_counts()\n", "vc_long = vc.loc[[idx for idx in vc.index if len(idx) >= 50]]\n", "blocked_tags = set(vc_long[vc_long>=3].index)\n", "print(f\"{len(blocked_tags):,}\")" ] }, { "cell_type": "code", "execution_count": 76, "id": "a561e539", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████| 2068743/2068743 [00:55<00:00, 37346.85it/s]\n" ] } ], "source": [ "import re\n", "metas_clean = []\n", "for m in tqdm.tqdm(metas_raw):\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " \"s3_filepath\": m[\"s3_filepath\"],\n", " }\n", " tags = [\n", " s for s in m[\"tags\"] \n", " if (\n", " s not in blocked_tags and\n", " not \"pond5\" in s.lower() and\n", " not \"collection\" in s.lower() and\n", " not re.search(r\"https?\\:\\/\\/\", s.lower()) and\n", " not \"www.\" in s.lower()\n", " )\n", " ]\n", " if len(tags) > 0:\n", " new_m[\"tags\"] = tags\n", " metas_clean.append(new_m)" ] }, { "cell_type": "code", "execution_count": 77, "id": "5eea5cd0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['duration_s', 'id', 's3_filepath', 'tags']\n" ] } ], "source": [ "assert len(metas_raw) == len(metas_clean)\n", "assert metas_raw[0][\"id\"] == metas_clean[0][\"id\"]\n", "assert len(metas_clean) == len(set([m[\"id\"] for m in metas_clean]))\n", "assert (\n", " \"id\" in metas_clean[0] and\n", " \"duration_s\" in metas_clean[0] and\n", " \"s3_filepath\" in metas_clean[0]\n", ")\n", "print(list(sorted(set(funcy.flatten([list(m.keys()) for m in metas_clean[:10_000]])))))\n", "# 'duration_s', 'id', 's3_filepath', 'tags'" ] }, { "cell_type": "code", "execution_count": 78, "id": "6316d51f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/clean_pond5_v0_metas.jsonl to s3://suno-data/datasets/bundles/v4/pond5/metas_v0.jsonl\n" ] } ], "source": [ "write_jsonl(metas_clean, \"/app/suno/tmp/clean_pond5_v0_metas.jsonl\")\n", "# !aws s3 cp /app/suno/tmp/clean_pond5_v0_metas.jsonl s3://suno-data/datasets/bundles/v4/pond5/metas_v0.jsonl" ] }, { "cell_type": "markdown", "id": "2b2e29ca", "metadata": {}, "source": [ "### Imslp" ] }, { "cell_type": "code", "execution_count": 5, "id": "8f6f0f23", "metadata": {}, "outputs": [], "source": [ "metas_raw = read_jsonl(\"/app/suno/tmp/raw_imslp_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 6, "id": "622c46f3", "metadata": {}, "outputs": [], "source": [ "# use quality model to filter which to keep\n", "with open(\"/app/suno/tmp/imslp_low_quality_ids_christian.txt\") as f:\n", " low_q_ids = set([s.strip() for s in f.read().strip().split() if len(s.strip()) > 0])" ] }, { "cell_type": "code", "execution_count": 13, "id": "4fcfb4e5", "metadata": {}, "outputs": [], "source": [ "# print(list(sorted(set(funcy.flatten([list(m.keys()) for m in metas_raw[:10_000]])))))" ] }, { "cell_type": "code", "execution_count": 7, "id": "eef72e01", "metadata": {}, "outputs": [], "source": [ "tony_metas = read_jsonl(\"/home/tony/Data/classical/augmented_imslp_metas.jsonl\")\n", "assert len(metas_raw) == len(tony_metas)\n", "assert metas_raw[0][\"id\"] == tony_metas[0][\"id\"]\n", "# TODO: 'Austrian', 'German' should be removed cause composer\n", "# TODO: some instruments are wrong like harpsichord vs piano\n", "gpt_tag_map = {m[\"id\"]: m[\"tags_clean\"] for m in tony_metas if len(m.get(\"tags_clean\", [])) > 0}" ] }, { "cell_type": "code", "execution_count": 253, "id": "2b1b4bf5", "metadata": {}, "outputs": [], "source": [ "# random.choice(tony_metas)" ] }, { "cell_type": "code", "execution_count": 69, "id": "b671d8fb", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "274,275 total items\n", "247,527 commercial recordings\n", "76,812 entries skipped for quality\n", "12,723 retained\n" ] } ], "source": [ "# filter steps\n", "# filter to only recording_category == \"Commercial Recordings\"\n", "# for each composer max 10 original IDs (most popula ones)\n", "# within original IDs sample to max 5\n", "assert(len(set([m[\"id\"] for m in metas_raw])) == len(metas_raw))\n", "\n", "import random\n", "random.seed(6006)\n", "\n", "n_orig_max_per_composer = 100\n", "n_max_per_orig_id = 2\n", "\n", "print(f\"{len(metas_raw):,} total items\")\n", "filtered_metas = [m for m in metas_raw if m[\"recording_category\"] == \"Commercial Recordings\"]\n", "print(f\"{len(filtered_metas):,} commercial recordings\")\n", "from collections import defaultdict\n", "d = defaultdict(set)\n", "nnn = 0\n", "for m in metas_raw:\n", " if m[\"id\"] in low_q_ids:\n", " nnn += 1\n", " continue\n", " d[m[\"composer\"]].add(m[\"original_id\"])\n", "print(f\"{nnn:,} entries skipped for quality\")\n", "from collections import Counter\n", "d2 = Counter()\n", "for m in metas_raw:\n", " d2[m[\"original_id\"]] += 1\n", "orig_id_list = []\n", "for k, v in d.items():\n", " \n", " id_plus_counts = [(vv, d2[vv]) for vv in v]\n", " top_n_ids = sorted(id_plus_counts, key=lambda x: x[1], reverse=True)[:n_orig_max_per_composer]\n", " top_n_ids = [e[0] for e in top_n_ids]\n", " orig_id_list.extend(top_n_ids)\n", "orig_id_set = set(orig_id_list)\n", "d = defaultdict(list)\n", "for m in metas_raw:\n", " if m[\"original_id\"] not in orig_id_set:\n", " continue\n", " d[m[\"original_id\"]].append(m[\"id\"])\n", "final_retain_id_list = []\n", "for k, v in d.items():\n", " l = [_id for _id in d[k]]\n", " random.shuffle(l)\n", " final_retain_id_list.extend(l[:n_max_per_orig_id])\n", "final_retain_id_set = set(final_retain_id_list)\n", "print(f\"{len(final_retain_id_set):,} retained\")\n", "# 274,275 total items\n", "# 247,527 commercial recordings\n", "# 76,812 entries skipped for quality\n", "# 31,936 retained" ] }, { "cell_type": "code", "execution_count": 70, "id": "7ccda81d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████| 274275/274275 [00:01<00:00, 153844.75it/s]\n" ] } ], "source": [ "metas_clean = []\n", "for m in tqdm.tqdm(metas_raw):\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " \"s3_filepath\": m[\"s3_filepath\"],\n", " \"is_reliable\": m[\"id\"] in final_retain_id_set,\n", "# \"misc\": m,\n", " }\n", " if m[\"id\"] in gpt_tag_map:\n", " new_m[\"tags\"] = gpt_tag_map[m[\"id\"]]\n", " metas_clean.append(new_m)" ] }, { "cell_type": "code", "execution_count": 57, "id": "e6883075", "metadata": {}, "outputs": [], "source": [ "# TODO: make sure is_reliable is ignored in memmap" ] }, { "cell_type": "code", "execution_count": null, "id": "b30dd464", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 71, "id": "2ef9c70d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "12,723 / 274,275 reliable tracks\n", "12,723 with tags\n", "1696.4 hours\n" ] } ], "source": [ "print(f\"{len([m for m in metas_clean if m['is_reliable']]):,} / {len(metas_clean):,} reliable tracks\")\n", "print(f\"{len([m for m in metas_clean if m['is_reliable'] and 'tags' in m]):,} with tags\")\n", "print(f\"{sum([max(8*60, min(0, m['duration_s'])) for m in metas_clean if m['is_reliable']])/60/60:.1f} hours\")" ] }, { "cell_type": "code", "execution_count": 72, "id": "ec863e19", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['duration_s', 'id', 'is_reliable', 's3_filepath', 'tags']\n" ] } ], "source": [ "assert len(metas_raw) == len(metas_clean)\n", "# assert metas_raw[0][\"id\"] == metas_clean[0][\"id\"]\n", "assert len(metas_clean) == len(set([m[\"id\"] for m in metas_clean]))\n", "assert (\n", " \"id\" in metas_clean[0] and\n", " \"duration_s\" in metas_clean[0] and\n", " \"s3_filepath\" in metas_clean[0]\n", ")\n", "print(list(sorted(set(funcy.flatten([list(m.keys()) for m in metas_clean[:10_000]])))))\n", "# 'duration_s', 'id', 'is_reliable', 's3_filepath', 'tags'" ] }, { "cell_type": "code", "execution_count": 73, "id": "82d4e5d8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/clean_imslp_v0_metas.jsonl to s3://suno-data/datasets/bundles/v4/imslp/metas_v0.jsonl\n" ] } ], "source": [ "write_jsonl(metas_clean, \"/app/suno/tmp/clean_imslp_v0_metas.jsonl\")\n", "!aws s3 cp /app/suno/tmp/clean_imslp_v0_metas.jsonl s3://suno-data/datasets/bundles/v4/imslp/metas_v0.jsonl" ] }, { "cell_type": "markdown", "id": "e77c0630", "metadata": {}, "source": [ "### Karaoke stems" ] }, { "cell_type": "code", "execution_count": 160, "id": "29138118", "metadata": {}, "outputs": [], "source": [ "metas_raw = read_jsonl(\"/app/suno/tmp/raw_karaoke_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 162, "id": "4c0b3263", "metadata": {}, "outputs": [], "source": [ "# print(list(sorted(set(funcy.flatten([list(m.keys()) for m in metas_raw[:10_000]])))))" ] }, { "cell_type": "code", "execution_count": 163, "id": "77710c06", "metadata": {}, "outputs": [], "source": [ "# random.choice(metas_raw)" ] }, { "cell_type": "code", "execution_count": 161, "id": "2a490469", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████| 193494/193494 [00:00<00:00, 602565.56it/s]\n" ] } ], "source": [ "metas_clean = []\n", "for m in tqdm.tqdm(metas_raw):\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " \"s3_filepath\": m[\"s3_filepath\"],\n", " }\n", " if \"no_vocals\" in m[\"id\"]:\n", " new_m[\"bundle_id\"] = m[\"bundel_id\"]\n", " new_m[\"type\"] = \"instrumental\"\n", " elif \"lead_vocals\" in m[\"id\"]:\n", " new_m[\"bundle_id\"] = m[\"bundel_id\"]\n", " new_m[\"type\"] = \"vocals\"\n", " else:\n", " new_m[\"bundle_id\"] = m[\"bundel_id\"]\n", " new_m[\"type\"] = \"full\"\n", " metas_clean.append(new_m)" ] }, { "cell_type": "code", "execution_count": 163, "id": "4f845f8f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['bundle_id', 'duration_s', 'id', 's3_filepath', 'type']\n" ] } ], "source": [ "assert len(metas_raw) == len(metas_clean)\n", "assert metas_raw[0][\"id\"] == metas_clean[0][\"id\"]\n", "assert len(metas_clean) == len(set([m[\"id\"] for m in metas_clean]))\n", "assert (\n", " \"id\" in metas_clean[0] and\n", " \"duration_s\" in metas_clean[0] and\n", " \"s3_filepath\" in metas_clean[0]\n", ")\n", "print(list(sorted(set(funcy.flatten([list(m.keys()) for m in metas_clean[:10_000]])))))\n", "# 'duration_s', 'id', 's3_filepath', 'bundle_id', 'type'" ] }, { "cell_type": "code", "execution_count": 164, "id": "69d55ee4", "metadata": {}, "outputs": [], "source": [ "assert all([\"bundle_id\" in m for m in metas_clean])\n", "assert all([\"type\" in m for m in metas_clean])" ] }, { "cell_type": "code", "execution_count": 166, "id": "e7dee2d2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/clean_karaoke_v0_metas.jsonl to s3://suno-data/datasets/bundles/v4/karaoke_stems/metas_v0.jsonl\n" ] } ], "source": [ "write_jsonl(metas_clean, \"/app/suno/tmp/clean_karaoke_v0_metas.jsonl\")\n", "# !aws s3 cp /app/suno/tmp/clean_karaoke_v0_metas.jsonl s3://suno-data/datasets/bundles/v4/karaoke_stems/metas_v0.jsonl" ] }, { "cell_type": "markdown", "id": "7a7c2e6f", "metadata": {}, "source": [ "### Musdb stems" ] }, { "cell_type": "code", "execution_count": 153, "id": "475b4fb5", "metadata": {}, "outputs": [], "source": [ "metas_raw = read_jsonl(\"/app/suno/tmp/raw_musdb_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 181, "id": "ca2c9804", "metadata": {}, "outputs": [], "source": [ "# print(list(sorted(set(funcy.flatten([list(m.keys()) for m in metas_raw[:10_000]])))))" ] }, { "cell_type": "code", "execution_count": 88, "id": "8241fbe6", "metadata": {}, "outputs": [], "source": [ "# random.choice(metas_raw)" ] }, { "cell_type": "code", "execution_count": 154, "id": "df6970cc", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[{'id': 'am_contra_-_heart_peripheral__full',\n", " 'bundle_id': 'am_contra_-_heart_peripheral',\n", " 'duration_s': 210.11,\n", " 's3_filepath': 's3://suno-data/datasets/harvest/musdb/generated/am_contra_-_heart_peripheral__full.mp3'},\n", " {'id': 'am_contra_-_heart_peripheral__vocals',\n", " 'bundle_id': 'am_contra_-_heart_peripheral',\n", " 'duration_s': 210.11,\n", " 's3_filepath': 's3://suno-data/datasets/harvest/musdb/generated/am_contra_-_heart_peripheral__vocals.mp3'},\n", " {'id': 'am_contra_-_heart_peripheral__no_vocals',\n", " 'bundle_id': 'am_contra_-_heart_peripheral',\n", " 'duration_s': 210.11,\n", " 's3_filepath': 's3://suno-data/datasets/harvest/musdb/generated/am_contra_-_heart_peripheral__no_vocals.mp3'}]" ] }, "execution_count": 154, "metadata": {}, "output_type": "execute_result" } ], "source": [ "metas_raw[:3]" ] }, { "cell_type": "code", "execution_count": 155, "id": "5d82ec00", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████| 450/450 [00:00<00:00, 419337.21it/s]\n" ] } ], "source": [ "metas_clean = []\n", "for m in tqdm.tqdm(metas_raw):\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " \"s3_filepath\": m[\"s3_filepath\"],\n", " }\n", " if \"_no_vocals\" in m[\"id\"]:\n", " new_m[\"bundle_id\"] = m[\"bundle_id\"]\n", " new_m[\"type\"] = \"instrumental\"\n", " elif \"_vocals\" in m[\"id\"]:\n", " new_m[\"bundle_id\"] = m[\"bundle_id\"]\n", " new_m[\"type\"] = \"vocals\"\n", " else:\n", " assert \"full\" in m[\"id\"]\n", " new_m[\"bundle_id\"] = m[\"bundle_id\"]\n", " new_m[\"type\"] = \"full\"\n", " metas_clean.append(new_m)" ] }, { "cell_type": "code", "execution_count": 156, "id": "5820873a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['bundle_id', 'duration_s', 'id', 's3_filepath', 'type']\n" ] } ], "source": [ "assert len(metas_raw) == len(metas_clean)\n", "assert metas_raw[0][\"id\"] == metas_clean[0][\"id\"]\n", "assert len(metas_clean) == len(set([m[\"id\"] for m in metas_clean]))\n", "assert (\n", " \"id\" in metas_clean[0] and\n", " \"duration_s\" in metas_clean[0] and\n", " \"s3_filepath\" in metas_clean[0]\n", ")\n", "print(list(sorted(set(funcy.flatten([list(m.keys()) for m in metas_clean[:10_000]])))))\n", "# 'duration_s', 'id', 's3_filepath', 'bundle_id', 'type'" ] }, { "cell_type": "code", "execution_count": 158, "id": "bcf3ab89", "metadata": {}, "outputs": [], "source": [ "assert all([\"bundle_id\" in m for m in metas_clean])\n", "assert all([\"type\" in m for m in metas_clean])" ] }, { "cell_type": "code", "execution_count": 159, "id": "c4674c90", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Completed 103.3 KiB/103.3 KiB (446.1 KiB/s) with 1 file(s) remaining\r", "upload: ../../../app/suno/tmp/clean_musdb_v0_metas.jsonl to s3://suno-data/datasets/bundles/v4/musdb_stems/metas_v0.jsonl\r\n" ] } ], "source": [ "write_jsonl(metas_clean, \"/app/suno/tmp/clean_musdb_v0_metas.jsonl\")\n", "# !aws s3 cp /app/suno/tmp/clean_musdb_v0_metas.jsonl s3://suno-data/datasets/bundles/v4/musdb_stems/metas_v0.jsonl" ] }, { "cell_type": "markdown", "id": "4c137572", "metadata": {}, "source": [ "## Embed" ] }, { "cell_type": "code", "execution_count": null, "id": "a131de79", "metadata": {}, "outputs": [], "source": [ "# discogs\n", "# discogs_subset\n", "# covers\n", "# genius\n", "# youtube_music\n", "# deezer\n", "# pond5\n", "# imslp\n", "# karaoke_stems\n", "# musdb_stems" ] }, { "cell_type": "code", "execution_count": null, "id": "99c38128", "metadata": {}, "outputs": [], "source": [ "modal run /home/georg/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type='mert_25' \\\n", " --base-s3-dir='s3://suno-data/datasets/bundles/v4/discogs/' \\\n", " --metas-filename='metas_v0.jsonl' \\\n", " --chunksize=500 \\\n", " --min-duration-s=5 \\\n", " --max-duration-s=960 \\\n", " --output-name='mert_25_2x4k'\n", "\n", "\n", "modal run /home/georg/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type='mert_25' \\\n", " --base-s3-dir='s3://suno-data/datasets/bundles/v4/discogs_subset/' \\\n", " --metas-filename='metas_v0.jsonl' \\\n", " --chunksize=500 \\\n", " --min-duration-s=5 \\\n", " --max-duration-s=960 \\\n", " --output-name='mert_25_2x4k'\n", "\n", "modal run /home/georg/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type='mert_25' \\\n", " --base-s3-dir='s3://suno-data/datasets/bundles/v4/covers/' \\\n", " --metas-filename='metas_v0.jsonl' \\\n", " --chunksize=500 \\\n", " --min-duration-s=5 \\\n", " --max-duration-s=960 \\\n", " --output-name='mert_25_2x4k'\n", "\n", "modal run /home/georg/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type='mert_25' \\\n", " --base-s3-dir='s3://suno-data/datasets/bundles/v4/genius/' \\\n", " --metas-filename='metas_v0.jsonl' \\\n", " --chunksize=500 \\\n", " --min-duration-s=5 \\\n", " --max-duration-s=960 \\\n", " --output-name='mert_25_2x4k'\n", "\n", "modal run /home/georg/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type='mert_25' \\\n", " --base-s3-dir='s3://suno-data/datasets/bundles/v4/youtube_music/' \\\n", " --metas-filename='metas_v0.jsonl' \\\n", " --chunksize=500 \\\n", " --min-duration-s=5 \\\n", " --max-duration-s=960 \\\n", " --output-name='mert_25_2x4k'\n", "\n", "modal run /home/georg/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type='mert_25' \\\n", " --base-s3-dir='s3://suno-data/datasets/bundles/v4/deezer/' \\\n", " --metas-filename='metas_v0.jsonl' \\\n", " --chunksize=500 \\\n", " --min-duration-s=5 \\\n", " --max-duration-s=960 \\\n", " --output-name='mert_25_2x4k'\n", "\n", "modal run /home/georg/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type='mert_25' \\\n", " --base-s3-dir='s3://suno-data/datasets/bundles/v4/pond5/' \\\n", " --metas-filename='metas_v0.jsonl' \\\n", " --chunksize=500 \\\n", " --min-duration-s=5 \\\n", " --max-duration-s=960 \\\n", " --output-name='mert_25_2x4k'\n", "\n", "modal run /home/georg/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type='mert_25' \\\n", " --base-s3-dir='s3://suno-data/datasets/bundles/v4/imslp/' \\\n", " --metas-filename='metas_v0.jsonl' \\\n", " --chunksize=500 \\\n", " --min-duration-s=5 \\\n", " --max-duration-s=960 \\\n", " --output-name='mert_25_2x4k'\n", "\n", "modal run /home/georg/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type='mert_25' \\\n", " --base-s3-dir='s3://suno-data/datasets/bundles/v4/karaoke_stems/' \\\n", " --metas-filename='metas_v0.jsonl' \\\n", " --chunksize=500 \\\n", " --min-duration-s=5 \\\n", " --max-duration-s=960 \\\n", " --output-name='mert_25_2x4k'\n", "\n", "modal run /home/georg/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type='mert_25' \\\n", " --base-s3-dir='s3://suno-data/datasets/bundles/v4/musdb_stems/' \\\n", " --metas-filename='metas_v0.jsonl' \\\n", " --chunksize=50 \\\n", " --min-duration-s=5 \\\n", " --max-duration-s=960 \\\n", " --output-name='mert_25_2x4k'" ] }, { "cell_type": "code", "execution_count": null, "id": "882188c1", "metadata": {}, "outputs": [], "source": [ "# ditto\n", "modal run /home/georg/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type='ditto_v2_self_sim' \\\n", " --base-s3-dir='s3://suno-data/datasets/bundles/v4/covers' \\\n", " --metas-filename='metas_v0.jsonl' \\\n", " --chunksize=500 \\\n", " --min-duration-s=20 \\\n", " --max-duration-s=960 \\\n", " --first-only=True \\\n", " --force-overwrite=True \\\n", " --output-name='test_ditto_v2_self_sim'\n", "\n", "# also ditto_v2_vox_artist_sim etc" ] }, { "cell_type": "code", "execution_count": 137, "id": "6a0e44c8", "metadata": {}, "outputs": [], "source": [ "# !aws s3 ls s3://suno-data/datasets/bundles/v4/covers/\n", "# !aws s3 rm --recursive s3://suno-data/datasets/bundles/v4/covers/test__mert_25_2x4k" ] }, { "cell_type": "code", "execution_count": null, "id": "f38a55cf", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7d3398d7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "18e34c00", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "7d5f8e63", "metadata": {}, "source": [ "## TODO: marc data" ] }, { "cell_type": "code", "execution_count": null, "id": "0c6e5c18", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "20f13f96", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fb84479b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "91043f2a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "c2bca7e1", "metadata": {}, "source": [ "## Enrich covers with reliability" ] }, { "cell_type": "code", "execution_count": 49, "id": "5f97b406", "metadata": {}, "outputs": [], "source": [ "# TODO: do this earlier in future and properly split raw into multiple buckets" ] }, { "cell_type": "code", "execution_count": 2, "id": "024f800e", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import tqdm\n", "from suno_utils.utils.text import read_jsonl\n", "from suno_utils.utils.s3 import read_from_s3, check_s3_file_exists\n", "from sklearn.metrics.pairwise import cosine_similarity" ] }, { "cell_type": "code", "execution_count": 3, "id": "cacb5c6f", "metadata": {}, "outputs": [], "source": [ "metas_clean = read_jsonl(\"/app/suno/tmp/clean_covers_v0_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 46, "id": "e8deabd5", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████| 3739/3739 [28:41<00:00, 2.17it/s]\n" ] } ], "source": [ "def _load_idx(idx):\n", " metas_chunk = [m.copy() for m in metas_clean[500*idx:500*(idx+1)]]\n", " # check_s3_file_exists\n", " archive = read_from_s3(\n", " f\"s3://suno-data/datasets/bundles/v4/covers/ditto_v2_self_sim/part_{idx}.npz\", \n", " read_f=np.load\n", " )\n", " archive_ids = list(archive.keys())\n", " assert len(set(archive_ids) - set([m[\"id\"] for m in metas_chunk])) == 0\n", " return metas_chunk, {k: v.mean(0) for k, v in archive.items()}\n", "\n", "parent_emb = None\n", "children_ids = []\n", "children_sims = []\n", "for idx in tqdm.tqdm(range(3739)):\n", " metas_chunk, embeds = _load_idx(idx)\n", " children_ids = []\n", " for m in metas_chunk:\n", " if \"parent_id\" not in m:\n", " if parent_emb is not None and len(children_ids) > 0:\n", " out = cosine_similarity(\n", " parent_emb[None], \n", " np.stack([embeds[_id] for _id in children_ids])\n", " )\n", " for n, _id in enumerate(children_ids):\n", " children_sims.append((_id, out[0, n]))\n", " parent_emb = embeds.get(m[\"id\"])\n", " children_ids = []\n", " continue\n", " if m[\"id\"] in embeds:\n", " children_ids.append(m[\"id\"])\n", "if parent_emb is not None and len(children_ids) > 0:\n", " out = cosine_similarity(\n", " parent_emb[None], \n", " np.stack([embeds[_id] for _id in children_ids])\n", " )\n", " for n, _id in enumerate(children_ids):\n", " children_sims.append((_id, out[0, n]))" ] }, { "cell_type": "code", "execution_count": 51, "id": "6735c7ee", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "56% retained\n" ] } ], "source": [ "retain_thres = 0.5\n", "retained_child_ids = set([_id for _id, v in children_sims if v <= retain_thres])\n", "print(f\"{int(round(len(retained_child_ids)/len(children_sims)*100))}% retained\")" ] }, { "cell_type": "code", "execution_count": 52, "id": "23b84a16", "metadata": {}, "outputs": [], "source": [ "from collections import Counter\n", "c = Counter()\n", "for m in metas_clean:\n", " if \"parent_id\" in m and m[\"id\"] in retained_child_ids:\n", " c[m[\"parent_id\"]] += 1\n", "retained_parent_ids = set(c.keys())" ] }, { "cell_type": "code", "execution_count": 53, "id": "6c31c75e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "282778 parents\n", "810265 children\n", "58% retained\n" ] } ], "source": [ "for m in metas_clean:\n", " if \"parent_id\" not in m and m[\"id\"] in retained_parent_ids:\n", " m[\"is_reliable\"] = True\n", " elif \"parent_id\" in m and m[\"id\"] in retained_child_ids:\n", " m[\"is_reliable\"] = True\n", " else:\n", " m[\"is_reliable\"] = False\n", "print(f\"{len([m for m in metas_clean if m.get('is_reliable', False) and 'parent_id' not in m])} parents\")\n", "print(f\"{len([m for m in metas_clean if m.get('is_reliable', False) and 'parent_id' in m])} children\")\n", "print(f\"{int(round(len([m for m in metas_clean if m.get('is_reliable', False)])/len(metas_clean)*100))}% retained\")" ] }, { "cell_type": "code", "execution_count": null, "id": "57272493", "metadata": {}, "outputs": [], "source": [ "# 0.27 full band cover\n", "# 0.55: solo guitar cover\n", "# 0.65: drum cover\n", "# 0.88: drum cover\n", "# 0.92: same instrumental different voice\n", "# 0.99: same" ] }, { "cell_type": "code", "execution_count": 50, "id": "6004fb66", "metadata": {}, "outputs": [], "source": [ "# child_id = \"5GHARURBwzQ\"\n", "# parent_id = None\n", "# for m in metas_chunk:\n", "# if m[\"id\"] == child_id:\n", "# parent_id = m[\"parent_id\"]\n", "# print(\"child\", \"https://www.youtube.com/watch?v=\" + m[\"s3_filepath\"].split(\"/\")[-2])\n", "# for m in metas_chunk:\n", "# if m[\"id\"] == parent_id:\n", "# print(\"parent\", \"https://www.youtube.com/watch?v=\" + m[\"s3_filepath\"].split(\"/\")[-2])" ] }, { "cell_type": "code", "execution_count": 61, "id": "55fcfe0d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['artists', 'duration_s', 'id', 'is_reliable', 'lang', 'parent_id', 's3_filepath', 'tags', 'text']\n" ] } ], "source": [ "import funcy\n", "assert (\n", " \"id\" in metas_clean[0] and\n", " \"duration_s\" in metas_clean[0] and\n", " \"s3_filepath\" in metas_clean[0]\n", ")\n", "print(list(sorted(set(funcy.flatten([list(m.keys()) for m in metas_clean[:10_000]])))))\n", "# ['artists', 'duration_s', id', 'is_reliable', 'lang', 'parent_id', 's3_filepath', 'tags', 'text']" ] }, { "cell_type": "code", "execution_count": 59, "id": "1900515c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ../../../app/suno/tmp/clean_covers_v0_metas.jsonl to s3://suno-data/datasets/bundles/v4/covers/metas_v0.jsonl\n" ] } ], "source": [ "from suno_utils.utils.text import write_jsonl\n", "# write_jsonl(metas_clean, \"/app/suno/tmp/clean_covers_v0_metas.jsonl\")\n", "# !aws s3 cp /app/suno/tmp/clean_covers_v0_metas.jsonl s3://suno-data/datasets/bundles/v4/covers/metas_v0.jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "a7409200", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ac0f3841", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "6dcd6880", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 81, "id": "ea99bf8b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "download: s3://suno-data/datasets/bundles/v4/covers/test_ditto_v2_self_sim/part_0.npz to ./self_part_0__2.npz\n" ] } ], "source": [ "!aws s3 cp s3://suno-data/datasets/bundles/v4/covers/test_ditto_v2_self_sim/part_0.npz self_part_0__2.npz" ] }, { "cell_type": "code", "execution_count": 74, "id": "c92e91b1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "download: s3://suno-data/datasets/bundles/v4/covers/ditto_v2_vox_artist_sim/part_0.npz to ./vox_part_0.npz\n" ] } ], "source": [ "!aws s3 cp s3://suno-data/datasets/bundles/v4/covers/test__ditto_v2_vox_artist_sim/part_0.npz vox_part_0__2.npz" ] }, { "cell_type": "code", "execution_count": 82, "id": "7d013d76", "metadata": {}, "outputs": [], "source": [ "a = np.load(\"self_part_0.npz\")\n", "b = np.load(\"vox_part_0.npz\")\n", "c = np.load(\"self_part_0__2.npz\")\n", "# d = np.load(\"vox_part_0__2.npz\")" ] }, { "cell_type": "code", "execution_count": 83, "id": "97eb8046", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[-0.20912765, -0.02596085, -0.10709284, ..., -0.08713456,\n", " 0.00653603, -0.08844404],\n", " [-0.14677235, -0.03125421, -0.12794687, ..., -0.09302081,\n", " -0.03909803, -0.05449587],\n", " [-0.16406988, -0.05666261, -0.08587117, ..., -0.06097156,\n", " -0.00844358, -0.0443479 ],\n", " ...,\n", " [-0.16830273, -0.01607269, -0.08255263, ..., 0.01098447,\n", " -0.04461223, -0.01598153],\n", " [-0.18522681, -0.01400119, 0.00824055, ..., 0.10837224,\n", " 0.01906458, -0.03402235],\n", " [-0.16476169, -0.0322213 , -0.06696141, ..., 0.07362465,\n", " 0.06927309, -0.10738888]], dtype=float32)" ] }, "execution_count": 83, "metadata": {}, "output_type": "execute_result" } ], "source": [ "c[list(a.keys())[0]]" ] }, { "cell_type": "code", "execution_count": 78, "id": "f2e0c277", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[-0.20912765, -0.02596085, -0.10709284, ..., -0.08713456,\n", " 0.00653603, -0.08844404],\n", " [-0.14677235, -0.03125421, -0.12794687, ..., -0.09302081,\n", " -0.03909803, -0.05449587],\n", " [-0.16406988, -0.05666261, -0.08587117, ..., -0.06097156,\n", " -0.00844358, -0.0443479 ],\n", " ...,\n", " [-0.16830273, -0.01607269, -0.08255263, ..., 0.01098447,\n", " -0.04461223, -0.01598153],\n", " [-0.18522681, -0.01400119, 0.00824055, ..., 0.10837224,\n", " 0.01906458, -0.03402235],\n", " [-0.16476169, -0.0322213 , -0.06696141, ..., 0.07362465,\n", " 0.06927309, -0.10738888]], dtype=float32)" ] }, "execution_count": 78, "metadata": {}, "output_type": "execute_result" } ], "source": [ "a[list(a.keys())[0]]" ] }, { "cell_type": "code", "execution_count": 79, "id": "7cd4f328", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[-0.20912765, -0.02596085, -0.10709284, ..., -0.08713456,\n", " 0.00653603, -0.08844404],\n", " [-0.14677235, -0.03125421, -0.12794687, ..., -0.09302081,\n", " -0.03909803, -0.05449587],\n", " [-0.16406988, -0.05666261, -0.08587117, ..., -0.06097156,\n", " -0.00844358, -0.0443479 ],\n", " ...,\n", " [-0.16830273, -0.01607269, -0.08255263, ..., 0.01098447,\n", " -0.04461223, -0.01598153],\n", " [-0.18522681, -0.01400119, 0.00824055, ..., 0.10837224,\n", " 0.01906458, -0.03402235],\n", " [-0.16476169, -0.0322213 , -0.06696141, ..., 0.07362465,\n", " 0.06927309, -0.10738888]], dtype=float32)" ] }, "execution_count": 79, "metadata": {}, "output_type": "execute_result" } ], "source": [ "b[list(a.keys())[0]]" ] }, { "cell_type": "code", "execution_count": null, "id": "e2544e17", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a28a7ed7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "358d3a8b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5b54bf49", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 62, "id": "04c7d8cf", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████| 10/10 [00:05<00:00, 1.84it/s]\n" ] } ], "source": [ "def _load_idx(idx):\n", " metas_chunk = [m.copy() for m in metas_clean[500*idx:500*(idx+1)]]\n", " # check_s3_file_exists\n", " archive = read_from_s3(\n", " f\"s3://suno-data/datasets/bundles/v4/covers/ditto_v2_vox_artist_sim/part_{idx}.npz\", \n", " read_f=np.load\n", " )\n", " archive_ids = list(archive.keys())\n", " assert len(set(archive_ids) - set([m[\"id\"] for m in metas_chunk])) == 0\n", " return metas_chunk, {k: v.mean(0) for k, v in archive.items()}\n", "\n", "parent_emb = None\n", "children_ids = []\n", "children_sims2 = []\n", "for idx in tqdm.tqdm(range(10)):\n", " metas_chunk, embeds = _load_idx(idx)\n", " children_ids = []\n", " for m in metas_chunk:\n", " if \"parent_id\" not in m:\n", " if parent_emb is not None and len(children_ids) > 0:\n", " out = cosine_similarity(\n", " parent_emb[None], \n", " np.stack([embeds[_id] for _id in children_ids])\n", " )\n", " for n, _id in enumerate(children_ids):\n", " children_sims2.append((_id, out[0, n]))\n", " parent_emb = embeds.get(m[\"id\"])\n", " children_ids = []\n", " continue\n", " if m[\"id\"] in embeds:\n", " children_ids.append(m[\"id\"])\n", "if parent_emb is not None and len(children_ids) > 0:\n", " out = cosine_similarity(\n", " parent_emb[None], \n", " np.stack([embeds[_id] for _id in children_ids])\n", " )\n", " for n, _id in enumerate(children_ids):\n", " children_sims2.append((_id, out[0, n]))" ] }, { "cell_type": "code", "execution_count": 68, "id": "d0c2f8b1", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "df = pd.DataFrame(children_sims2, columns=[\"id\", \"vox_sim\"])\n", "df[\"self_sim\"] = df[\"id\"].map({k: v for k, v in children_sims})\n", "df[\"parent_id\"] = df[\"id\"].map({m[\"id\"]: m[\"parent_id\"] for m in metas_clean if \"parent_id\" in m})" ] }, { "cell_type": "code", "execution_count": null, "id": "97bfbed9", "metadata": {}, "outputs": [], "source": [ "# check high vox but low self" ] }, { "cell_type": "code", "execution_count": 72, "id": "2275fe67", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idvox_simself_simparent_id
07ziNxcI26oc0.3717250.371725qF8-suCt1lQ
1MI5sEF19FTg0.5206930.520693JdFm6rw0oqo
2NCXKaeeca-U0.2402260.240226JdFm6rw0oqo
3U0dJav63Igg0.1083690.108369JdFm6rw0oqo
4GoVn53JUZuc0.1583120.158312JdFm6rw0oqo
\n", "
" ], "text/plain": [ " id vox_sim self_sim parent_id\n", "0 7ziNxcI26oc 0.371725 0.371725 qF8-suCt1lQ\n", "1 MI5sEF19FTg 0.520693 0.520693 JdFm6rw0oqo\n", "2 NCXKaeeca-U 0.240226 0.240226 JdFm6rw0oqo\n", "3 U0dJav63Igg 0.108369 0.108369 JdFm6rw0oqo\n", "4 GoVn53JUZuc 0.158312 0.158312 JdFm6rw0oqo" ] }, "execution_count": 72, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head()" ] }, { "cell_type": "code", "execution_count": null, "id": "191dd357", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "56e2952f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "617473ab", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "edc66901", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "4c0919ba", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" } }, "nbformat": 4, "nbformat_minor": 5 }