{ "cells": [ { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "# this notebook is for converting the old uuid's from genius to the new youtube ids\n", "\n", "\n", "import os\n", "import glob\n", "from tqdm import tqdm\n", "from suno_utils.utils.text import read_jsonl, write_jsonl" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "loading genius alignments\n" ] } ], "source": [ "# load the alignments\n", "\n", "genius_alignments_filepath = (\n", " \"/home/tony/Work/tony/hoot/tmp/genius_hq_alignments_t30_v1.jsonl\"\n", ")\n", "\n", "print(\"loading genius alignments\")\n", "genius_alignments = read_jsonl(genius_alignments_filepath, progress=False)\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "# load the old metas\n", "genius_old_metas_filepath = \"/home/christian/code/christian/metadata/genius_hq_metas.jsonl\"\n", "genius_old_metas = read_jsonl(genius_old_metas_filepath, progress=False)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "genius_old_metas_map = {m[\"id\"]: m for m in genius_old_metas}" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 1747804/1747804 [00:02<00:00, 782833.05it/s]\n" ] } ], "source": [ "new_alignments = []\n", "\n", "for orig_id, alignemnts in tqdm(genius_alignments):\n", " # find the new id in the old metas\n", " new_id = genius_old_metas_map[orig_id][\"original_id\"]\n", " new_alignments.append((new_id, alignemnts))\n", "\n", "write_jsonl(new_alignments, \"/home/christian/code/christian/metadata/genius_hq_alignments_t30_v1_yt_ids.jsonl\")\n", "\n" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2916657\n" ] } ], "source": [ "METAS_DIR = \"/app/suno/tmp\"\n", "discogs_subset = read_jsonl(os.path.join(METAS_DIR, \"clean_discogs_subset_v0_metas.jsonl\"))\n", "\n", "print(len(discogs_subset))" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2916657\n", "2199518\n" ] } ], "source": [ "# look for the most common lyrics \n", "# first create a list of all the lyrics\n", "lyrics_list = [m.get(\"text\", \"\") for m in discogs_subset]\n", "print(len(lyrics_list))\n", "# kick out the empty ones\n", "lyrics_list = [l for l in lyrics_list if l]\n", "print(len(lyrics_list))\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 58, "metadata": {}, "outputs": [], "source": [ "import re\n", "from collections import defaultdict\n", "from tqdm import tqdm\n", "\n", "def clean_text(text):\n", " \"\"\"Clean text by removing punctuation and extra spaces.\"\"\"\n", " text = re.sub(r'[^\\w\\s]', '', text.lower())\n", " return ' '.join(text.split())\n", "\n", "def get_text_hash(text):\n", " \"\"\"Create a simple hash from text by taking first and last words plus length.\"\"\"\n", " words = clean_text(text).split()\n", " if len(words) < 2:\n", " return words[0] if words else ''\n", " return f\"{words[0]}_{words[-1]}_{len(words)}\"\n", "\n", "def find_similar_lyrics(lyrics_list):\n", " \"\"\"Find similar lyrics using a simple hashing approach.\"\"\"\n", " print(f\"Processing {len(lyrics_list)} lyrics...\")\n", " \n", " # Group lyrics by hash\n", " hash_groups = defaultdict(list)\n", " \n", " # Process each lyric with progress bar\n", " for idx, lyric in tqdm(enumerate(lyrics_list), total=len(lyrics_list)):\n", " text_hash = get_text_hash(lyric)\n", " hash_groups[text_hash].append((idx, lyric))\n", " \n", " # Filter to only groups with multiple entries\n", " similar_groups = [group for group in hash_groups.values() if len(group) > 1]\n", " \n", " print(f\"\\nFound {len(similar_groups)} groups of similar lyrics\")\n", " return similar_groups" ] }, { "cell_type": "code", "execution_count": 59, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Processing 2199518 lyrics...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 2199518/2199518 [02:07<00:00, 17292.22it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Found 209401 groups of similar lyrics\n" ] } ], "source": [ "similar_groups = find_similar_lyrics(lyrics_list)" ] }, { "cell_type": "code", "execution_count": 60, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "209401\n" ] } ], "source": [ "print(len(similar_groups))\n", "# sort the groups by the number of matches\n", "similar_groups = sorted(similar_groups, key=lambda x: len(x), reverse=True)\n" ] }, { "cell_type": "code", "execution_count": 61, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Group 1: has 178 lyric matches\n", "Index 7706: Mason\n", "\n", "Picture this\n", "Little Fred out there really hittin' licks\n", "By 15, he's locked in jail but no one didn't visit him\n", "It's cool though\n", "'Cause most of his friends were still in school though\n", "But he thought a knife and bike was really cooler than some school clothes\n", "\n", "Chasin' money young\n", "Yeah, my teachers and my school knows\n", "I was livin' with my OG's\n", "Used to tell their mum, \"My school's closed\"\n", "Picture this\n", "Niggas slippin' on their blocks for Insta pics\n", "We pull up with the smoke\n", "Let it blow and now your picture's ripped\n", "\n", "Me and mine was broke with hopes of coke fixing it\n", "I was broke, 'til I got coke and put mix in it\n", "She knows I'm risky risking it, got my wrist in, whisking it\n", "I'm either mashin' works on little pricks or I'm assisting it\n", "Slice a nigga's chest and watch it ooze like there's a cyst in it\n", "I get 300 thousand an album and a Insta pic\n", "Been seeing man grinding for some years just for the pinky drip\n", "Niggas had their eye on it, he lost it in the blink of it\n", "\n", "Yo, I don't even think, when I come to think of it\n", "Forget Titanic, I got enough ice on to sink the ship\n", "Bitches wanna fuck me quick, they don't even need to drink a bit\n", "Rule number one, don't drive no cars without no tints on it\n", "I know why they hate and lie, it's clear to see they're mad at me\n", "I got so many hoes, I need to donate some to charity\n", "Fredo started rapping with a bust down and a real chain\n", "Them man didn't even have no money 'til the deal came\n", "\n", "You dropped 20 on your wrist, you need a better freezer\n", "That's the kinda money me and mine go bet on Fifa\n", "Bruddas acting like it's beef but they look sorta dumb\n", "They don't pull up 'round here, I got more beef with my daughter's mum\n", "My Rolex and gelato, the same number, they're both 41\n", "I been in the hood all day today, my niggas call me dumb\n", "But if I stopped goin', then I'm knowing the love will all be done\n", "I don't wanna change or stay the same, it is an awkward one\n", "\n", "Picture that, now all I got's these pics, I want my niggas back\n", "Girls say they love me, it isn't that\n", "I steal their heart then give it back\n", "I'm responsible for everything that's popping here\n", "Niggas ask for shit, don't wanna hear\n", "What would you do if I wasn't here?\n", "\n", "Rest in peace my nigga Muscle, you made me drop a tear\n", "Two rambos up in Harvey Nichols, no, we don't shop in fear\n", "And I don't blame them lame bruddas for watchin' what we do\n", "'Cause if I was you lame bruddas, I'd be copying me too\n", "\n", "Why would I ever marry a hoe?\n", "So she can get bored, take half of my pattern and go? No\n", "Crazy bro, I was in jail, hearing me on the radio\n", "Didn't know if I would make it home\n", "But life is what you make it though\n", "\n", "Even though I'm seein' this crazy money\n", "I know they want the crazy freestyle, they been waiting on me\n", "My youngens sliding every night, yeah, they try daily duppy\n", "Hundred 'round my neck and I won't let no one take it from me\n", "\n", "Straight in like that, yeah?\n", "Ah, let's go again\n", "\n", "Made mistakes in my life and it needs a rewind\n", "But then again, when I check these the reasons I'm out here\n", "When I was broke, they couldn't see me, them people was blind\n", "Now I'm up, everyone seems to be seein' me fine\n", "\n", "Bad on my own, they only move in a herd\n", "Last time I checked, pussy boy, you was a nerd\n", "Whistle on this new stick, is how we move on the curb\n", "Lil' prick, we put the stick here on do not disturb\n", "\n", "Men lie, women lie but them numbers are real\n", "So the opps must be lying 'cause their numbers are nil\n", "You can't name one man that you done in the field\n", "But you got hundreds of tracks, spittin' nothin' but drill\n", "\n", "Any time I'm spittin', they know it's nothin' but real\n", "I'm so lit, girls are always tryna come in some heels\n", "But you look silly dressed up when you're just coming to chill\n", "I'm tryna fuck you on the sofa then run up a mill\n", "\n", "Little Troy got a twenty, shit is fuckin' for real\n", "Anytime his Mum ask, I gotta cover them bills\n", "Got skills in the pot, whipped a 100ml's\n", "Niggas start to think my left wrist must've come from Brazil\n", "\n", "I done brought a couple man in to the party\n", "But they ain't here to party\n", "You play? Then they're coming to drill\n", "If niggas think that I ain't gang no more and I'm a rapper\n", "Come test me and you'll find out that I'm one of them still\n", "\n", "Two sticks in the whip, no bamboo\n", "Even Usain Bolt would lose track of all them hoes I ran through\n", "You niggas doing loads, there's not much that I'd do for hoes\n", "Most I bought a bitch was Hakkasan and a Uber home\n", "\n", "I'm doin' festivals, them man are doing tuna rolls\n", "We're the teachers of the drip, gotta tutor loads\n", "Lost my dargie and I know that he ain't coming back\n", "If you lose your jewels by my dawgs, we don't run it back\n", "\n", "I swear it's hard to be a boss\n", "I just lost Muscle, I feel like part of me is lost\n", "If I could, I swear I'd probably have an argument with God\n", "I know you got your reasons, I'm in my feelings, I don't understand\n", "Dotz died before he even was a fucking man\n", "My step dad caught an M, there ain't no coming back\n", "But before this, I already was my brother's dad\n", "That's how it goes when your people wanna fuck with crack\n", "\n", "Then the feds wanna send my niggas to pen'\n", "When deep down, we all know the real villains is them\n", "Why start taking crack? It just poisons estates\n", "I thought that people gotta live with all the choices they made\n", "\n", "We don't force them to call, they been callin' for days\n", "And you can lock me up, cool, they'll just be calling my mates\n", "It's safe to say, when shotters getting more than a rapist case\n", "How can I abide by the law or what the labour says?\n", "\n", "I feel them guys in suits got all these laws tailor made\n", "'Cause they might touch a kid but they won't never touch a razor blade\n", "Where I'm from, there's not one, we had to make a way\n", "Started with some fast food, the T is like a takeaway\n", "\n", "Straight from the block\n", "We could go away for years\n", "And no one here can't be taking the spot\n", "Came from poverty so I bought a bust down watch\n", "And a box before I ever thought about buying any property\n", "\n", "I gotta give it honestly, that's why my niggas honour me\n", "Feds might as well make a Instagram, how they follow me\n", "My young niggas stab you up, you're dead, probably\n", "'Cause they're walking 'round with machetes that's made for doner meat\n", "\n", "Yo, they're actin', the love is just cheap\n", "And that's why that shit there don't mean nothing to me\n", "I'm getting six figures now, just to go jump on a beat\n", "So this my last freestyle 'cause I do nothin' for free\n", "\n", "Group 2: has 111 lyric matches\n", "Index 1116: Instrumental\n", "\n", "Group 3: has 83 lyric matches\n", "Index 664: Chestnuts roasting on an open fire\n", "Jack Frost nipping at your nose\n", "Yule-tide carols being sung by a choir\n", "And folks dressed up like Eskimos.\n", "\n", "Everybody knows a turkey and some mistletoe\n", "Help to make the season bright\n", "Tiny tots with their eyes all aglow\n", "Will find it hard to sleep tonight.\n", "\n", "They know that Santa's on his way\n", "He's loaded lots of toys and goodies on his sleigh\n", "And every mother's child is gonna spy\n", "To see if reindeer really know how to fly.\n", "\n", "And so I'm offering this simple phrase\n", "To kids from one to ninety-two\n", "Although it's been said many times, many ways\n", "Merry Christmas to you!\n", "\n", "Group 4: has 78 lyric matches\n", "Index 21817: The falling leaves drift by the window\n", "The autumn leaves of red and gold\n", "I see your lips, the summer kisses\n", "The sun-burned hands I used to hold\n", "\n", "Since you went away the days grow long\n", "And soon I'll hear old winter's song\n", "But I miss you most of all my darling\n", "When autumn leaves start to fall\n", "\n", "C'est une chanson, qui nous ressemble\n", "Toi tu m'aimais et je t'aimais\n", "Nous vivions tous deux ensemble\n", "Toi qui m'aimais moi qui t'aimais\n", "Mais la vie separe ceux qui s'aiment\n", "Tout doucement sans faire de bruit\n", "Et la mer efface sur le sable les pas des amants desunis\n", "\n", "Group 5: has 76 lyric matches\n", "Index 5722: Come, they told me pa-rum pum pum pum\n", "Our newborn King to see, pa-rum pum pum pum\n", "Our finest gifts we bring pa-rum pum pum pum\n", "To lay before the King pa-rum pum pum pum\n", "Rum pum pum pum. rum pum pum pum\n", "So to honor Him pa-rum pum pum pum\n", "When we come\n", "\n", "Little Baby pa-rum pum pum pum\n", "I am a poor boy too, pa-rum pum pum pum\n", "I have no gift to bring pa-rum pum pum pum\n", "That's fit to give our King pa- rum pum pum pum\n", "Rum pum pum pum, rum pum pum pum\n", "Shall I play for you, pa-rum pum pum pum\n", "on my drum?\n", "\n", "Mary nodded pa-rum pum pum pum\n", "The Ox and Lamb kept time pa-rum pum pum pum\n", "I played my drum for Him pa-rum pum pum pum\n", "I played my best for Him pa -rum pum pum pum\n", "Rum pum pum pum, rum pum pum pum\n", "Then He smiled at me pa-rum pum pum pum\n", "Me and my drum\n" ] } ], "source": [ "for i, group in enumerate(similar_groups[:5]):\n", " print(f\"\\nGroup {i+1}: has {len(group)} lyric matches\")\n", " for idx, lyric in group:\n", " print(f\"Index {idx}: {lyric}\")\n", " break" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.9" } }, "nbformat": 4, "nbformat_minor": 2 }