{ "cells": [ { "cell_type": "code", "execution_count": 19, "id": "d32a6c40", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T02:50:52.151827Z", "start_time": "2023-11-08T02:50:52.149685Z" } }, "outputs": [], "source": [ "import sklearn\n", "import os\n", "import json\n", "import tqdm\n", "import re\n", "from suno_utils.utils.lyrics import remove_speakers\n", "from sklearn.feature_extraction.text import TfidfVectorizer\n", "from suno_utils.utils.text import (\n", " write_jsonl,\n", " read_jsonl,\n", " write_json,\n", " read_json,\n", " normalize_whitespace,\n", ")\n", "import numpy as np" ] }, { "cell_type": "code", "execution_count": 2, "id": "4c1f7c58", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T02:36:10.838764Z", "start_time": "2023-11-08T02:35:50.021328Z" } }, "outputs": [], "source": [ "## only need to save this json once\n", "with open(\"/home/tony/Data/Hoot/metas.json\", \"r\") as fp:\n", " genius_metas = json.load(fp)" ] }, { "cell_type": "code", "execution_count": 3, "id": "233c5370", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T02:36:10.844447Z", "start_time": "2023-11-08T02:36:10.840866Z" } }, "outputs": [ { "data": { "text/plain": [ "1836168" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(genius_metas)" ] }, { "cell_type": "code", "execution_count": 4, "id": "ba57259f", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T02:36:10.895732Z", "start_time": "2023-11-08T02:36:10.845534Z" } }, "outputs": [], "source": [ "def clean_text(text):\n", " \"\"\"General text cleaning.\"\"\"\n", " text = \"\\n\" + text\n", " text = text.replace(\"’\", \"'\").lower()\n", " text = text.replace('\"', \"\").lower()\n", " text = re.sub(r\"\\[.+?\\]\", \" \", text) # tags\n", " text = re.sub(r\"\\n.+?\\:\", \" \", text) # new line ends with :\n", " text = re.sub(r\"\\n.+?\\：\", \" \", text) # new line ends with :\n", " text = re.sub(r\"\\n\\(.+?\\)\", \" \", text) # new line with ()\n", " text = re.sub(r\"[\\d]\", \" \", text) # digits\n", " text = re.sub(r\"▁\", \"\", text) # special stuff\n", " text = remove_speakers(text)\n", " text = re.sub(r\"[^\\w\\'\\s]\", \" \", text) # keep only the words\n", " text = normalize_whitespace(text)\n", " return text" ] }, { "cell_type": "code", "execution_count": 5, "id": "aa542c31", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T02:41:29.394152Z", "start_time": "2023-11-08T02:36:10.897350Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████| 1836168/1836168 [05:18<00:00, 5766.01it/s]\n" ] } ], "source": [ "corpus = {} #original_id, cleand_lyrics\n", "for meta in tqdm.tqdm(genius_metas):\n", " cleaned_text = clean_text(meta[\"lyrics\"])\n", " if len(cleaned_text) > 20 and len(cleaned_text) < 10000:\n", " corpus[meta[\"original_id\"]] = cleaned_text" ] }, { "cell_type": "code", "execution_count": 6, "id": "663e7363", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T02:41:29.397532Z", "start_time": "2023-11-08T02:41:29.395285Z" } }, "outputs": [ { "data": { "text/plain": [ "1835566" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(corpus)" ] }, { "cell_type": "code", "execution_count": 9, "id": "df98a1d9", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T02:43:17.391289Z", "start_time": "2023-11-08T02:43:17.389358Z" } }, "outputs": [], "source": [ "vectorizer = TfidfVectorizer(max_df=0.01, max_features=100_000)" ] }, { "cell_type": "code", "execution_count": 10, "id": "41835a24", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T02:43:19.415535Z", "start_time": "2023-11-08T02:43:19.414057Z" } }, "outputs": [], "source": [ "documents = corpus.values()" ] }, { "cell_type": "code", "execution_count": 11, "id": "0ab7f5a2", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T02:47:51.730262Z", "start_time": "2023-11-08T02:43:25.481833Z" } }, "outputs": [], "source": [ "X = vectorizer.fit_transform(documents)" ] }, { "cell_type": "code", "execution_count": 12, "id": "99157362", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T02:48:18.775497Z", "start_time": "2023-11-08T02:48:18.773922Z" } }, "outputs": [], "source": [ "vocabs = vectorizer.vocabulary_\n", "lookup_ids = list(corpus.keys())" ] }, { "cell_type": "code", "execution_count": 54, "id": "962d0177", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T03:19:58.792144Z", "start_time": "2023-11-08T03:19:58.790589Z" } }, "outputs": [], "source": [ "test_input_text = \"\"\"\n", "Nants ingonyama bagithi baba\n", "Sithi uhm ingonyama\n", "Nants ingonyama bagithi baba\n", "Sithi uhhmm ingonyama\n", "\"\"\"\n", "# test_input_text = \"\"\"\n", "# I love you, Tony\n", "# \"\"\"" ] }, { "cell_type": "code", "execution_count": 55, "id": "29efb13e", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T03:19:59.550967Z", "start_time": "2023-11-08T03:19:59.549471Z" } }, "outputs": [], "source": [ "test_input_text = clean_text(test_input_text)" ] }, { "cell_type": "code", "execution_count": 56, "id": "d24e9994", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T03:19:59.736300Z", "start_time": "2023-11-08T03:19:59.733694Z" } }, "outputs": [], "source": [ "y = vectorizer.transform([test_input_text])" ] }, { "cell_type": "code", "execution_count": 57, "id": "199025e9", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T03:19:59.907323Z", "start_time": "2023-11-08T03:19:59.905373Z" } }, "outputs": [ { "data": { "text/plain": [ "(1835566, 100000)" ] }, "execution_count": 57, "metadata": {}, "output_type": "execute_result" } ], "source": [ "X.shape" ] }, { "cell_type": "code", "execution_count": 58, "id": "4fc8ab2a", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T03:20:00.067795Z", "start_time": "2023-11-08T03:20:00.065935Z" } }, "outputs": [ { "data": { "text/plain": [ "(1, 100000)" ] }, "execution_count": 58, "metadata": {}, "output_type": "execute_result" } ], "source": [ "y.shape" ] }, { "cell_type": "code", "execution_count": 59, "id": "7f602ef3", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T03:20:00.435302Z", "start_time": "2023-11-08T03:20:00.229380Z" } }, "outputs": [], "source": [ "y_pred = np.dot(X, y.T)" ] }, { "cell_type": "code", "execution_count": 60, "id": "2b2ebff2", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T03:20:00.440955Z", "start_time": "2023-11-08T03:20:00.437086Z" } }, "outputs": [ { "data": { "text/plain": [ "0.9149019590687457" ] }, "execution_count": 60, "metadata": {}, "output_type": "execute_result" } ], "source": [ "np.max(y_pred)" ] }, { "cell_type": "code", "execution_count": 61, "id": "d0735f88", "metadata": { "ExecuteTime": { "end_time": "2023-11-08T03:20:01.829788Z", "start_time": "2023-11-08T03:20:01.826210Z" } }, "outputs": [ { "data": { "text/plain": [ "\"nants ingonyama bagithi baba sithi uhm ingonyama 'ngonyama 'ngengw'ebo mai ba bo ha ingonyama baba nants ingonyama bagithi baba sithi uhm ingonyama 'ngonyama 'ngengw'ebo oh khusani bo pegi akalela eshe nants ingonyama bagithi baba sithi uhm ingonyama ingonyama siyo nqoba ingonyama ingonyama nengwe 'namabala ingonyama nengwe 'namabala ingonyama nengwe 'namabala ingonyama nengwe 'namabala ingonyama nengwe 'namabala ta na na na na ta na na na na na ta na na na na ta na na na na ta na na na na ta na na na na na from the day we arrive on the planet and blinking step into the sun there's more to see than can ever be seen more to do than can ever be done there is far too much to take in here more to find than can ever be found but the sun rolling high through the sapphire sky keeps great and small on the endless round it's the circle of life and it moves us all through despair and hope through faith and love 'til we find our place on the path unwinding in the circle the circle of life bayede ndebe sutha khosi m'khosi siyabonga baba bayede helele helele helele ukele we sizwe khosi wen khofu siyabonga baba bayede siyabonga baba khwathu ma fikele khwene ndaba khwathu ma fikele wen o wen khofu siyabonga baba bayede it's the circle of life and it moves us all through despair and hope through faith and love 'til we find our place on the path unwinding in the circle the circle of life\"" ] }, "execution_count": 61, "metadata": {}, "output_type": "execute_result" } ], "source": [ "corpus[lookup_ids[np.argmax(y_pred)]]" ] }, { "cell_type": "code", "execution_count": null, "id": "8f7e72d4", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env", "language": "python", "name": "suno_env" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }