{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "2e2253c7", "metadata": {}, "outputs": [], "source": [ "import os" ] }, { "cell_type": "code", "execution_count": 2, "id": "c5e1499e", "metadata": {}, "outputs": [], "source": [ "os.environ[\"OPENAI_API_KEY\"] = \"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\"" ] }, { "cell_type": "code", "execution_count": 3, "id": "dabe64d3-8bc5-444c-8f94-b54167ec015f", "metadata": { "execution": { "iopub.execute_input": "2024-10-22T03:21:36.087854Z", "iopub.status.busy": "2024-10-22T03:21:36.087174Z", "iopub.status.idle": "2024-10-22T03:21:36.677530Z", "shell.execute_reply": "2024-10-22T03:21:36.677128Z", "shell.execute_reply.started": "2024-10-22T03:21:36.087823Z" } }, "outputs": [], "source": [ "from openai import OpenAI\n", "\n", "client = OpenAI()" ] }, { "cell_type": "code", "execution_count": 4, "id": "ac46fe09", "metadata": {}, "outputs": [], "source": [ "input_artists = \"\"\"\n", "Aitana\n", "Aitch\n", "Ajay Chandisar\n", "Ajay Gogavale\n", "Ajay Mali\n", "Ajay Thakor\n", "Ajay-Atul\n", "Ajda Pekkan\n", "Ajeet\n", "Ajeet Anand\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": 5, "id": "dfb11d51", "metadata": {}, "outputs": [], "source": [ "def get_response(input_artists):\n", " response = client.chat.completions.create(\n", " model=\"gpt-4o\",\n", " messages=[\n", " {\n", " \"role\": \"user\",\n", " \"content\": [\n", " {\n", " \"type\": \"text\",\n", " \"text\": f\"\"\"\n", " You are a music expert.\n", " Given a list of input artists, please annotate each artist's music style.\n", " Try to be as accurate as possible, but also comprehensive as possible.\n", " Only annotate the artists that are in the input list.\n", " Keep the answer to be max 7 different styles per artist.\n", " Cross reference with your knowledge of music to make sure the answer is correct.\n", " Remove all the non-confident answers please.\n", " \n", " Format the output as a JSON object where the keys are the artist names and the values are the styles.\n", " The values should be a comma-separated list of styles.\n", "\n", " \n", " Here is the list of artists (separated by new lines):\n", " \"{input_artists}\"\n", " \"\"\",\n", " }\n", " ],\n", " },\n", " ],\n", " temperature=1,\n", " max_tokens=4096,\n", " top_p=1,\n", " frequency_penalty=0,\n", " presence_penalty=0,\n", " response_format={\"type\": \"text\"},\n", " )\n", " return response" ] }, { "cell_type": "code", "execution_count": 6, "id": "193b48ec-1ec0-454c-ad1f-a1746a949eff", "metadata": {}, "outputs": [], "source": [ "response = client.chat.completions.create(\n", " model=\"gpt-4o\",\n", " messages=[\n", " {\n", " \"role\": \"user\",\n", " \"content\": [\n", " {\n", " \"type\": \"text\",\n", " \"text\": f\"\"\"\n", " You are a music expert.\n", " Given a list of input artists, please annotate each artist's music style.\n", " Try to be as accurate as possible, but also comprehensive as possible.\n", " Only annotate the artists that are in the input list.\n", " Keep the answer to be max 7 different styles per artist.\n", " Cross reference with your knowledge of music to make sure the answer is correct.\n", " Remove all the non-confident answers please.\n", " \n", " Format the output as a JSON object where the keys are the artist names and the values are the styles.\n", " The values should be a comma-separated list of styles.\n", "\n", " \n", " Here is the list of artists (separated by new lines):\n", " \"{input_artists}\"\n", " \"\"\",\n", " }\n", " ],\n", " },\n", " ],\n", " temperature=1,\n", " max_tokens=4096,\n", " top_p=1,\n", " frequency_penalty=0,\n", " presence_penalty=0,\n", " response_format={\"type\": \"text\"},\n", ")" ] }, { "cell_type": "code", "execution_count": 8, "id": "6543063f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "```json\n", "{\n", " \"Aitana\": \"Latin pop, pop, reggaeton, dance-pop\",\n", " \"Aitch\": \"British hip hop, trap, grime, rap\",\n", " \"Ajay-Atul\": \"Indian film music, Bollywood, Marathi music, classical, folk\",\n", " \"Ajda Pekkan\": \"Turkish pop, pop, classic Turkish music\"\n", "}\n", "```\n" ] } ], "source": [ "print(response.choices[0].message.content)" ] }, { "cell_type": "code", "execution_count": 78, "id": "4bc5e0e4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "21238\n" ] } ], "source": [ "with open(\n", " \"/Users/tonytong/Work/Git/glockenspiel/suno_utils/suno_utils/worker/assets/top_artists.txt.py\"\n", ") as f:\n", " TOP_ARTISTS = [line.strip() for line in f.readlines()]\n", "\n", "print(len(TOP_ARTISTS))" ] }, { "cell_type": "code", "execution_count": 52, "id": "52ae54bf", "metadata": {}, "outputs": [], "source": [ "import json\n", "from collections import defaultdict\n", "\n", "total_ans = defaultdict(str)" ] }, { "cell_type": "code", "execution_count": 53, "id": "8e58c328", "metadata": {}, "outputs": [], "source": [ "# import json\n", "\n", "# # Remove 'json\\n' if present, and any leading/trailing backticks and newlines\n", "# cleaned_ans = response.choices[0].message.content.replace(\"json\\n\", \"\").strip(\"`\\n\")\n", "\n", "# # Parse the cleaned string into a dictionary\n", "# curr_ans = json.loads(cleaned_ans)" ] }, { "cell_type": "markdown", "id": "4a4d31ef", "metadata": {}, "source": [] }, { "cell_type": "code", "execution_count": 55, "id": "c786577a", "metadata": {}, "outputs": [], "source": [ "import tqdm" ] }, { "cell_type": "code", "execution_count": 59, "id": "b2e67da6", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 21238/21238 [00:00<00:00, 3214094.47it/s]\n" ] } ], "source": [ "total_ans = defaultdict(str)\n", "for artist in tqdm.tqdm(TOP_ARTISTS):\n", " total_ans[artist] = \"\"" ] }, { "cell_type": "code", "execution_count": 9, "id": "0a13dc55", "metadata": {}, "outputs": [], "source": [ "# # for i in tqdm.tqdm(range(0, len(TOP_ARTISTS), 20)):\n", "# for i in [2160, 3280, 3900, 4080, 5280, 8980, 10920, 15460, 18280, 19260]:\n", "# artist_chunk = \"\\n\".join(TOP_ARTISTS[i : i + 20])\n", "# response = get_response(artist_chunk)\n", "# try:\n", "# curr_ans = (\n", "# response.choices[0].message.content.replace(\"json\\n\", \"\").strip(\"`\\n\")\n", "# )\n", "# curr_ans = json.loads(curr_ans)\n", "# if isinstance(curr_ans, dict):\n", "# for artist, generes in curr_ans.items():\n", "# total_ans[artist] = generes\n", "# except Exception as e:\n", "# print(f\"Error on chunk {i}, {e}\")\n", "# continue" ] }, { "cell_type": "code", "execution_count": 10, "id": "e147bc77", "metadata": {}, "outputs": [], "source": [ "# mistaken_artists = []\n", "# unknown_artists = set(TOP_ARTISTS)\n", "# for k in total_ans:\n", "# if k not in unknown_artists:\n", "# mistaken_artists.append(k)\n", "# print(len(mistaken_artists), mistaken_artists)\n", "# for k in mistaken_artists:\n", "# total_ans.pop(k)" ] }, { "cell_type": "code", "execution_count": 66, "id": "685987bc", "metadata": {}, "outputs": [], "source": [ "final_ans = {}\n", "for artist in TOP_ARTISTS:\n", " if artist in total_ans:\n", " final_ans[artist] = total_ans[artist]\n", " else:\n", " final_ans[artist] = \"\"" ] }, { "cell_type": "code", "execution_count": 69, "id": "b30cec7c", "metadata": {}, "outputs": [], "source": [ "with open(\"final_ans.json\", \"w\") as f:\n", " json.dump(final_ans, f, indent=4, ensure_ascii=False)" ] }, { "cell_type": "code", "execution_count": 72, "id": "7f694ccb", "metadata": {}, "outputs": [], "source": [ "def get_response_second_pass(input_artists):\n", " response = client.chat.completions.create(\n", " model=\"gpt-4o\",\n", " messages=[\n", " {\n", " \"role\": \"user\",\n", " \"content\": [\n", " {\n", " \"type\": \"text\",\n", " \"text\": f\"\"\"\n", " You are a music expert with strong domain knowledge.\n", " Given a list of input artists and their guessed music styles, \n", " please annotate or translate each artist's music style into only English words.\n", " Try to be as accurate as possible, correct the wrong styles, but add the missing styles.\n", " Only annotate the artists that are in the input list.\n", " Keep the answer to be max 7 different styles per artist.\n", " Cross reference with your knowledge of music to make sure the answer is correct.\n", " Remove all the non-confident styles please.\n", " \n", " Format the output as a JSON object where the keys are the artist names and the values are the styles.\n", " The values should be a comma-separated list of styles.\n", " \n", " Artist name is followed by \":\", then the guessed styles.\n", " Like \"ArtistA: style1, style2, style3\"\n", " Each artist is separated by a new line.\n", " Here is the list of artists and their styles.\n", " \"{input_artists}\"\n", " \"\"\",\n", " }\n", " ],\n", " },\n", " ],\n", " temperature=1,\n", " max_tokens=4096,\n", " top_p=1,\n", " frequency_penalty=0,\n", " presence_penalty=0,\n", " response_format={\"type\": \"text\"},\n", " )\n", " return response" ] }, { "cell_type": "code", "execution_count": 83, "id": "efd4ca46", "metadata": {}, "outputs": [], "source": [ "total_ans = defaultdict(str)" ] }, { "cell_type": "code", "execution_count": 84, "id": "5a117f0a", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 1062/1062 [1:02:49<00:00, 3.55s/it]\n" ] } ], "source": [ "missed_indices = []\n", "for i in tqdm.tqdm(range(0, len(TOP_ARTISTS), 20)):\n", " artist_chunk = \"\"\n", " for artist in TOP_ARTISTS[i : i + 20]:\n", " artist_chunk += f\"{artist}: {final_ans[artist]}\\n\"\n", " response = get_response_second_pass(artist_chunk)\n", " try:\n", " curr_ans = (\n", " response.choices[0].message.content.replace(\"json\\n\", \"\").strip(\"`\\n\")\n", " )\n", " curr_ans = json.loads(curr_ans)\n", " if isinstance(curr_ans, dict):\n", " for artist, generes in curr_ans.items():\n", " total_ans[artist] = generes\n", " except Exception as e:\n", " print(f\"Error on chunk {i}, {e}\")\n", " missed_indices.append(i)\n", " continue" ] }, { "cell_type": "code", "execution_count": 85, "id": "9f7f4e2a", "metadata": {}, "outputs": [], "source": [ "final_ans = {}\n", "for artist in TOP_ARTISTS:\n", " if artist in total_ans:\n", " final_ans[artist] = total_ans[artist]\n", " else:\n", " final_ans[artist] = \"\"" ] }, { "cell_type": "code", "execution_count": 86, "id": "fca7935a", "metadata": {}, "outputs": [], "source": [ "with open(\"final_ans_second_pass.json\", \"w\") as f:\n", " json.dump(final_ans, f, indent=4, ensure_ascii=False)" ] }, { "cell_type": "code", "execution_count": 87, "id": "1813ec2b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "21238\n" ] } ], "source": [ "with open(\n", " \"/Users/tonytong/Work/Git/glockenspiel/suno_utils/suno_utils/worker/assets/top_artists.txt.py\"\n", ") as f:\n", " TOP_ARTISTS = [line.strip() for line in f.readlines()]\n", "\n", "print(len(TOP_ARTISTS))" ] }, { "cell_type": "code", "execution_count": 89, "id": "013c6edc", "metadata": {}, "outputs": [], "source": [ "for top_artist in TOP_ARTISTS:\n", " if top_artist not in final_ans:\n", " print(top_artist)" ] }, { "cell_type": "code", "execution_count": null, "id": "9d72632e", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env_dev", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" } }, "nbformat": 4, "nbformat_minor": 5 }