{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:12:43.054877Z",
     "iopub.status.busy": "2025-01-27T21:12:43.054743Z",
     "iopub.status.idle": "2025-01-27T21:12:43.066109Z",
     "shell.execute_reply": "2025-01-27T21:12:43.065692Z",
     "shell.execute_reply.started": "2025-01-27T21:12:43.054862Z"
    }
   },
   "outputs": [],
   "source": [
    "# setup autoload\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:12:43.066706Z",
     "iopub.status.busy": "2025-01-27T21:12:43.066573Z",
     "iopub.status.idle": "2025-01-27T21:12:44.707089Z",
     "shell.execute_reply": "2025-01-27T21:12:44.706555Z",
     "shell.execute_reply.started": "2025-01-27T21:12:43.066692Z"
    }
   },
   "outputs": [],
   "source": [
    "import ast\n",
    "import os\n",
    "import shutil\n",
    "import sys\n",
    "from collections import defaultdict, Counter\n",
    "import json\n",
    "import re\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from suno_utils.utils.s3 import download_s3_files\n",
    "from suno_utils.utils.text import read_json, read_jsonl, write_json, write_jsonl\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "from suno_utils.audio import Audio\n",
    "\n",
    "pd.set_option(\"display.max_rows\", 500)\n",
    "pd.set_option(\"display.max_columns\", 500)\n",
    "pd.set_option(\"display.width\", 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:12:44.707823Z",
     "iopub.status.busy": "2025-01-27T21:12:44.707617Z",
     "iopub.status.idle": "2025-01-27T21:12:48.198458Z",
     "shell.execute_reply": "2025-01-27T21:12:48.197871Z",
     "shell.execute_reply.started": "2025-01-27T21:12:44.707807Z"
    }
   },
   "outputs": [],
   "source": [
    "data_metas = read_jsonl(\"/app/suno/tmp/raw_imslp_metas.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:12:48.199201Z",
     "iopub.status.busy": "2025-01-27T21:12:48.199045Z",
     "iopub.status.idle": "2025-01-27T21:12:48.215338Z",
     "shell.execute_reply": "2025-01-27T21:12:48.214845Z",
     "shell.execute_reply.started": "2025-01-27T21:12:48.199184Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of data:  274275\n"
     ]
    }
   ],
   "source": [
    "print(\"Total number of data: \", len(data_metas))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:12:48.215952Z",
     "iopub.status.busy": "2025-01-27T21:12:48.215813Z",
     "iopub.status.idle": "2025-01-27T21:12:48.439297Z",
     "shell.execute_reply": "2025-01-27T21:12:48.438816Z",
     "shell.execute_reply.started": "2025-01-27T21:12:48.215937Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of unique titles:  26343\n"
     ]
    }
   ],
   "source": [
    "unique_titles = defaultdict(str)\n",
    "for meta in data_metas:\n",
    "    unique_titles[meta.get(\"title\", \"\")] = meta.get(\"composer\", \"\")\n",
    "\n",
    "print(\"Total number of unique titles: \", len(unique_titles))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:12:48.441009Z",
     "iopub.status.busy": "2025-01-27T21:12:48.440676Z",
     "iopub.status.idle": "2025-01-27T21:12:48.560712Z",
     "shell.execute_reply": "2025-01-27T21:12:48.560245Z",
     "shell.execute_reply.started": "2025-01-27T21:12:48.440993Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of unique genres:  18\n",
      "[('Romantic', 124695), ('Baroque', 69226), ('Classical', 47391), ('Early 20th century', 21593), ('Modern', 7641), ('Renaissance', 3030), ('Medieval', 158), ('Traditional (folk)', 116), ('early 20th century', 101), ('Jazz', 95)]\n"
     ]
    }
   ],
   "source": [
    "unique_genres = Counter()\n",
    "for meta in data_metas:\n",
    "    unique_genres[meta.get(\"genre\", \"\")] += 1\n",
    "\n",
    "print(\"Total number of unique genres: \", len(unique_genres))\n",
    "print(unique_genres.most_common(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:12:48.561319Z",
     "iopub.status.busy": "2025-01-27T21:12:48.561181Z",
     "iopub.status.idle": "2025-01-27T21:12:48.727096Z",
     "shell.execute_reply": "2025-01-27T21:12:48.726624Z",
     "shell.execute_reply.started": "2025-01-27T21:12:48.561304Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of unique instruments:  3217\n",
      "[('piano', 48867), ('orchestra', 24642), ('voices, chorus, orchestra', 22794), ('voice, piano', 9966), ('voices, mixed chorus (SATB), orchestra', 8613), ('organ', 8315), ('voices, mixed chorus, orchestra', 7975), ('piano, orchestra', 5576), ('2 violins, viola, cello', 5320), ('keyboard', 5249)]\n"
     ]
    }
   ],
   "source": [
    "unique_instruments = Counter()\n",
    "for meta in data_metas:\n",
    "    unique_instruments[meta.get(\"instruments\", \"\")] += 1\n",
    "\n",
    "print(\"Total number of unique instruments: \", len(unique_instruments))\n",
    "print(unique_instruments.most_common(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:12:48.727720Z",
     "iopub.status.busy": "2025-01-27T21:12:48.727583Z",
     "iopub.status.idle": "2025-01-27T21:12:48.740704Z",
     "shell.execute_reply": "2025-01-27T21:12:48.740254Z",
     "shell.execute_reply.started": "2025-01-27T21:12:48.727705Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id: 92ea248d-c192-4702-bf7e-5662b3ab3551\n",
      "s3_filepath: s3://suno-data/datasets/harvest/imslp/audio/92ea248d-c192-4702-bf7e-5662b3ab3551.mp4\n",
      "duration_s: 248\n",
      "composer: Mozart, Wolfgang Amadeus\n",
      "recording_category: Commercial Recordings\n",
      "tags: ['orchestra Solo Voices (soprano', '2 horns', 'chorus', 'Mass No. 16 in C Major', '2 trumpets', 'bass) Mixed Chorus Orchestra 2 oboes', 'Classical', 'Wolfgang Amadeus', '3 trombones timpani', 'organ', '\"Kronungsmesse\" (Coronation Mass)', 'tenor', '2 bassoons', 'alto', 'strings (no violas)', 'K. 317', 'Mozart', 'voices']\n",
      "tags_redacted: ['orchestra Solo Voices (soprano', '2 horns', 'chorus', '2 trumpets', 'organ', 'bass) Mixed Chorus Orchestra 2 oboes', 'Classical', '3 trombones timpani', 'tenor', '2 bassoons', 'alto', 'strings (no violas)', 'voices']\n",
      "title: Mass No. 16 in C Major, K. 317, \"Kronungsmesse\" (Coronation Mass)\n",
      "instruments: voices, chorus, orchestra \n",
      "    Solo Voices (soprano, alto, tenor, bass)\n",
      "Mixed Chorus\n",
      "Orchestra\n",
      "        2 oboes, 2 bassoons, 2 horns, 2 trumpets, 3 trombones\n",
      "timpani, organ, strings (no violas)\n",
      "genre: Classical\n",
      "original_id: 61633\n"
     ]
    }
   ],
   "source": [
    "for meta in data_metas:\n",
    "    if \"mass\" in meta.get(\"title\", \"\").lower():\n",
    "        for key, value in meta.items():\n",
    "            print(f\"{key}: {value}\")\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:12:48.741303Z",
     "iopub.status.busy": "2025-01-27T21:12:48.741169Z",
     "iopub.status.idle": "2025-01-27T21:12:48.895860Z",
     "shell.execute_reply": "2025-01-27T21:12:48.895419Z",
     "shell.execute_reply.started": "2025-01-27T21:12:48.741289Z"
    }
   },
   "outputs": [],
   "source": [
    "cleaned_titles = {}\n",
    "for title in unique_titles:\n",
    "    cleaned_title = re.sub(r\"\\d+\", \"\", title.split(\",\")[0].strip())\n",
    "    cleaned_title = re.sub(r\"\\b(no|No|NO)\\b\", \"\", cleaned_title)\n",
    "    cleaned_title = re.sub(r\"[^\\w\\s]\", \"\", cleaned_title)\n",
    "    cleaned_title = re.sub(r\"\\s+\", \" \", cleaned_title)\n",
    "    cleaned_titles[title] = cleaned_title\n",
    "\n",
    "# for title, count in unique_titles.most_common(100):\n",
    "#     print(f\"{title}: {count}, --> {cleaned_titles[title]} \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:12:48.896476Z",
     "iopub.status.busy": "2025-01-27T21:12:48.896338Z",
     "iopub.status.idle": "2025-01-27T21:12:48.910014Z",
     "shell.execute_reply": "2025-01-27T21:12:48.909596Z",
     "shell.execute_reply.started": "2025-01-27T21:12:48.896461Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Piano Concerto in A Minor'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_titles[\"Piano Concerto in A Minor, Op. 16\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:12:48.910665Z",
     "iopub.status.busy": "2025-01-27T21:12:48.910528Z",
     "iopub.status.idle": "2025-01-27T21:12:51.143354Z",
     "shell.execute_reply": "2025-01-27T21:12:51.142779Z",
     "shell.execute_reply.started": "2025-01-27T21:12:48.910650Z"
    }
   },
   "outputs": [],
   "source": [
    "id_to_cleaned_tags = {}\n",
    "for meta in data_metas:\n",
    "    cleaned_tags = []\n",
    "    cleaned_title = re.sub(r\"\\d+\", \"\", meta.get(\"title\", \"\").split(\",\")[0].strip())\n",
    "    cleaned_title = re.sub(r\"\\b(no|No|NO)\\b\", \"\", cleaned_title)\n",
    "    cleaned_title = re.sub(r\"[^\\w\\s]\", \"\", cleaned_title)\n",
    "    cleaned_title = re.sub(r\"\\s+\", \" \", cleaned_title)\n",
    "    if cleaned_title != \"\":\n",
    "        cleaned_tags.append(cleaned_title)\n",
    "    for i, instrument in enumerate(meta.get(\"instruments\", \"\").split(\"\\n\")):\n",
    "        if i == 0:\n",
    "            parsed_instrument = \"\".join(\n",
    "                x_instrument\n",
    "                for x_instrument in instrument.split(\",\")\n",
    "                if \" \" not in x_instrument\n",
    "            )\n",
    "            if parsed_instrument != \"\":\n",
    "                cleaned_tags.append(parsed_instrument)\n",
    "        elif len(instrument.split(\" \")) == 1 and instrument != \"\":\n",
    "            cleaned_tags.append(instrument)\n",
    "    if genre := meta.get(\"genre\", \"\"):\n",
    "        cleaned_tags.append(genre)\n",
    "    meta[\"tags_clean\"] = cleaned_tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:12:51.144062Z",
     "iopub.status.busy": "2025-01-27T21:12:51.143909Z",
     "iopub.status.idle": "2025-01-27T21:12:51.159637Z",
     "shell.execute_reply": "2025-01-27T21:12:51.159139Z",
     "shell.execute_reply.started": "2025-01-27T21:12:51.144046Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id: 23b54f5d-1317-4b45-8587-e0842096b735\n",
      "s3_filepath: s3://suno-data/datasets/harvest/imslp/audio/23b54f5d-1317-4b45-8587-e0842096b735.mp3\n",
      "duration_s: 365\n",
      "composer: Beethoven, Ludwig van\n",
      "recording_category: Commercial Recordings\n",
      "tags: ['Classical', 'Ludwig van', 'Beethoven', 'Op. 55 \"Héroïque\"', 'Symphonie No. 3 in E-Flat Major', 'orchestra']\n",
      "tags_redacted: ['Classical', 'orchestra']\n",
      "title: Symphonie No. 3 in E-Flat Major, Op. 55 \"Héroïque\"\n",
      "instruments: orchestra\n",
      "genre: Classical\n",
      "original_id: 2581\n",
      "tags_clean: ['Symphonie in EFlat Major', 'orchestra', 'Classical']\n"
     ]
    }
   ],
   "source": [
    "for meta in data_metas:\n",
    "    if \"symphonie\" in meta.get(\"title\", \"\").lower():\n",
    "        for key, value in meta.items():\n",
    "            print(f\"{key}: {value}\")\n",
    "        # print(\"--->\", id_to_cleaned_tags[meta.get(\"id\")])\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:12:51.160291Z",
     "iopub.status.busy": "2025-01-27T21:12:51.160143Z",
     "iopub.status.idle": "2025-01-27T21:12:51.204054Z",
     "shell.execute_reply": "2025-01-27T21:12:51.203631Z",
     "shell.execute_reply.started": "2025-01-27T21:12:51.160276Z"
    }
   },
   "outputs": [],
   "source": [
    "# write_jsonl(data_metas, \"/home/tony/Data/classical/cleand_imslp_metas.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:13:03.375853Z",
     "iopub.status.busy": "2025-01-27T21:13:03.375500Z",
     "iopub.status.idle": "2025-01-27T21:13:03.392417Z",
     "shell.execute_reply": "2025-01-27T21:13:03.391919Z",
     "shell.execute_reply.started": "2025-01-27T21:13:03.375833Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "\n",
    "# Assuming df is already defined and loaded with data\n",
    "# Use regular expression to search for 'wir ziehen' in a specific column, e.g., 'text_column'\n",
    "# Use vectorized string operations for better performance\n",
    "# matches = df[\n",
    "#     df[\"text_column\"].str.contains(r\"wir ziehen\", case=False, regex=True, na=False)\n",
    "# ]\n",
    "\n",
    "# print(matches)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# OPEN AI SHIT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:14:38.555959Z",
     "iopub.status.busy": "2025-01-27T21:14:38.555663Z",
     "iopub.status.idle": "2025-01-27T21:14:38.599961Z",
     "shell.execute_reply": "2025-01-27T21:14:38.599502Z",
     "shell.execute_reply.started": "2025-01-27T21:14:38.555942Z"
    }
   },
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "\n",
    "client = OpenAI()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:14:40.842448Z",
     "iopub.status.busy": "2025-01-27T21:14:40.842297Z",
     "iopub.status.idle": "2025-01-27T21:14:40.857303Z",
     "shell.execute_reply": "2025-01-27T21:14:40.856882Z",
     "shell.execute_reply.started": "2025-01-27T21:14:40.842433Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26343"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(unique_titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:14:41.390838Z",
     "iopub.status.busy": "2025-01-27T21:14:41.390692Z",
     "iopub.status.idle": "2025-01-27T21:14:41.446502Z",
     "shell.execute_reply": "2025-01-27T21:14:41.446080Z",
     "shell.execute_reply.started": "2025-01-27T21:14:41.390823Z"
    }
   },
   "outputs": [],
   "source": [
    "title_composers = [\n",
    "    \"titile: {} composer: {}\".format(title, composer)\n",
    "    for title, composer in unique_titles.items()\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:14:42.034997Z",
     "iopub.status.busy": "2025-01-27T21:14:42.034852Z",
     "iopub.status.idle": "2025-01-27T21:14:42.049222Z",
     "shell.execute_reply": "2025-01-27T21:14:42.048767Z",
     "shell.execute_reply.started": "2025-01-27T21:14:42.034982Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['titile: Contrasts, for Clarinet, Violin and Piano, Sz. 111 composer: Bartók, Béla', 'titile:  composer: Schumann, Robert', 'titile: Improvisations for Piano composer: Zintl, Frank']\n"
     ]
    }
   ],
   "source": [
    "print(title_composers[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:17:53.872275Z",
     "iopub.status.busy": "2025-01-27T21:17:53.871844Z",
     "iopub.status.idle": "2025-01-27T21:17:53.894303Z",
     "shell.execute_reply": "2025-01-27T21:17:53.893829Z",
     "shell.execute_reply.started": "2025-01-27T21:17:53.872253Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_response(input_artists):\n",
    "    response = client.chat.completions.create(\n",
    "        model=\"gpt-4o\",\n",
    "        messages=[\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": [\n",
    "                    {\n",
    "                        \"type\": \"text\",\n",
    "                        \"text\": f\"\"\"\n",
    "          You are a music expert.\n",
    "          Given a list of input music title and the composer, \n",
    "          please annotate piece's genre, key, and short description words.\n",
    "          \n",
    "          Try to be as accurate as possible, but also comprehensive as possible.\n",
    "          Only annotate the inputs that are in the input list.\n",
    "          Keep the answer to be original to the title and try not to augment.\n",
    "          Cross reference with your knowledge of music to make sure the answer is correct.\n",
    "          Remove all the non-confident answers please.\n",
    "        \n",
    "          Format the output as a JSON object with a list of the values.\n",
    "          The values should be a comma-separated list of styles.\n",
    "          You should return the same number of outputs as the number of inputs.\n",
    "\n",
    "          For example, \n",
    "          if input is:\n",
    "              title: \"Symphonie No. 3 in E-Flat Major, Op. 55 \"Héroïque\"\" composer: Beethoven\n",
    "              title: \"Contrasts, for Clarinet, Violin and Piano, Sz. 111\" composer: Bartók, Béla\n",
    "          the output should be:\n",
    "          'Symphonie No. 3 in E-Flat Major, Op. 55 \"Héroïque\"': \"Symphony, Classical, Romantic, German, E-Flat Major, Heroic, Dramatic, Epic\"\n",
    "          'Contrasts, for Clarinet, Violin and Piano, Sz. 111': \"Chamber Music, 20th Century, Hungarian, Rhythmic, Folk Influences\"\n",
    "\n",
    "          Here is the list of titles and composers (separated by new lines):\n",
    "          \"{input_artists}\"\n",
    "          \"\"\",\n",
    "                    }\n",
    "                ],\n",
    "            },\n",
    "        ],\n",
    "        temperature=0.5,\n",
    "        max_tokens=4096,\n",
    "        top_p=1,\n",
    "        frequency_penalty=0,\n",
    "        presence_penalty=0,\n",
    "        response_format={\"type\": \"text\"},\n",
    "    )\n",
    "    return response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:17:54.652043Z",
     "iopub.status.busy": "2025-01-27T21:17:54.651666Z",
     "iopub.status.idle": "2025-01-27T21:17:54.672280Z",
     "shell.execute_reply": "2025-01-27T21:17:54.671806Z",
     "shell.execute_reply.started": "2025-01-27T21:17:54.652024Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "titile: Contrasts, for Clarinet, Violin and Piano, Sz. 111 composer: Bartók, Béla\n",
      "titile:  composer: Schumann, Robert\n",
      "titile: Improvisations for Piano composer: Zintl, Frank\n"
     ]
    }
   ],
   "source": [
    "test_title_composers = title_composers[:3]\n",
    "for title_composer in test_title_composers:\n",
    "    print(title_composer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:17:57.433953Z",
     "iopub.status.busy": "2025-01-27T21:17:57.433786Z",
     "iopub.status.idle": "2025-01-27T21:18:03.898857Z",
     "shell.execute_reply": "2025-01-27T21:18:03.898268Z",
     "shell.execute_reply.started": "2025-01-27T21:17:57.433937Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ChatCompletion(id='chatcmpl-AuQmTDBBnmfG3qU7U3QYZ7QM7pOlz', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content='```json\\n{\\n    \"Contrasts, for Clarinet, Violin and Piano, Sz. 111\": \"Chamber Music, 20th Century, Hungarian, Rhythmic, Folk Influences\",\\n    \"Improvisations for Piano\": \"Piano, Contemporary, Improvisational, Modern\"\\n}\\n```', refusal=None, role='assistant', audio=None, function_call=None, tool_calls=None))], created=1738012677, model='gpt-4o-2024-08-06', object='chat.completion', service_tier='default', system_fingerprint='fp_50cad350e4', usage=CompletionUsage(completion_tokens=66, prompt_tokens=393, total_tokens=459, completion_tokens_details=CompletionTokensDetails(audio_tokens=0, reasoning_tokens=0, accepted_prediction_tokens=0, rejected_prediction_tokens=0), prompt_tokens_details=PromptTokensDetails(audio_tokens=0, cached_tokens=0)))\n"
     ]
    }
   ],
   "source": [
    "test_response = get_response(test_title_composers)\n",
    "print(test_response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:18:04.539084Z",
     "iopub.status.busy": "2025-01-27T21:18:04.538897Z",
     "iopub.status.idle": "2025-01-27T21:18:05.679689Z",
     "shell.execute_reply": "2025-01-27T21:18:05.679177Z",
     "shell.execute_reply.started": "2025-01-27T21:18:04.539064Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "```json\n",
      "{\n",
      "    \"Contrasts, for Clarinet, Violin and Piano, Sz. 111\": \"Chamber Music, 20th Century, Hungarian, Rhythmic, Folk Influences\",\n",
      "    \"Improvisations for Piano\": \"Piano, Contemporary, Improvisational, Modern\"\n",
      "}\n",
      "```\n"
     ]
    }
   ],
   "source": [
    "print(test_response.choices[0].message.content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Production run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:20:56.093433Z",
     "iopub.status.busy": "2025-01-27T21:20:56.093002Z",
     "iopub.status.idle": "2025-01-27T21:20:56.113908Z",
     "shell.execute_reply": "2025-01-27T21:20:56.113423Z",
     "shell.execute_reply.started": "2025-01-27T21:20:56.093410Z"
    }
   },
   "outputs": [],
   "source": [
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T21:20:56.660116Z",
     "iopub.status.busy": "2025-01-27T21:20:56.659956Z",
     "iopub.status.idle": "2025-01-27T21:20:56.675963Z",
     "shell.execute_reply": "2025-01-27T21:20:56.675530Z",
     "shell.execute_reply.started": "2025-01-27T21:20:56.660099Z"
    }
   },
   "outputs": [],
   "source": [
    "total_ans = defaultdict(str)\n",
    "# for title_composer in tqdm.tqdm(title_composers):\n",
    "#     total_ans[title_composer] = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-28T02:07:24.773231Z",
     "iopub.status.busy": "2025-01-28T02:07:24.772793Z",
     "iopub.status.idle": "2025-01-28T02:07:36.968338Z",
     "shell.execute_reply": "2025-01-28T02:07:36.967781Z",
     "shell.execute_reply.started": "2025-01-28T02:07:24.773208Z"
    }
   },
   "outputs": [],
   "source": [
    "failed_chunks = []\n",
    "# for i in tqdm.tqdm(range(0, len(title_composers), 20)):\n",
    "for i in [\n",
    "    6700,\n",
    "]:\n",
    "    artist_chunk = \"\\n\".join(title_composers[i : i + 20])\n",
    "    response = get_response(artist_chunk)\n",
    "    try:\n",
    "        curr_ans = (\n",
    "            response.choices[0].message.content.replace(\"json\\n\", \"\").strip(\"`\\n\")\n",
    "        )\n",
    "        curr_ans = json.loads(curr_ans)\n",
    "        if isinstance(curr_ans, dict):\n",
    "            for artist, generes in curr_ans.items():\n",
    "                total_ans[artist] = generes\n",
    "    except Exception as e:\n",
    "        print(f\"Error on chunk {i}, {e}\")\n",
    "        failed_chunks.append(i)\n",
    "        continue\n",
    "    # break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-28T02:07:37.688323Z",
     "iopub.status.busy": "2025-01-28T02:07:37.688153Z",
     "iopub.status.idle": "2025-01-28T02:07:37.708524Z",
     "shell.execute_reply": "2025-01-28T02:07:37.708057Z",
     "shell.execute_reply.started": "2025-01-28T02:07:37.688306Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26221\n"
     ]
    }
   ],
   "source": [
    "print(len(total_ans))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-28T02:07:40.296433Z",
     "iopub.status.busy": "2025-01-28T02:07:40.296045Z",
     "iopub.status.idle": "2025-01-28T02:07:40.313573Z",
     "shell.execute_reply": "2025-01-28T02:07:40.313161Z",
     "shell.execute_reply.started": "2025-01-28T02:07:40.296413Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "failed_chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-28T02:18:48.306666Z",
     "iopub.status.busy": "2025-01-28T02:18:48.306104Z",
     "iopub.status.idle": "2025-01-28T02:18:48.946326Z",
     "shell.execute_reply": "2025-01-28T02:18:48.945819Z",
     "shell.execute_reply.started": "2025-01-28T02:18:48.306645Z"
    }
   },
   "outputs": [],
   "source": [
    "id_to_cleaned_tags = {}\n",
    "for meta in data_metas:\n",
    "    cleaned_tags = []\n",
    "    current_title = meta.get(\"title\", \"\") or \"\"\n",
    "    cleaned_title = total_ans.get(current_title) or \"\"\n",
    "    cleaned_tags.extend([x.strip() for x in cleaned_title.split(\",\")])\n",
    "    if \"in\" in current_title and (\"minor\" in current_title.lower() or \"major\" in current_title.lower()):\n",
    "        try:\n",
    "            work_key = meta.get(\"title\", \"\").split(\"in \")[1].split(\",\")[0]\n",
    "            if work_key not in cleaned_tags:\n",
    "                cleaned_tags.append(work_key.strip())\n",
    "        except:\n",
    "            pass\n",
    "    # for i, instrument in enumerate(meta.get(\"instruments\", \"\").split(\"\\n\")):\n",
    "    #     if i == 0:\n",
    "    #         parsed_instrument = \"\".join(\n",
    "    #             x_instrument\n",
    "    #             for x_instrument in instrument.split(\",\")\n",
    "    #             if \" \" not in x_instrument\n",
    "    #         )\n",
    "    #         if parsed_instrument != \"\":\n",
    "    #             cleaned_tags.append(parsed_instrument)\n",
    "    #     elif len(instrument.split(\" \")) == 1 and instrument != \"\":\n",
    "    #         cleaned_tags.append(instrument)\n",
    "    # if genre := meta.get(\"genre\", \"\"):\n",
    "    #     cleaned_tags.append(genre)\n",
    "    meta[\"tags_clean\"] = cleaned_tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-28T02:18:54.346103Z",
     "iopub.status.busy": "2025-01-28T02:18:54.345800Z",
     "iopub.status.idle": "2025-01-28T02:18:54.366699Z",
     "shell.execute_reply": "2025-01-28T02:18:54.366229Z",
     "shell.execute_reply.started": "2025-01-28T02:18:54.346083Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id: d2c26f32-720b-4127-9a38-50d59c4e9563\n",
      "s3_filepath: s3://suno-data/datasets/harvest/imslp/audio/d2c26f32-720b-4127-9a38-50d59c4e9563.mp4\n",
      "duration_s: 622\n",
      "composer: Grieg, Edvard\n",
      "recording_category: Commercial Recordings\n",
      "tags: ['Romantic', 'piano', 'orchestra', 'Grieg', 'Op. 16', 'Piano Concerto in A Minor', 'Edvard']\n",
      "tags_redacted: ['Romantic', 'piano', 'orchestra']\n",
      "title: Piano Concerto in A Minor, Op. 16\n",
      "instruments: piano, orchestra\n",
      "genre: Romantic\n",
      "original_id: 3728\n",
      "tags_clean: ['Concerto', 'Romantic', 'Norwegian', 'Lyrical', 'Dramatic', 'A Minor']\n"
     ]
    }
   ],
   "source": [
    "for meta in data_metas:\n",
    "    if \"piano concerto\" in meta.get(\"title\", \"\").lower():\n",
    "        for key, value in meta.items():\n",
    "            print(f\"{key}: {value}\")\n",
    "        # print(\"--->\", id_to_cleaned_tags[meta.get(\"id\")])\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-28T02:18:57.637370Z",
     "iopub.status.busy": "2025-01-28T02:18:57.636685Z",
     "iopub.status.idle": "2025-01-28T02:19:00.648942Z",
     "shell.execute_reply": "2025-01-28T02:19:00.648332Z",
     "shell.execute_reply.started": "2025-01-28T02:18:57.637348Z"
    }
   },
   "outputs": [],
   "source": [
    "write_jsonl(data_metas, \"/home/tony/Data/classical/augmented_imslp_metas.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}