{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T23:13:24.025156Z", "iopub.status.busy": "2025-11-21T23:13:24.024741Z", "iopub.status.idle": "2025-11-21T23:13:24.040984Z", "shell.execute_reply": "2025-11-21T23:13:24.040435Z", "shell.execute_reply.started": "2025-11-21T23:13:24.025134Z" } }, "outputs": [], "source": [ "# setup autoload\n", "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.040680Z", "start_time": "2024-05-16T13:58:19.777010Z" }, "execution": { "iopub.execute_input": "2025-11-21T23:13:24.289432Z", "iopub.status.busy": "2025-11-21T23:13:24.289084Z", "iopub.status.idle": "2025-11-21T23:13:27.505581Z", "shell.execute_reply": "2025-11-21T23:13:27.504809Z", "shell.execute_reply.started": "2025-11-21T23:13:24.289414Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The autoreload extension is already loaded. To reload it, use:\n", " %reload_ext autoreload\n" ] } ], "source": [ "import ast\n", "import os\n", "import shutil\n", "import sys\n", "from collections import defaultdict\n", "\n", "import json\n", "import numpy as np\n", "import pandas as pd\n", "from preference_data_preparation_auk import *\n", "from preference_helper import *\n", "from sklearn.model_selection import train_test_split\n", "from suno_utils.utils.s3 import download_s3_files\n", "from suno_utils.utils.text import read_json, read_jsonl, write_json, write_jsonl\n", "from tqdm import tqdm\n", "\n", "pd.set_option(\"display.max_rows\", 500)\n", "pd.set_option(\"display.max_columns\", 500)\n", "pd.set_option(\"display.width\", 1000)\n", "\n", "# setup autoload\n", "%load_ext autoreload\n", "%autoreload 2\n", "\n", "\n", "def custom_parse(x):\n", " try:\n", " return json.loads(x)\n", " except:\n", " return {}" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.082172Z", "start_time": "2024-05-16T13:58:21.041926Z" }, "execution": { "iopub.execute_input": "2025-11-21T23:13:27.506971Z", "iopub.status.busy": "2025-11-21T23:13:27.506600Z", "iopub.status.idle": "2025-11-21T23:13:27.578311Z", "shell.execute_reply": "2025-11-21T23:13:27.577751Z", "shell.execute_reply.started": "2025-11-21T23:13:27.506954Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "N_TOKENS_AUDIO 12000\n" ] } ], "source": [ "OUT_DATA_DIR = \"/app2/suno/data/dpo/crow_t1_v61\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "shutil.copyfile(\n", " \"/app/suno/data/dpo/7v_v20_full/tokenizer_60k.json\",\n", " os.path.join(OUT_DATA_DIR, \"tokenizer_60k.json\"),\n", ")\n", "NPZ_DIR = \"/app2/suno/data/dpo/crow_t1_npz\"\n", "N_TOKENS_AUDIO = 25 * 8 * 60\n", "print(\"N_TOKENS_AUDIO\", N_TOKENS_AUDIO)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T15:20:03.378865Z", "iopub.status.busy": "2025-11-21T15:20:03.378723Z", "iopub.status.idle": "2025-11-21T15:20:03.397835Z", "shell.execute_reply": "2025-11-21T15:20:03.397280Z", "shell.execute_reply.started": "2025-11-21T15:20:03.378849Z" } }, "outputs": [], "source": [ "# import glob\n", "# import os\n", "# from tqdm import tqdm\n", "\n", "# # Load the base dataframe\n", "# base_file = \"/home/tony/Data/Preference/crow_t1/fully_merged_crow_t1_20251119.pkl\"\n", "# if os.path.exists(base_file):\n", "# df = pd.read_pickle(base_file)\n", "# base_time_file = base_file\n", "# base_ctime = os.path.getctime(base_time_file)\n", "# else:\n", "# # If no base file exists, start with empty dataframe\n", "# df = pd.DataFrame()\n", "# base_ctime = 0\n", "\n", "# # Find all pkl files in the directory with same name pattern\n", "# # pkl_files = glob.glob(\"/home/tony/Data/Preference/up_v2_d5/interesting_clips_*.pkl\")\n", "# pkl_files = [\"/home/tony/Data/Preference/crow_t1/interesting_clips_crow_t1_20251120.pkl\"]\n", "\n", "# # Filter files that are newer than the base file and print debug info\n", "# newer_files = []\n", "# for f in pkl_files:\n", "# f_ctime = os.path.getctime(f)\n", "# if f_ctime > base_ctime:\n", "# newer_files.append(f)\n", "# print(f\"Found newer file: {os.path.basename(f)}\")\n", "# print(f\" Base file ctime: {base_ctime}\")\n", "# print(f\" File ctime: {f_ctime}\")\n", "# print(f\" Difference: {f_ctime - base_ctime} seconds\")\n", "\n", "# newer_files.sort(key=lambda x: os.path.getctime(x))\n", "\n", "# print(f\"Found {len(newer_files)} newer files to process\")\n", "\n", "# # Process each newer file\n", "# for pkl_file in tqdm(newer_files, desc=\"Loading newer pickle files\"):\n", "# print(f\"\\nProcessing {os.path.basename(pkl_file)}\")\n", "# prev_size = len(df)\n", "# temp_df = pd.read_pickle(pkl_file)\n", "# new_size = len(temp_df)\n", "\n", "# # Convert datetime columns if they exist\n", "# for col in [\"created_at\", \"updated_at\"]:\n", "# if col in temp_df.columns:\n", "# temp_df[col] = pd.to_datetime(temp_df[col], utc=True)\n", "\n", "# # Handle duplicates based on id\n", "# if \"id\" in temp_df.columns:\n", "# df = pd.concat([df, temp_df], ignore_index=True)\n", "# df = df.drop_duplicates(subset=[\"id\"], keep=\"last\")\n", "# else:\n", "# df = pd.concat([df, temp_df], ignore_index=True)\n", "\n", "# # Print size statistics\n", "# current_size = len(df)\n", "# net_increase = current_size - prev_size\n", "# print(f\"Previous size: {prev_size:,}\")\n", "# print(f\"New input size: {new_size:,}\")\n", "# print(f\"Current total size: {current_size:,}\")\n", "# print(f\"Net increase: {net_increase:,}\")\n", "\n", "# print(\"\\nFinal dataframe shape:\", df.shape)\n", "# print(\n", "# \"Unique ids:\",\n", "# df[\"id\"].nunique() if \"id\" in df.columns else \"No id column\",\n", "# )\n", "# df.to_pickle(\"/home/tony/Data/Preference/crow_t1/fully_merged_crow_t1_20251120.pkl\")" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T15:20:03.399880Z", "iopub.status.busy": "2025-11-21T15:20:03.399748Z", "iopub.status.idle": "2025-11-21T15:30:04.789156Z", "shell.execute_reply": "2025-11-21T15:30:04.788569Z", "shell.execute_reply.started": "2025-11-21T15:20:03.399866Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "before dropna (20426651, 96)\n", "after dropna (20426651, 90)\n" ] } ], "source": [ "df = pd.read_pickle(\n", " \"/home/tony/Data/Preference/crow_t1/fully_merged_crow_t1_20251120.pkl\"\n", ")\n", "print(\"before dropna\", df.shape)\n", "df = df.dropna(axis=1, how=\"all\")\n", "print(\"after dropna\", df.shape)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T15:30:04.789849Z", "iopub.status.busy": "2025-11-21T15:30:04.789696Z", "iopub.status.idle": "2025-11-21T15:30:04.879775Z", "shell.execute_reply": "2025-11-21T15:30:04.879300Z", "shell.execute_reply.started": "2025-11-21T15:30:04.789834Z" } }, "outputs": [ { "data": { "text/plain": [ "preference\n", "True 10218547\n", "False 10208104\n", "Name: count, dtype: int64" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"preference\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T15:30:04.880440Z", "iopub.status.busy": "2025-11-21T15:30:04.880295Z", "iopub.status.idle": "2025-11-21T15:30:39.265189Z", "shell.execute_reply": "2025-11-21T15:30:39.264582Z", "shell.execute_reply.started": "2025-11-21T15:30:04.880426Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(20426651, 90)\n", "(20423832, 91)\n" ] } ], "source": [ "# sth maybe happening with clip_id duplication?\n", "print(df.shape)\n", "df[\"request_sum\"] = df.groupby(\"request_id\")[\"preference\"].transform(\"sum\")\n", "# creation of interesting_clips\n", "df = df[(df[\"request_sum\"] == 1)]\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.199480Z", "start_time": "2024-05-16T13:58:53.963687Z" }, "execution": { "iopub.execute_input": "2025-11-21T15:30:39.265954Z", "iopub.status.busy": "2025-11-21T15:30:39.265802Z", "iopub.status.idle": "2025-11-21T15:50:33.338162Z", "shell.execute_reply": "2025-11-21T15:50:33.337375Z", "shell.execute_reply.started": "2025-11-21T15:30:39.265938Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "20610494\n", "20610494\n", "pre-downloaded df (20423832, 91)\n", "downloaded df (20286457, 91)\n" ] } ], "source": [ "converted_paths = os.listdir(NPZ_DIR)\n", "print(len(converted_paths))\n", "\n", "converted_paths = set([f.replace(\".npz\", \"\") for f in converted_paths])\n", "print(len(converted_paths))\n", "\n", "print(\"pre-downloaded df\", df.shape)\n", "df[df[\"s3_id\"].isin(converted_paths)].shape\n", "df = df[df[\"s3_id\"].isin(converted_paths)].copy()\n", "print(\"downloaded df\", df.shape)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.467253Z", "start_time": "2024-05-16T13:58:56.207647Z" }, "execution": { "iopub.execute_input": "2025-11-21T15:50:33.340831Z", "iopub.status.busy": "2025-11-21T15:50:33.340609Z", "iopub.status.idle": "2025-11-21T15:50:35.139602Z", "shell.execute_reply": "2025-11-21T15:50:35.139088Z", "shell.execute_reply.started": "2025-11-21T15:50:33.340812Z" } }, "outputs": [ { "data": { "text/plain": [ "task\n", " 11962034\n", "cover 5461528\n", "artist_consistency 1179851\n", "artist_cover 734982\n", "extend 258184\n", "playlist_condition 225686\n", "overpainting 116420\n", "upload_extend 92170\n", "underpainting 90073\n", "infill 74154\n", "artist_extend 49719\n", "stem_condition 33316\n", "stem_condition_infill 8336\n", "generate 4\n", "Name: count, dtype: int64" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"task\"].value_counts()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# LET's do the data prep" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.592883Z", "start_time": "2024-05-16T13:58:56.470781Z" }, "execution": { "iopub.execute_input": "2025-11-21T15:50:35.140334Z", "iopub.status.busy": "2025-11-21T15:50:35.140187Z", "iopub.status.idle": "2025-11-21T15:50:49.858862Z", "shell.execute_reply": "2025-11-21T15:50:49.858305Z", "shell.execute_reply.started": "2025-11-21T15:50:35.140319Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "preference model_name \n", "False chirp-crow-t1 10115820\n", " chirp-crow-t2 20825\n", " chirp-crow-t1-c2 30\n", " chirp-crow-t1-c1 11\n", "True chirp-crow-t1 10043419\n", " chirp-crow-t1-c2 48946\n", " chirp-crow-t1-c1 36579\n", " chirp-crow-t2 20827\n", "Name: count, dtype: int64\n", "before filter on model name (20286457, 91)\n", "after filter on model name (20200891, 91)\n", "is_public\n", "False 19342930\n", "True 857961\n", "Name: count, dtype: int64\n" ] } ], "source": [ "## for 13b this is easy for now\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "print(\"before filter on model name\", df.shape)\n", "df = df[df[\"model_name\"].isin([\"chirp-crow-t1\", \"chirp-crow-t2\"])]\n", "# df = df[df[\"model_name\"].isin([\"chirp-v3p5-engine-t-6\"])]\n", "print(\"after filter on model name\", df.shape)\n", "print(df[\"is_public\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.909539Z", "start_time": "2024-05-16T13:58:56.595736Z" }, "execution": { "iopub.execute_input": "2025-11-21T15:50:49.863138Z", "iopub.status.busy": "2025-11-21T15:50:49.862851Z", "iopub.status.idle": "2025-11-21T15:51:39.221969Z", "shell.execute_reply": "2025-11-21T15:51:39.221390Z", "shell.execute_reply.started": "2025-11-21T15:50:49.863122Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "before filter on request id pairs (20200891, 91)\n", "after filter on request id pairs (20102320, 91)\n", "preference model_name \n", "False chirp-crow-t1 10030335\n", " chirp-crow-t2 20825\n", "True chirp-crow-t1 10030335\n", " chirp-crow-t2 20825\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(\"before filter on request id pairs\", df.shape)\n", "df = df[\n", " df[\"request_id\"].isin(\n", " df[\"request_id\"].value_counts().index[df[\"request_id\"].value_counts() == 2]\n", " )\n", "]\n", "print(\"after filter on request id pairs\", df.shape)\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "assert df.shape[0] == df[\"request_id\"].nunique() * 2" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T15:51:39.222717Z", "iopub.status.busy": "2025-11-21T15:51:39.222573Z", "iopub.status.idle": "2025-11-21T17:06:35.715188Z", "shell.execute_reply": "2025-11-21T17:06:35.714536Z", "shell.execute_reply.started": "2025-11-21T15:51:39.222702Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "unique_requests 10051160\n", "before removing duplicates (20102320, 218)\n", "after removing duplicates (20102320, 211)\n" ] } ], "source": [ "# Let's use the old selection for now -- for quality assurance\n", "# expand the metadata columns -- this takes forever...~ 6 mins\n", "# test_slice = df[\"metadata\"].apply(lambda x: ast.literal_eval(str(x)))\n", "# test_slice = df[\"metadata\"].apply(lambda x: custom_parse(x))\n", "test_slice = df[\"metadata\"] # .apply(lambda x: custom_parse(x))\n", "test_slice_series = test_slice.apply(pd.Series)\n", "df = pd.concat([df, test_slice_series], axis=1, join=\"inner\")\n", "print(\"unique_requests\", df[\"request_id\"].nunique())\n", "# remove the duplicates\n", "print(\"before removing duplicates\", df.shape)\n", "df = df.loc[:, ~df.columns.duplicated()].copy()\n", "print(\"after removing duplicates\", df.shape)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T17:06:35.716117Z", "iopub.status.busy": "2025-11-21T17:06:35.715959Z", "iopub.status.idle": "2025-11-21T17:08:25.435120Z", "shell.execute_reply": "2025-11-21T17:08:25.433466Z", "shell.execute_reply.started": "2025-11-21T17:06:35.716100Z" } }, "outputs": [ { "data": { "text/plain": [ "pos_diff_preference\n", "1.0 7090943\n", "2.0 2960208\n", "0.0 9\n", "Name: count, dtype: int64" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = df.sort_values(by=[\"request_id\", \"preference\", \"diff_preference\"])\n", "df[\"pos_diff_preference\"] = df[\"diff_preference\"].diff()\n", "# df[\"cer_diff_preference\"] = df[\"cer\"].diff()\n", "df[df[\"preference\"]][\"pos_diff_preference\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T17:08:25.436444Z", "iopub.status.busy": "2025-11-21T17:08:25.436264Z", "iopub.status.idle": "2025-11-21T17:09:01.418150Z", "shell.execute_reply": "2025-11-21T17:09:01.417201Z", "shell.execute_reply.started": "2025-11-21T17:08:25.436429Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive param_experiment\n", "mask_control_slider 573883\n", "cfg_steps_10 501742\n", "text_cfg_15 484449\n", "text_cfg_12 481627\n", "cfg_steps_30 335203\n", "cfg_steps_480 171261\n", "max_tag_len_1024 123186\n", "def_audio_text_cfg_05 39247\n", "def_audio_cfg_05 38927\n", "tag_cfg_05 16187\n", "tag_cfg_20 10379\n", "temp_s_85 9399\n", "temp_s_95 9353\n", "temp_s_100 9349\n", "temp_s_80 9231\n", "diff_text_cfg_25 6116\n", "diff_text_cfg_15 5871\n", "n_tag_3 5046\n", "text_cfg_20 1228\n", "Name: count, dtype: int64\n" ] } ], "source": [ "try:\n", " print(\"positive\", df[df[\"preference\"]][\"param_experiment\"].value_counts())\n", "except:\n", " pass" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T17:09:01.420447Z", "iopub.status.busy": "2025-11-21T17:09:01.419122Z", "iopub.status.idle": "2025-11-21T17:15:10.366568Z", "shell.execute_reply": "2025-11-21T17:15:10.365984Z", "shell.execute_reply.started": "2025-11-21T17:09:01.420398Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Found 1262917 duplicated prompts 631460 unique requests\n", "Found 337951 request_ids with duplicate prompts but not highest play counts in their group\n", "['008f4d3a-71f6-42b5-8e86-62e28560492e', '6c5f074f-ef28-4f77-9f8f-88870b41f0f3', '92c2c907-9c77-4040-9452-76638133688d', '293a5d2a-54f9-4c38-a265-6ca531cb7fb9', '1b4804f4-5ae7-4771-b155-7565ee5aef84', '2fc0a3c9-5b91-4c26-aad2-53046d62a65e', 'e1bed84f-9d4e-40d9-aa23-78be7f355f1c', '0acf422a-deab-4792-b0fd-180920cfd5eb', '0adde2d1-5cee-4877-a59f-9ce311e46132', '0bfc985d-734f-41bb-8670-187e1d7daa23']\n", "Before dedup user gen requests 20102320\n", "After dedup user gen requests 20102320\n" ] } ], "source": [ "# Find duplicated prompts with count > 2\n", "duplicate_entries = df.groupby(\n", " [\"user_id\", \"prompt_text\", \"tags\", \"task\", \"edited_clip_id\"]\n", ").filter(lambda x: len(x) > 2)\n", "print(\n", " \"Found\",\n", " len(duplicate_entries),\n", " \"duplicated prompts\",\n", " len(duplicate_entries[\"request_id\"].unique()),\n", " \"unique requests\",\n", ")\n", "\n", "# Group by user_id, prompt_text, and tags to find duplicate prompt groups\n", "prompt_groups = duplicate_entries.groupby(\n", " [\"user_id\", \"prompt_text\", \"tags\", \"task\", \"edited_clip_id\"]\n", ")\n", "\n", "# For each prompt group, find the request_id with the highest total reaction_play_count\n", "low_play_count_request_ids = []\n", "for prompt_key, prompt_group in prompt_groups:\n", " # Get the sum of reaction_play_count for each request_id in this group\n", " request_play_counts = prompt_group.groupby(\"request_id\")[\n", " \"reaction_play_count\"\n", " ].sum()\n", "\n", " # Find the max play count in this group\n", " max_play_count = request_play_counts.max()\n", "\n", " # Add request_ids that don't have the max play count to our filter list\n", " lower_play_count_request_ids = request_play_counts[\n", " request_play_counts < max_play_count\n", " ].index.tolist()\n", " low_play_count_request_ids.extend(lower_play_count_request_ids)\n", "\n", "# Display the filtered request IDs\n", "print(\n", " f\"Found {len(low_play_count_request_ids)} request_ids with duplicate prompts but not highest play counts in their group\"\n", ")\n", "print(\n", " low_play_count_request_ids[:10]\n", " if len(low_play_count_request_ids) > 10\n", " else low_play_count_request_ids\n", ")\n", "print(\"Before dedup user gen requests\", df.shape[0])\n", "# df = df[~df[\"request_id\"].isin(low_play_count_request_ids)]\n", "print(\"After dedup user gen requests\", df.shape[0])" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:40.799375Z", "start_time": "2024-05-16T13:59:36.394236Z" }, "execution": { "iopub.execute_input": "2025-11-21T17:15:10.367367Z", "iopub.status.busy": "2025-11-21T17:15:10.367219Z", "iopub.status.idle": "2025-11-21T17:19:50.829691Z", "shell.execute_reply": "2025-11-21T17:19:50.828913Z", "shell.execute_reply.started": "2025-11-21T17:15:10.367352Z" }, "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "193791\n", "good_continue_at\n", "True 20072376\n", "False 29944\n", "Name: count, dtype: int64\n", "\n", " Check some basics... \n", " preference\n", "False 10051160\n", "True 10051160\n", "Name: count, dtype: int64 model_name\n", "chirp-crow-t1 20060670\n", "chirp-crow-t2 41650\n", "Name: count, dtype: int64 preference model_name \n", "False chirp-crow-t1 10030335\n", " chirp-crow-t2 20825\n", "True chirp-crow-t1 10030335\n", " chirp-crow-t2 20825\n", "Name: count, dtype: int64\n", "task\n", " 11857524\n", "cover 5410490\n", "artist_consistency 1168610\n", "artist_cover 727886\n", "extend 254606\n", "playlist_condition 222672\n", "overpainting 115526\n", "upload_extend 91102\n", "underpainting 89472\n", "infill 73632\n", "artist_extend 49146\n", "stem_condition 33314\n", "stem_condition_infill 8336\n", "generate 4\n", "Name: count, dtype: int64\n" ] } ], "source": [ "df[\"id\"] = df[\"str_id\"]\n", "# get the original duration of the clips, if they are concacted\n", "df[\"original_duration_s\"] = df[\"total_start_s\"] + df[\"duration\"]\n", "# classify the continue at behavoirs by the duration choice\n", "audio_prompt_id_to_continue_at = {}\n", "\n", "for _, row in df[~df[\"continued_parent\"].isna()].iterrows():\n", " audio_prompt_id = row[\"continued_parent\"]\n", " if audio_prompt_id not in audio_prompt_id_to_continue_at:\n", " audio_prompt_id_to_continue_at[audio_prompt_id] = row[\"continue_at\"]\n", " else:\n", " # pick the max\n", " audio_prompt_id = max(\n", " audio_prompt_id_to_continue_at[audio_prompt_id], row[\"continue_at\"]\n", " )\n", "print(len(audio_prompt_id_to_continue_at))\n", "df[\"has_continue_and_start_continue_at\"] = df[\"id\"].apply(\n", " lambda x: audio_prompt_id_to_continue_at.get(x)\n", ")\n", "# we want continue at to be at most of the clip...\n", "df[\"good_continue_at\"] = (\n", " (df[\"has_continue_and_start_continue_at\"] / df[\"duration\"]) > 0.9\n", ") | df[\"has_continue_and_start_continue_at\"].isna()\n", "print(df[\"good_continue_at\"].value_counts())\n", "\n", "\n", "print(\n", " \"\\n Check some basics... \\n\",\n", " df[\"preference\"].value_counts(),\n", " df[\"model_name\"].value_counts(),\n", " df.groupby([\"preference\"])[\"model_name\"].value_counts(),\n", ")\n", "\n", "df = df.sort_values(by=[\"request_id\", \"preference\"])\n", "df[\"duration_rel_diff\"] = df[\"duration\"].diff()\n", "df[\"play_rel_diff\"] = df[\"reaction_play_count\"].diff()\n", "print(df[\"task\"].value_counts())\n", "\n", "df[\"post_infill_duration\"] = (\n", " df[\"duration\"]\n", " + df[\"infill_context_end_s\"]\n", " - df[\"infill_context_start_s\"]\n", " - df[\"include_future_s\"]\n", " - df[\"include_history_s\"]\n", " - df[\"infill_dur_s\"]\n", ")" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T17:19:50.831150Z", "iopub.status.busy": "2025-11-21T17:19:50.830475Z", "iopub.status.idle": "2025-11-21T17:20:49.996119Z", "shell.execute_reply": "2025-11-21T17:20:49.995530Z", "shell.execute_reply.started": "2025-11-21T17:19:50.831130Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive preference_score\n", "1 5384051\n", "2 2726299\n", "0 1357860\n", "3 546048\n", "4 34797\n", "5 2105\n", "Name: count, dtype: int64\n", "negative preference_score\n", "0 9848390\n", "1 185624\n", "2 16047\n", "3 1062\n", "4 34\n", "5 3\n", "Name: count, dtype: int64\n" ] } ], "source": [ "df[\"preference_score\"] = (\n", " df[\"upvoted\"].astype(int)\n", " + df[\"has_action\"].astype(int)\n", " + df[\"part_of_concat\"].astype(int)\n", " + df[\"is_in_playlist\"].astype(int)\n", " + (df[\"n_edits\"] >= 10).astype(int)\n", ")\n", "print(\"positive\", df[df[\"preference\"]][\"preference_score\"].value_counts())\n", "print(\"negative\", df[~df[\"preference\"]][\"preference_score\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.035167Z", "start_time": "2024-05-16T13:59:40.801098Z" }, "execution": { "iopub.execute_input": "2025-11-21T17:20:49.997041Z", "iopub.status.busy": "2025-11-21T17:20:49.996888Z", "iopub.status.idle": "2025-11-21T17:23:01.546453Z", "shell.execute_reply": "2025-11-21T17:23:01.545655Z", "shell.execute_reply.started": "2025-11-21T17:20:49.997026Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "after duration 0.9848332928736584\n", "after infill duration 0.9999429419091925\n", "neg_filter_reaction_play_count 1.0\n", "neg_filter_upvote_count 0.9895\n", "neg_filter_norm_play_frac 1.0\n", "neg_filter_continues 0.9999\n", "neg_filter_preference_score 0.9798\n", "----------------\n", "pos_filter_continues 0.9971\n", "pos_filter_reaction_play_count 1.0\n", "pos_filter_relative_play_count 0.9708\n", "pos_filter_cer_diff_preference 1.0\n", "pos_filter_bad_flags 0.9998\n", "after filter on play counts 0.9878\n", "after filter on higher quality 0.3094\n", "pos_filter_preference_score 0.8649\n", "----------------\n", "negative 9682109 positive 1887585\n", "----------------\n", "total pair requests 10051160 --> selected pair requests 1850889 frac 0.184 --> total intitial users 817271\n" ] } ], "source": [ "normal_pos_play_count = 3\n", "# this is lower, cause a concat is probably already ensuring that it is good\n", "concat_pos_play_count = 1\n", "# this is a filter on the concated clip\n", "concat_total_play_count = 3\n", "\n", "all_fitlers = (df[\"duration\"] >= 10) & (df[\"duration\"] <= 480) & (df[\"clip_type\"] != \"preview\")\n", "print(\"after duration\", all_fitlers.sum() / df.shape[0])\n", "infill_duration_filter = ~df[\"task\"].isin(\n", " [\n", " \"infill\",\n", " \"infill_intro\",\n", " \"infill_outro\",\n", " ]\n", ") | (df[\"post_infill_duration\"] <= 239)\n", "print(\"after infill duration\", infill_duration_filter.sum() / df.shape[0])\n", "# negative fitlers\n", "total_negative = df[~df[\"preference\"]].shape[0]\n", "neg_filter_reaction_play_count = (~df[\"preference\"]) & (df[\"reaction_play_count\"] >= 1)\n", "print(\n", " \"neg_filter_reaction_play_count\",\n", " round(neg_filter_reaction_play_count.sum() / total_negative, 4),\n", ")\n", "neg_filter_upvote_count = (~df[\"preference\"]) & (df[\"upvote_count\"] == 0)\n", "print(\n", " \"neg_filter_upvote_count\",\n", " round(neg_filter_upvote_count.sum() / total_negative, 4),\n", ")\n", "neg_filter_norm_play_frac = (~df[\"preference\"]) & (df[\"norm_play_frac\"] <= 3.1)\n", "print(\n", " \"neg_filter_norm_play_frac\",\n", " round(neg_filter_norm_play_frac.sum() / total_negative, 4),\n", ")\n", "neg_filter_continues = (~df[\"preference\"]) & (\n", " df[\"has_continue_and_start_continue_at\"].isna()\n", ")\n", "print(\n", " \"neg_filter_continues\",\n", " round(neg_filter_continues.sum() / total_negative, 4),\n", ")\n", "neg_filter_preference_score = (~df[\"preference\"]) & (\n", " df[\"preference_score\"] == 0\n", ")\n", "print(\n", " \"neg_filter_preference_score\",\n", " round(neg_filter_preference_score.sum() / total_negative, 4),\n", ")\n", "\n", "neg_filter_selection_mask = (\n", " all_fitlers\n", " & infill_duration_filter\n", " & neg_filter_reaction_play_count\n", " & neg_filter_upvote_count\n", " & neg_filter_norm_play_frac\n", " & neg_filter_continues\n", " & neg_filter_preference_score\n", ")\n", "\n", "print(\"----------------\")\n", "total_positive = df[df[\"preference\"]].shape[0]\n", "assert total_positive == total_negative\n", "pos_filter_continues = (df[\"preference\"]) & (df[\"good_continue_at\"])\n", "print(\"pos_filter_continues\", round(pos_filter_continues.sum() / total_positive, 4))\n", "pos_filter_reaction_play_count = (df[\"preference\"]) & (df[\"reaction_play_count\"] >= 1)\n", "print(\n", " \"pos_filter_reaction_play_count\",\n", " round(pos_filter_reaction_play_count.sum() / total_positive, 4),\n", ")\n", "pos_filter_relative_play_count = (df[\"preference\"]) & (df[\"play_rel_diff\"] >= 0)\n", "print(\n", " \"pos_filter_relative_play_count\",\n", " round(pos_filter_relative_play_count.sum() / total_positive, 4),\n", ")\n", "pos_filter_cer_diff_preference = (\n", " df[\n", " \"preference\"\n", " ] # & (df[\"pos_diff_preference\"] == 2) # & (df[\"cer_diff_preference\"] < 0.5) & (df[\"cer\"] < 0.99)\n", ")\n", "print(\n", " \"pos_filter_cer_diff_preference\",\n", " round(pos_filter_cer_diff_preference.sum() / total_positive, 4),\n", ")\n", "pos_filter_bad_flags = (\n", " (df[\"preference\"]) & (df[\"flag_count\"] == 0) & (df[\"dislike_count\"] == 0)\n", ")\n", "print(\n", " \"pos_filter_bad_flags\",\n", " round(pos_filter_bad_flags.sum() / total_positive, 4),\n", ")\n", "pos_filter_play_counts = (df[\"preference\"]) & (\n", " (\n", " (df[\"part_of_concat\"])\n", " & (df[\"reaction_play_count\"] >= concat_pos_play_count)\n", " & (df[\"concat_play_counts\"] >= concat_total_play_count)\n", " )\n", " | (\n", " (~df[\"part_of_concat\"]) & (df[\"reaction_play_count\"] >= normal_pos_play_count)\n", " # & (df[\"norm_play_frac\"] >= 2.1) # this is a bit of a luxury cut...\n", " )\n", " | (df[\"task\"].isin([\"infill\", \"infill_intro\", \"infill_outro\"]))\n", ")\n", "print(\n", " \"after filter on play counts\",\n", " round(pos_filter_play_counts.sum() / total_positive, 4),\n", ")\n", "high_quality_tasks_filter = (\n", " (\n", " df[\"task\"].isin(\n", " [\n", " \"cover\",\n", " \"upload_extend\",\n", " \"cover_extend\",\n", " \"artist_cover\",\n", " \"extend\",\n", " \"artist_consistency\",\n", " \"artist_extend\",\n", " \"playlist_condition\",\n", " \"overpainting\",\n", " \"underpainting\",\n", " \"\",\n", " ]\n", " )\n", " )\n", " # & (\n", " # (df[\"upvote_count\"] >= 1) # (df[\"upvote_count\"] >= 1)\n", " # | (df[\"reaction_play_count\"] >= 5)\n", " # | (df[\"concat_play_counts\"] >= 5)\n", " # )\n", " & (\n", " (df[\"part_of_concat\"])\n", " | (\n", " (~df[\"part_of_concat\"])\n", " & (df[\"norm_play_frac\"] >= 5.1) # this is a bit of a luxury cut...\n", " & (\n", " df[\"norm_play_frac\"] >= df[\"reaction_play_count\"] / 3\n", " ) # play duration is not low on average -- huh only small difference with this cut\n", " )\n", " )\n", ")\n", "medium_quality_tasks_filter = (\n", " df[\"task\"].isin(\n", " [\n", " \"infill\",\n", " \"infill_intro\",\n", " \"infill_outro\",\n", " \"stem_condition\",\n", " ]\n", " )\n", ") & ( # let more infill through only in this case...\n", " (\n", " df[\"upvote_count\"] >= 1\n", " ) # (df[\"upvote_count\"] >= 1) (df[\"pos_diff_preference\"] == 2)\n", " | (df[\"reaction_play_count\"] >= 1)\n", " | (df[\"concat_play_counts\"] >= 1)\n", ")\n", "pos_filter_higher_quality = (df[\"preference\"]) & (\n", " high_quality_tasks_filter | medium_quality_tasks_filter\n", ")\n", "print(\n", " \"after filter on higher quality\",\n", " round(pos_filter_higher_quality.sum() / total_positive, 4),\n", ")\n", "\n", "user_gen_filter = (\n", " df[\"user_n_clips\"] >= 100\n", ") # user needs to have genereated at least 100 over the time period\n", "\n", "pos_filter_preference_score = (df[\"preference\"]) & (\n", " df[\"preference_score\"] >= 1\n", ")\n", "print(\n", " \"pos_filter_preference_score\",\n", " round(pos_filter_preference_score.sum() / total_positive, 4),\n", ")\n", "\n", "print(\"----------------\")\n", "pos_filter_selectin_mask = (\n", " (df[\"preference\"]) # get basics aligned\n", " & all_fitlers\n", " & infill_duration_filter\n", " & pos_filter_continues\n", " & pos_filter_reaction_play_count\n", " & pos_filter_relative_play_count\n", " & pos_filter_cer_diff_preference\n", " & pos_filter_bad_flags\n", " & pos_filter_play_counts\n", " & pos_filter_higher_quality\n", " & user_gen_filter\n", " & pos_filter_preference_score\n", ")\n", "print(\n", " \"negative\",\n", " sum(neg_filter_selection_mask),\n", " \"positive\",\n", " sum(pos_filter_selectin_mask),\n", ")\n", "\n", "neg_filter_requests = df[neg_filter_selection_mask][\"request_id\"].unique()\n", "pos_filter_requests = df[pos_filter_selectin_mask][\"request_id\"].unique()\n", "# looking for very strong signal here:\n", "# listen to the positive/negative more than once\n", "# disliked one of the clips\n", "unique_requests = set(pos_filter_requests).intersection(neg_filter_requests)\n", "print(\"----------------\")\n", "print(\n", " \"total pair requests\",\n", " df[\"request_id\"].nunique(),\n", " \" --> selected pair requests\",\n", " len(unique_requests),\n", " f\"frac {len(unique_requests) / df['request_id'].nunique():.3f}\",\n", " \" --> total intitial users\",\n", " df[\"user_id\"].nunique(),\n", ")" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T17:23:01.547430Z", "iopub.status.busy": "2025-11-21T17:23:01.547262Z", "iopub.status.idle": "2025-11-21T17:23:01.575071Z", "shell.execute_reply": "2025-11-21T17:23:01.574469Z", "shell.execute_reply.started": "2025-11-21T17:23:01.547413Z" } }, "outputs": [], "source": [ "# total pair requests 3409842 --> selected pair requests 383914 frac 0.113 --> total intitial users 355873\n", "# total pair requests 3669633 --> selected pair requests 902413 frac 0.246 --> total intitial users 376949\n", "# total pair requests 3574058 --> selected pair requests 955401 frac 0.267 --> total intitial users 394204" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.250737Z", "start_time": "2024-05-16T13:59:41.036434Z" }, "execution": { "iopub.execute_input": "2025-11-21T17:23:01.683558Z", "iopub.status.busy": "2025-11-21T17:23:01.683058Z", "iopub.status.idle": "2025-11-21T17:23:36.216323Z", "shell.execute_reply": "2025-11-21T17:23:36.215534Z", "shell.execute_reply.started": "2025-11-21T17:23:01.683538Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "task\n", " 2088274\n", "cover 938470\n", "artist_consistency 249056\n", "artist_cover 146550\n", "extend 99052\n", "infill 48724\n", "playlist_condition 43472\n", "stem_condition 24170\n", "artist_extend 19438\n", "overpainting 18060\n", "upload_extend 15194\n", "underpainting 11318\n", "Name: count, dtype: int64\n", "crow_t1_v61 requests 1850889 clips 3701778 total khrs 186.587; N gpus for 1000 iters 231.361; 4 gpus for x iters 57840.281; n unique users 304941 n pro users 300670\n" ] } ], "source": [ "df_slice = df[df[\"request_id\"].isin(set(unique_requests))].copy()\n", "print(df_slice[\"task\"].value_counts())\n", "print(\n", " f\"{os.path.basename(OUT_DATA_DIR)} requests\",\n", " df_slice[\"request_id\"].nunique(),\n", " \"clips\",\n", " df_slice.shape[0],\n", " f\"total khrs {sum(df_slice['duration'] / 3600 / 1000):.3f};\",\n", " f\"N gpus for 1000 iters {df_slice.shape[0] / 8 / 2 / 1000:.3f};\",\n", " f\"4 gpus for x iters {df_slice.shape[0] / 8 / 2 / 4:.3f};\",\n", " f\"n unique users {df_slice['user_id'].nunique()}\",\n", " f\"n pro users {df_slice[df_slice['is_pro_user']]['user_id'].nunique()}\",\n", ")\n", "# auk_mix_t1_v2 requests 102002 clips 204004 total khrs 9.191; N gpus for 1000 iters 12.750; 4 gpus for x iters 3187.562; n unique users 36408 n pro users 34038\n", "# auk_t1_v1 requests 9179 clips 18358 total khrs 0.854; N gpus for 1000 iters 1.147; 4 gpus for x iters 286.844; n unique users 6288 n pro users 6275\n", "# auk_t1_v2 requests 40903 clips 81806 total khrs 3.864; N gpus for 1000 iters 5.113; 4 gpus for x iters 1278.219; n unique users 21079 n pro users 20966\n", "# auk_t1_v3 requests 102015 clips 204030 total khrs 9.703; N gpus for 1000 iters 12.752; 4 gpus for x iters 3187.969; n unique users 42837 n pro users 42462\n", "# auk_t1_v4 requests 211452 clips 422904 total khrs 20.216; N gpus for 1000 iters 26.431; 4 gpus for x iters 6607.875; n unique users 71182 n pro users 70059\n", "# auk_t1_v9 requests 547743 clips 1095486 total khrs 56.567; N gpus for 1000 iters 68.468; 4 gpus for x iters 17116.969; n unique users 129354 n pro users 125126\n", "# auk_t1_v13 requests 627646 clips 1255292 total khrs 64.530; N gpus for 1000 iters 78.456; 4 gpus for x iters 19613.938; n unique users 139873 n pro users 134738\n", "# auk_t1_v17 requests 494902 clips 989804 total khrs 51.445; N gpus for 1000 iters 61.863; 4 gpus for x iters 15465.688; n unique users 114240 n pro users 109708\n", "# auk_t1_v19 requests 740881 clips 1481762 total khrs 76.133; N gpus for 1000 iters 92.610; 4 gpus for x iters 23152.531; n unique users 154570 n pro users 147261\n", "# auk_t1_v24 requests 717745 clips 1435490 total khrs 74.515; N gpus for 1000 iters 89.718; 4 gpus for x iters 22429.531; n unique users 139344 n pro users 130740\n", "# auk_t1_v29 requests 1097586 clips 2195172 total khrs 112.614; N gpus for 1000 iters 137.198; 4 gpus for x iters 34299.562; n unique users 193138 n pro users 179429\n", "# auk_t1_v30 requests 1224422 clips 2448844 total khrs 125.330; N gpus for 1000 iters 153.053; 4 gpus for x iters 38263.188; n unique users 203138 n pro users 186758\n", "# auk_t1_v31 requests 389265 clips 778530 total khrs 40.761; N gpus for 1000 iters 48.658; 4 gpus for x iters 12164.531; n unique users 85198 n pro users 78445\n", "# auk_t1_v33 requests 1285261 clips 2570522 total khrs 131.585; N gpus for 1000 iters 160.658; 4 gpus for x iters 40164.406; n unique users 208932 n pro users 191380\n", "# auk_t1_v33 requests 1086639 clips 2173278 total khrs 110.641; N gpus for 1000 iters 135.830; 4 gpus for x iters 33957.469; n unique users 197293 n pro users 180968 -- play dur from /3 to /2\n", "# auk_t1_v33 requests 806877 clips 1613754 total khrs 81.964; N gpus for 1000 iters 100.860; 4 gpus for x iters 25214.906; n unique users 156679 n pro users 144253 -- filter to web\n", "# auk_t1_v37 requests 1339132 clips 2678264 total khrs 137.055; N gpus for 1000 iters 167.392; 4 gpus for x iters 41847.875; n unique users 214313 n pro users 196815\n", "# auk_t1_v38 requests 979451 clips 1958902 total khrs 102.038; N gpus for 1000 iters 122.431; 4 gpus for x iters 30607.844; n unique users 170773 n pro users 156737\n", "# auk_t1_v43 requests 184775 clips 369550 total khrs 19.089; N gpus for 1000 iters 23.097; 4 gpus for x iters 5774.219; n unique users 60652 n pro users 59126\n", "# auk_t1_v45 requests 1176216 clips 2352432 total khrs 122.102; N gpus for 1000 iters 147.027; 4 gpus for x iters 36756.750; n unique users 176751 n pro users 163097\n", "# auk_t1_v48 requests 294407 clips 588814 total khrs 29.992; N gpus for 1000 iters 36.801; 4 gpus for x iters 9200.219; n unique users 80991 n pro users 78825\n", "# bluejay_t1_v1 requests 47405 clips 94810 total khrs 5.515; N gpus for 1000 iters 5.926; 4 gpus for x iters 1481.406; n unique users 24344 n pro users 24194\n", "# bluejay_t1_v2 requests 101265 clips 202530 total khrs 11.753; N gpus for 1000 iters 12.658; 4 gpus for x iters 3164.531; n unique users 44574 n pro users 44024\n", "# bluejay_t1_v3 requests 144765 clips 289530 total khrs 16.756; N gpus for 1000 iters 18.096; 4 gpus for x iters 4523.906; n unique users 58421 n pro users 57449\n", "# bluejay_t1_v5 requests 179289 clips 358578 total khrs 20.764; N gpus for 1000 iters 22.411; 4 gpus for x iters 5602.781; n unique users 67850 n pro users 66572\n", "# bluejay_t1_v7 requests 282305 clips 564610 total khrs 32.692; N gpus for 1000 iters 35.288; 4 gpus for x iters 8822.031; n unique users 93023 n pro users 90936\n", "# bluejay_t1_v9 requests 540570 clips 1081140 total khrs 62.455; N gpus for 1000 iters 67.571; 4 gpus for x iters 16892.812; n unique users 145720 n pro users 142027\n", "# bluejay_t1_v11 requests 184433 clips 368866 total khrs 20.955; N gpus for 1000 iters 23.054; 4 gpus for x iters 5763.531; n unique users 52149 n pro users 51402\n", "# bluejay_t1_v12 requests 293821 clips 587642 total khrs 33.820; N gpus for 1000 iters 36.728; 4 gpus for x iters 9181.906; n unique users 98138 n pro users 95838\n", "# bluejay_t1_v13 requests 355678 clips 711356 total khrs 40.908; N gpus for 1000 iters 44.460; 4 gpus for x iters 11114.938; n unique users 111544 n pro users 108497\n", "# bluejay_t1_v17 requests 415411 clips 830822 total khrs 48.043; N gpus for 1000 iters 51.926; 4 gpus for x iters 12981.594; n unique users 124572 n pro users 120683\n", "# bluejay_t1_v24 requests 486884 clips 973768 total khrs 56.292; N gpus for 1000 iters 60.861; 4 gpus for x iters 15215.125; n unique users 138727 n pro users 133829\n", "# bluejay_t1_v26 requests 336220 clips 672440 total khrs 38.800; N gpus for 1000 iters 42.028; 4 gpus for x iters 10506.875; n unique users 80832 n pro users 78854\n", "# bluejay_t1_v28 requests 683709 clips 1367418 total khrs 78.507; N gpus for 1000 iters 85.464; 4 gpus for x iters 21365.906; n unique users 172776 n pro users 164584\n", "# bluejay_t1_v28 requests 662650 clips 1325300 total khrs 76.539; N gpus for 1000 iters 82.831; 4 gpus for x iters 20707.812; n unique users 170192 n pro users 162222\n", "# bluejay_t1_v31 requests 855246 clips 1710492 total khrs 97.846; N gpus for 1000 iters 106.906; 4 gpus for x iters 26726.438; n unique users 198438 n pro users 186068\n", "# bluejay_t1_v31 requests 829339 clips 1658678 total khrs 95.430; N gpus for 1000 iters 103.667; 4 gpus for x iters 25916.844; n unique users 195575 n pro users 183523\n", "# bluejay_t1_v31 requests 932058 clips 1864116 total khrs 107.136; N gpus for 1000 iters 116.507; 4 gpus for x iters 29126.812; n unique users 203738 n pro users 190639\n", "# bluejay_t1_v41 requests 1122075 clips 2244150 total khrs 128.936; N gpus for 1000 iters 140.259; 4 gpus for x iters 35064.844; n unique users 232511 n pro users 214225\n", "# bluejay_t1_v42 requests 1234273 clips 2468546 total khrs 140.767; N gpus for 1000 iters 154.284; 4 gpus for x iters 38571.031; n unique users 246967 n pro users 226020\n", "# crow_t1_v1 requests 131955 clips 263910 total khrs 12.518; N gpus for 1000 iters 16.494; 4 gpus for x iters 4123.594; n unique users 61682 n pro users 56836\n", "# crow_t1_v3 requests 272342 clips 544684 total khrs 26.003; N gpus for 1000 iters 34.043; 4 gpus for x iters 8510.688; n unique users 106845 n pro users 96483\n", "# crow_t1_v6 requests 369774 clips 739548 total khrs 35.356; N gpus for 1000 iters 46.222; 4 gpus for x iters 11555.438; n unique users 134281 n pro users 119919\n", "# crow_t1_v13 requests 558885 clips 1117770 total khrs 53.496; N gpus for 1000 iters 69.861; 4 gpus for x iters 17465.156; n unique users 182267 n pro users 160132\n", "# crow_t1_v17 requests 792420 clips 1584840 total khrs 77.740; N gpus for 1000 iters 99.052; 4 gpus for x iters 24763.125; n unique users 212279 n pro users 199173\n", "# crow_t1_v21 requests 1078035 clips 2156070 total khrs 106.094; N gpus for 1000 iters 134.754; 4 gpus for x iters 33688.594; n unique users 263938 n pro users 245247\n", "# crow_t1_v31 requests 1569448 clips 3138896 total khrs 155.285; N gpus for 1000 iters 196.181; 4 gpus for x iters 49045.250; n unique users 343002 n pro users 310320\n", "# crow_t1_v31 requests 1128871 clips 2257742 total khrs 113.348; N gpus for 1000 iters 141.109; 4 gpus for x iters 35277.219; n unique users 198445 n pro users 192068\n", "# crow_t1_vxx requests 1275697 clips 2551394 total khrs 128.287; N gpus for 1000 iters 159.462; 4 gpus for x iters 39865.531; n unique users 216731 n pro users 207174\n", "# crow_t1_v61 requests 902413 clips 1804826 total khrs 90.607; N gpus for 1000 iters 112.802; 4 gpus for x iters 28200.406; n unique users 187576 n pro users 180076\n", "# crow_t1_v61 requests 955401 clips 1910802 total khrs 95.996; N gpus for 1000 iters 119.425; 4 gpus for x iters 29856.281; n unique users 199274 n pro users 190340\n", "# crow_t1_v61 requests 2047710 clips 4095420 total khrs 205.935; N gpus for 1000 iters 255.964; 4 gpus for x iters 63990.938; n unique users 311228 n pro users 306971\n", "# crow_t1_v61 requests 1850868 clips 3701736 total khrs 186.586; N gpus for 1000 iters 231.358; 4 gpus for x iters 57839.625; n unique users 304940 n pro users 300669" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T17:23:36.217341Z", "iopub.status.busy": "2025-11-21T17:23:36.217052Z", "iopub.status.idle": "2025-11-21T17:23:36.235580Z", "shell.execute_reply": "2025-11-21T17:23:36.235009Z", "shell.execute_reply.started": "2025-11-21T17:23:36.217323Z" } }, "outputs": [], "source": [ "# import time\n", "# time.sleep(3600)" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T17:23:36.236437Z", "iopub.status.busy": "2025-11-21T17:23:36.236283Z", "iopub.status.idle": "2025-11-21T17:31:51.934090Z", "shell.execute_reply": "2025-11-21T17:31:51.933405Z", "shell.execute_reply.started": "2025-11-21T17:23:36.236422Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Total hoot cer scores: 5714109\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████| 181129/181129 [07:59<00:00, 377.41it/s]\n" ] } ], "source": [ "# Load existing hoot CER cache\n", "with open(\"/home/tony/Data/Preference/crow_t1/hoot_cer.json\", \"r\") as file:\n", " clip_id_to_cer = json.load(file)\n", "print(\"Total hoot cer scores:\", len(clip_id_to_cer))\n", "# clip_id_to_cer = {}\n", "JSON_DIR = \"/app2/suno/data/dpo/crow_t1_json/\"\n", "\n", "# Extract unique s3_ids from df_slice that are not already in the cache\n", "clip_ids = set(df_slice[\"s3_id\"]) - set(clip_id_to_cer.keys())\n", "\n", "for clip_id in tqdm(clip_ids):\n", " hoot_json_path = os.path.join(JSON_DIR, f\"{clip_id}_hoot.json\")\n", " if not os.path.exists(hoot_json_path):\n", " clip_id_to_cer[clip_id] = 1.0\n", " continue\n", " with open(hoot_json_path, \"r\") as f:\n", " data = json.load(f)\n", " for data_dict in data:\n", " if \"hoot_cer\" in data_dict:\n", " clip_id_to_cer[clip_id] = data_dict[\"hoot_cer\"]\n", " break\n", "\n", "# Update the hoot CER cache\n", "with open(\"/home/tony/Data/Preference/crow_t1/hoot_cer.json\", \"w\") as file:\n", " json.dump(clip_id_to_cer, file)" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T17:31:51.934972Z", "iopub.status.busy": "2025-11-21T17:31:51.934805Z", "iopub.status.idle": "2025-11-21T17:34:22.694098Z", "shell.execute_reply": "2025-11-21T17:34:22.693367Z", "shell.execute_reply.started": "2025-11-21T17:31:51.934954Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "count 1.850889e+06\n", "mean -4.738277e-03\n", "std 8.706825e-02\n", "min -1.000000e+00\n", "10% -6.228374e-02\n", "50% 0.000000e+00\n", "90% 5.072464e-02\n", "95% 8.585227e-02\n", "96% 9.959623e-02\n", "97% 1.190476e-01\n", "98% 1.504077e-01\n", "99% 2.208640e-01\n", "99.5% 3.485360e-01\n", "99.9% 7.286512e-01\n", "max 1.000000e+00\n", "Name: cer_diff, dtype: float64\n", "crow_t1_v61 requests 1828863 clips 3657726 total khrs 184.821; N gpus for 1000 iters 228.608; 4 gpus for x iters 57151.969; n unique users 303693 n pro users 299436\n" ] } ], "source": [ "# # add the cer to the df\n", "df_slice[\"cer\"] = df_slice[\"s3_id\"].map(clip_id_to_cer)\n", "df_slice = df_slice.fillna({\"cer\": 1})\n", "df_slice[\"cer_diff\"] = df_slice[\"cer\"].diff()\n", "df_slice = df_slice.fillna({\"cer_diff\": 0})\n", "print(df_slice[df_slice[\"preference\"]][\"cer_diff\"].describe(percentiles=[0.1, 0.5, 0.9, 0.95, 0.96, 0.97, 0.98, 0.99, 0.995, 0.999]))\n", "\n", "# just trimming off tail is fine and safe. Used to be 0.2. Multiple rounds now so probably 0.3 is still fine.\n", "cer_mask = (df_slice[\"preference\"]) & (df_slice[\"cer_diff\"] < 0.2)\n", "pos_cer_filter_requests = df_slice[cer_mask][\"request_id\"].unique()\n", "df_slice = df_slice[df_slice[\"request_id\"].isin(set(pos_cer_filter_requests))].copy()\n", "print(\n", " f\"{os.path.basename(OUT_DATA_DIR)} requests\",\n", " df_slice[\"request_id\"].nunique(),\n", " \"clips\",\n", " df_slice.shape[0],\n", " f\"total khrs {sum(df_slice['duration'] / 3600 / 1000):.3f};\",\n", " f\"N gpus for 1000 iters {df_slice.shape[0] / 8 / 2 / 1000:.3f};\",\n", " f\"4 gpus for x iters {df_slice.shape[0] / 8 / 2 / 4:.3f};\",\n", " f\"n unique users {df_slice['user_id'].nunique()}\",\n", " f\"n pro users {df_slice[df_slice['is_pro_user']]['user_id'].nunique()}\",\n", ")" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T17:34:22.695037Z", "iopub.status.busy": "2025-11-21T17:34:22.694782Z", "iopub.status.idle": "2025-11-21T17:34:23.929028Z", "shell.execute_reply": "2025-11-21T17:34:23.928477Z", "shell.execute_reply.started": "2025-11-21T17:34:22.695019Z" } }, "outputs": [ { "data": { "text/plain": [ "source\n", "web 2716420\n", "ios 497880\n", "android 443424\n", "living-radio 2\n", "Name: count, dtype: int64" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_slice[\"source\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.277006Z", "start_time": "2024-05-16T13:59:41.252105Z" }, "execution": { "iopub.execute_input": "2025-11-21T17:34:23.929899Z", "iopub.status.busy": "2025-11-21T17:34:23.929739Z", "iopub.status.idle": "2025-11-21T17:34:26.587826Z", "shell.execute_reply": "2025-11-21T17:34:26.587226Z", "shell.execute_reply.started": "2025-11-21T17:34:23.929882Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive in playlist (584757, 221)\n", "task\n", " 2064300\n", "cover 929254\n", "artist_consistency 246808\n", "artist_cover 145034\n", "extend 96032\n", "infill 46942\n", "playlist_condition 42952\n", "stem_condition 24026\n", "artist_extend 18804\n", "overpainting 17600\n", "upload_extend 14818\n", "underpainting 11156\n", "Name: count, dtype: int64\n" ] } ], "source": [ "test_mask = (df_slice[\"preference\"]) & (\n", " (df_slice[\"is_in_playlist\"]) | (df_slice[\"concat_in_playlist\"])\n", ")\n", "print(\"positive in playlist\", df_slice[test_mask].shape)\n", "print(df_slice[\"task\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T17:34:26.588635Z", "iopub.status.busy": "2025-11-21T17:34:26.588476Z", "iopub.status.idle": "2025-11-21T17:34:26.619192Z", "shell.execute_reply": "2025-11-21T17:34:26.618663Z", "shell.execute_reply.started": "2025-11-21T17:34:26.588618Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "is_public\n", "False 3370607\n", "True 287119\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(df_slice[\"is_public\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T17:34:26.619888Z", "iopub.status.busy": "2025-11-21T17:34:26.619738Z", "iopub.status.idle": "2025-11-21T17:34:28.362433Z", "shell.execute_reply": "2025-11-21T17:34:28.361778Z", "shell.execute_reply.started": "2025-11-21T17:34:26.619872Z" } }, "outputs": [], "source": [ "df_slice[\"npz_path\"] = df_slice[\"s3_id\"].map(lambda x: f\"{NPZ_DIR}/{x}.npz\")" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T17:34:28.363257Z", "iopub.status.busy": "2025-11-21T17:34:28.363101Z", "iopub.status.idle": "2025-11-21T17:34:50.554367Z", "shell.execute_reply": "2025-11-21T17:34:50.553742Z", "shell.execute_reply.started": "2025-11-21T17:34:28.363241Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(3657726, 222)\n" ] } ], "source": [ "df_total = df_slice.copy()\n", "print(df_total.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## SOME SNOWFLAKE LYRICS SHIT YOU DON\"T WNAT TO KNOW" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T17:34:50.555161Z", "iopub.status.busy": "2025-11-21T17:34:50.555003Z", "iopub.status.idle": "2025-11-21T17:34:54.048349Z", "shell.execute_reply": "2025-11-21T17:34:54.047748Z", "shell.execute_reply.started": "2025-11-21T17:34:50.555144Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "PROD\n" ] } ], "source": [ "home_dir = os.path.expanduser(\"~\")\n", "snow_password_path = os.path.join(home_dir, \".aws\", \"snow_pw.txt\")\n", "if os.path.exists(snow_password_path):\n", " # !pip install snowflake\n", " from snowflake.core import Root\n", " from snowflake.snowpark import Session\n", "\n", " with open(snow_password_path, \"r\") as fp:\n", " fp_lines = fp.readlines()\n", " snow_password = fp_lines[0].strip()\n", " snow_username = fp_lines[1].strip()\n", "\n", " CONNECTION_PARAMETERS = {\n", " \"account\": \"fu90569.us-east-2.aws\",\n", " \"user\": snow_username,\n", " \"private_key_file\": \"/home/tony/.aws/rsa_key.p8\",\n", " \"role\": \"ACCOUNTADMIN\",\n", " \"database\": \"SUNO_PROD\",\n", " \"warehouse\": \"SUNO_PROD_LARGE\",\n", " \"schema\": \"PROD\",\n", " }\n", "\n", "if not os.path.exists(snow_password_path):\n", " raise Exception(\"you are not authorized to access snowflake -- please setup\")\n", "\n", "snow_session = Session.builder.configs(CONNECTION_PARAMETERS).create()\n", "\n", "snow_root = Root(snow_session)\n", "snow_schema = snow_root.databases[\"SUNO_PROD\"].schemas[\"PROD\"]\n", "print(snow_schema.name)" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T17:34:54.049447Z", "iopub.status.busy": "2025-11-21T17:34:54.049126Z", "iopub.status.idle": "2025-11-21T17:37:28.091856Z", "shell.execute_reply": "2025-11-21T17:37:28.091079Z", "shell.execute_reply.started": "2025-11-21T17:34:54.049428Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 0/1 [00:00= start_cutoff_date)\n", "print(df_total.shape, df_total[date_mask].shape)\n", "df_slice = df_total[date_mask].copy()\n", "print(\"after date cut\", df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 141, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T04:58:23.530467Z", "iopub.status.busy": "2025-11-22T04:58:23.530309Z", "iopub.status.idle": "2025-11-22T04:58:23.546368Z", "shell.execute_reply": "2025-11-22T04:58:23.545905Z", "shell.execute_reply.started": "2025-11-22T04:58:23.530451Z" } }, "outputs": [], "source": [ "# df_slice = df_total.copy()" ] }, { "cell_type": "code", "execution_count": 142, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T04:58:23.547712Z", "iopub.status.busy": "2025-11-22T04:58:23.547351Z", "iopub.status.idle": "2025-11-22T04:58:23.562266Z", "shell.execute_reply": "2025-11-22T04:58:23.561837Z", "shell.execute_reply.started": "2025-11-22T04:58:23.547696Z" } }, "outputs": [], "source": [ "# from tqdm import tqdm\n", "\n", "# list_of_past_data = [\n", "# \"/app2/suno/data/dpo/crow_t1_v51\",\n", "# \"/app2/suno/data/dpo/crow_t1_v52\",\n", "# \"/app2/suno/data/dpo/crow_t1_v53\",\n", "# \"/app2/suno/data/dpo/crow_t1_v54\",\n", "# \"/app2/suno/data/dpo/crow_t1_v55\",\n", "# \"/app2/suno/data/dpo/crow_t1_v56\",\n", "# \"/app2/suno/data/dpo/crow_t1_v57\", \n", "# ]\n", "\n", "# # Collect all known train IDs from previous meta_tr.jsonl files, showing progress and set growth\n", "# known_train_ids = set()\n", "# for data_dir in tqdm(list_of_past_data, desc=\"Collecting known train IDs\"):\n", "# metas = read_jsonl(os.path.join(data_dir, \"meta_tr.jsonl\"))\n", "# before = len(known_train_ids)\n", "# known_train_ids.update(meta[\"id\"] for meta in metas)\n", "# after = len(known_train_ids)\n", "# tqdm.write(f\"Added {after - before} new IDs from {data_dir} (total: {after})\")\n", "\n", "# print(f\"Total known train IDs: {len(known_train_ids)}\")\n", "# print(f\"Original df_slice shape: {df_slice.shape}\")\n", "# df_slice = df_slice[~df_slice[\"id\"].isin(known_train_ids)].copy()\n", "# print(f\"Filtered df_slice shape: {df_slice.shape}\")" ] }, { "cell_type": "code", "execution_count": 143, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T04:58:23.562919Z", "iopub.status.busy": "2025-11-22T04:58:23.562784Z", "iopub.status.idle": "2025-11-22T04:58:23.575211Z", "shell.execute_reply": "2025-11-22T04:58:23.574775Z", "shell.execute_reply.started": "2025-11-22T04:58:23.562905Z" } }, "outputs": [], "source": [ "# from suno_analytics.preference_data_selection import plot_clip_distribution\n", "# plot_clip_distribution(df)" ] }, { "cell_type": "code", "execution_count": 144, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T04:58:23.575886Z", "iopub.status.busy": "2025-11-22T04:58:23.575683Z", "iopub.status.idle": "2025-11-22T04:58:23.587636Z", "shell.execute_reply": "2025-11-22T04:58:23.587213Z", "shell.execute_reply.started": "2025-11-22T04:58:23.575872Z" } }, "outputs": [], "source": [ "# from suno_analytics.preference_data_selection import plot_clip_distribution\n", "# plot_clip_distribution(df_total)" ] }, { "cell_type": "code", "execution_count": 145, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T04:58:23.588224Z", "iopub.status.busy": "2025-11-22T04:58:23.588094Z", "iopub.status.idle": "2025-11-22T04:58:23.600287Z", "shell.execute_reply": "2025-11-22T04:58:23.599861Z", "shell.execute_reply.started": "2025-11-22T04:58:23.588212Z" } }, "outputs": [], "source": [ "# from collections import Counter, defaultdict\n", "# from typing import List, Dict, Set\n", "\n", "# # Read all meta files\n", "# meta_files: List[str] = [\n", "# \"/app/suno/data/dpo/30b_t1_v23/meta_tr.jsonl\",\n", "# \"/app/suno/data/dpo/30b_t6_v35/meta_tr.jsonl\",\n", "# \"/app/suno/data/dpo/13b_s32_v34/meta_tr.jsonl\",\n", "# \"/app/suno/data/dpo/auk_mix_t1_v6/meta_tr.jsonl\",\n", "# \"/app/suno/data/dpo/auk_mix_t1_v14/meta_tr.jsonl\",\n", "# \"/app2/suno/data/dpo/auk_t1_v6/meta_tr.jsonl\",\n", "# \"/app2/suno/data/dpo/auk_t1_v7/meta_tr.jsonl\",\n", "# \"/app2/suno/data/dpo/auk_t1_v19/meta_tr.jsonl\",\n", "# \"/app2/suno/data/dpo/auk_t1_v29/meta_tr.jsonl\",\n", "# ]\n", "\n", "# user_id_counter: Counter[str] = Counter()\n", "# for meta_file in tqdm(meta_files):\n", "# metas = read_jsonl(meta_file)\n", "# # Count each user_id once per dataset\n", "# user_ids: Set[str] = {meta[\"user_id\"] for meta in metas if \"user_id\" in meta}\n", "# user_id_counter.update(user_ids)\n", "\n", "# # Map from count to set of user_ids\n", "# count_to_users: Dict[int, Set[str]] = defaultdict(set)\n", "# for user_id, count in user_id_counter.items():\n", "# count_to_users[count].add(user_id)\n", "\n", "# for count in sorted(count_to_users):\n", "# users = count_to_users[count]\n", "# print(f\"Count {count}: {len(users)} users\")\n", "\n", "# # with open(\"/home/tony/Data/top_user_8.json\", \"w\") as fp:\n", "# # json.dump(list(count_to_users[8]) , fp)" ] }, { "cell_type": "code", "execution_count": 146, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T04:58:23.600965Z", "iopub.status.busy": "2025-11-22T04:58:23.600830Z", "iopub.status.idle": "2025-11-22T04:58:23.612579Z", "shell.execute_reply": "2025-11-22T04:58:23.612151Z", "shell.execute_reply.started": "2025-11-22T04:58:23.600952Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/top_user/mask_control_low.json\", \"r\") as fp:\n", "# very_good_users = json.load(fp)\n", "# # very_good_users = count_to_users[5]\n", "# very_good_users_mask = df_total[\"user_id\"].isin(very_good_users)\n", "# print(df_total[very_good_users_mask].shape, df_total.shape)\n", "# print(df_total[very_good_users_mask][\"user_id\"].nunique(), df_total[\"user_id\"].nunique())\n", "# # df_total[very_good_users_mask][\"task\"].value_counts()\n", "# df_slice = df_total[very_good_users_mask].copy()" ] }, { "cell_type": "code", "execution_count": 147, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T04:58:23.613314Z", "iopub.status.busy": "2025-11-22T04:58:23.613183Z", "iopub.status.idle": "2025-11-22T04:58:23.625014Z", "shell.execute_reply": "2025-11-22T04:58:23.624586Z", "shell.execute_reply.started": "2025-11-22T04:58:23.613300Z" } }, "outputs": [], "source": [ "# print(\n", "# \"Before filtering by user_id and task\",\n", "# df_slice.shape[0],\n", "# \"user_id unique:\",\n", "# df_slice[\"user_id\"].nunique(),\n", "# )\n", "\n", "# # Create a copy to avoid fragmentation warning\n", "# df_slice = df_slice.copy()\n", "\n", "# # Calculate score for each row: reaction_play_count + 5 if preference is True, else 0\n", "# score_values = (\n", "# df_slice[\"reaction_play_count\"] + (5 * df_slice[\"upvote_count\"].astype(int))\n", "# ) * df_slice[\"preference\"].astype(int)\n", "\n", "# # Use pd.concat to add the score column efficiently\n", "# df_slice = pd.concat(\n", "# [df_slice, pd.DataFrame({\"score\": score_values}, index=df_slice.index)], axis=1\n", "# )\n", "\n", "# # Group by user_id and task, then for each group find the request_id with highest score\n", "# best_request_ids = []\n", "# for (user_id, task), group in tqdm(\n", "# df_slice.groupby([\"user_id\", \"task\"]), desc=\"Processing user_id and task groups\"\n", "# ):\n", "# # Get the request_id with the highest score in this group\n", "# best_request_id = group.loc[group[\"score\"].idxmax(), \"request_id\"]\n", "# best_request_ids.append(best_request_id)\n", "\n", "# # Filter df_slice to keep only the best request_ids for each user_id, task combination\n", "# df_slice = df_slice[df_slice[\"request_id\"].isin(best_request_ids)].copy()\n", "\n", "# print(\n", "# \"After filtering by user_id and task\",\n", "# df_slice.shape[0],\n", "# \"user_id unique:\",\n", "# df_slice[\"user_id\"].nunique(),\n", "# )" ] }, { "cell_type": "code", "execution_count": 148, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T04:58:23.626781Z", "iopub.status.busy": "2025-11-22T04:58:23.626513Z", "iopub.status.idle": "2025-11-22T04:58:23.637927Z", "shell.execute_reply": "2025-11-22T04:58:23.637504Z", "shell.execute_reply.started": "2025-11-22T04:58:23.626766Z" } }, "outputs": [], "source": [ "# df_total.to_pickle(\n", "# \"/home/tony/Data/Preference/crow_t1/fully_merged_crow_t1_20251120_full_slice.pkl\"\n", "# )" ] }, { "cell_type": "code", "execution_count": 149, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T14:00:20.866354Z", "start_time": "2024-05-16T14:00:12.443344Z" }, "execution": { "iopub.execute_input": "2025-11-22T04:58:23.638521Z", "iopub.status.busy": "2025-11-22T04:58:23.638386Z", "iopub.status.idle": "2025-11-22T04:58:23.696528Z", "shell.execute_reply": "2025-11-22T04:58:23.694631Z", "shell.execute_reply.started": "2025-11-22T04:58:23.638507Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(479262, 223)\n", "task\n", " 263170\n", "cover 127586\n", "artist_consistency 30526\n", "artist_cover 18816\n", "extend 13496\n", "infill 7408\n", "playlist_condition 6184\n", "stem_condition 3404\n", "artist_extend 2620\n", "overpainting 2402\n", "upload_extend 2080\n", "underpainting 1570\n", "Name: count, dtype: int64\n", "/app2/suno/data/dpo/crow_t1_v69\n" ] }, { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[149], line 7\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_slice[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mvalue_counts())\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28mprint\u001b[39m(OUT_DATA_DIR)\n\u001b[0;32m----> 7\u001b[0m \u001b[43mBREAK\u001b[49m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;66;03m# (269282, 199)\u001b[39;00m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "# df_slice.to_pickle(\n", "# \"/home/tony/Data/Preference/30b_v6/interesting_clips_v4_h_t_6_20250426_full_long_slice.pkl\"\n", "# )\n", "print(df_slice.shape)\n", "print(df_slice[\"task\"].value_counts())\n", "print(OUT_DATA_DIR)\n", "BREAK\n", "# (269282, 199)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Need to kick out the ones has gpt prompt -- these are pairs with different text inputs" ] }, { "cell_type": "code", "execution_count": 150, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T04:58:28.427024Z", "iopub.status.busy": "2025-11-22T04:58:28.426323Z", "iopub.status.idle": "2025-11-22T04:58:29.888553Z", "shell.execute_reply": "2025-11-22T04:58:29.887939Z", "shell.execute_reply.started": "2025-11-22T04:58:28.427004Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(479262, 223)\n", "(479262, 223)\n", "(479262, 223)\n" ] } ], "source": [ "print(df_slice.shape)\n", "df_slice = df_slice[df_slice[\"request_id\"].apply(lambda x: len(x) > 3)]\n", "print(df_slice.shape)\n", "# df_slice = df_slice[df_slice[\"is_pro_user\"]].copy()\n", "print(df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 151, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932296Z", "start_time": "2024-05-16T13:59:41.932287Z" }, "execution": { "iopub.execute_input": "2025-11-22T04:58:29.889678Z", "iopub.status.busy": "2025-11-22T04:58:29.889502Z", "iopub.status.idle": "2025-11-22T04:58:29.923309Z", "shell.execute_reply": "2025-11-22T04:58:29.922776Z", "shell.execute_reply.started": "2025-11-22T04:58:29.889661Z" } }, "outputs": [], "source": [ "# don't have continue at\n", "df_slice[\"request_id\"] = df_slice[\"request_id\"].astype(str)\n", "# df_slice[\"npz_path\"] = df_slice[\"npz_path\"].apply(lambda x: str(x).replace(\"_npz\", \"_npz/\"))\n", "# df_slice[df_slice[\"continue_at\"].isna()][\"request_id\"].nunique(), df_slice[\"request_id\"].nunique()" ] }, { "cell_type": "code", "execution_count": 152, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932966Z", "start_time": "2024-05-16T13:59:41.932957Z" }, "execution": { "iopub.execute_input": "2025-11-22T04:58:29.924232Z", "iopub.status.busy": "2025-11-22T04:58:29.923877Z", "iopub.status.idle": "2025-11-22T04:58:30.007464Z", "shell.execute_reply": "2025-11-22T04:58:30.006900Z", "shell.execute_reply.started": "2025-11-22T04:58:29.924214Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "239631\n" ] } ], "source": [ "final_filtered_requests = df_slice[\"request_id\"].astype(str).unique()\n", "# final_filtered_requests = df_slice[df_slice[\"is_pro_user\"]][\"request_id\"].astype(str).unique()\n", "print(len(final_filtered_requests))" ] }, { "cell_type": "code", "execution_count": 153, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934277Z", "start_time": "2024-05-16T13:59:41.934268Z" }, "execution": { "iopub.execute_input": "2025-11-22T04:58:30.008228Z", "iopub.status.busy": "2025-11-22T04:58:30.008075Z", "iopub.status.idle": "2025-11-22T04:58:38.661383Z", "shell.execute_reply": "2025-11-22T04:58:38.660768Z", "shell.execute_reply.started": "2025-11-22T04:58:30.008212Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "237234 2397\n", "(474468, 223) (4794, 223)\n" ] } ], "source": [ "train_requests, val_requests = train_test_split(\n", " sorted(list(final_filtered_requests)), test_size=0.01, random_state=42\n", ")\n", "print(len(train_requests), len(val_requests))\n", "\n", "train_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(train_requests))].copy()\n", "val_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(val_requests))].copy()\n", "train_df = train_df.sort_values(by=[\"request_id\", \"preference\"])\n", "train_df = train_df # .reset_index()\n", "val_df = val_df.sort_values(by=[\"request_id\", \"preference\"])\n", "val_df = val_df # .reset_index()\n", "train_df = train_df.reset_index(drop=True)\n", "val_df = val_df.reset_index(drop=True)\n", "print(train_df.shape, val_df.shape)" ] }, { "cell_type": "code", "execution_count": 154, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T04:58:38.662838Z", "iopub.status.busy": "2025-11-22T04:58:38.662660Z", "iopub.status.idle": "2025-11-22T04:58:38.679507Z", "shell.execute_reply": "2025-11-22T04:58:38.679045Z", "shell.execute_reply.started": "2025-11-22T04:58:38.662822Z" } }, "outputs": [], "source": [ "# with open(\"./reward_filter_results/multi_model_passed_indices_std-1.0.json\", \"r\") as fp:\n", "# crow_t1_v59_filtered = json.load(fp)\n", "# old_train_metas = read_jsonl(os.path.join(\"/app2/suno/data/dpo/crow_t1_v59\", f\"meta_tr.jsonl\"))\n", "# filtered_train_ids = set([old_train_metas[index][\"id\"] for index in crow_t1_v59_filtered[\"train\"]])\n", "# print(len(filtered_train_ids))" ] }, { "cell_type": "code", "execution_count": 155, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T04:58:38.680257Z", "iopub.status.busy": "2025-11-22T04:58:38.680116Z", "iopub.status.idle": "2025-11-22T04:58:38.694337Z", "shell.execute_reply": "2025-11-22T04:58:38.693910Z", "shell.execute_reply.started": "2025-11-22T04:58:38.680242Z" } }, "outputs": [], "source": [ "# OUT_DATA_DIR = \"/app2/suno/data/dpo/crow_t1_v59_reward_f1\"\n", "# os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "# shutil.copyfile(\n", "# \"/app/suno/data/dpo/7v_v20_full/tokenizer_60k.json\",\n", "# os.path.join(OUT_DATA_DIR, \"tokenizer_60k.json\"),\n", "# )\n", "# NPZ_DIR = \"/app2/suno/data/dpo/bluejay_t1_npz\"\n", "# N_TOKENS_AUDIO = 25 * 8 * 60\n", "# print(\"N_TOKENS_AUDIO\", N_TOKENS_AUDIO)" ] }, { "cell_type": "code", "execution_count": 156, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T04:58:38.695037Z", "iopub.status.busy": "2025-11-22T04:58:38.694901Z", "iopub.status.idle": "2025-11-22T04:58:38.706173Z", "shell.execute_reply": "2025-11-22T04:58:38.705759Z", "shell.execute_reply.started": "2025-11-22T04:58:38.695024Z" } }, "outputs": [], "source": [ "# train_df = train_df[train_df[\"id\"].isin(filtered_train_ids)].copy()\n", "# print(train_df.shape, val_df.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Actually make" ] }, { "cell_type": "code", "execution_count": 157, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.935620Z", "start_time": "2024-05-16T13:59:41.935613Z" }, "execution": { "iopub.execute_input": "2025-11-22T04:58:38.706786Z", "iopub.status.busy": "2025-11-22T04:58:38.706653Z", "iopub.status.idle": "2025-11-22T04:58:57.713017Z", "shell.execute_reply": "2025-11-22T04:58:57.712447Z", "shell.execute_reply.started": "2025-11-22T04:58:38.706773Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 474468/474468 [00:18<00:00, 24982.61it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "23,826 hours of 474468 clips, 29.65425 nodes, 617.796875 iters\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "total_duration = 0\n", "for i, row in tqdm(train_df.iterrows(), total=len(train_df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " try:\n", " assert row[\"preference\"] == (i % 2 == 1)\n", " total_duration += row[\"duration\"]\n", " except Exception as E:\n", " print(i, row)\n", " print(E)\n", " raise ValueError()\n", "\n", "print(\n", " f\"{round(total_duration / 60 / 60):,} hours of {train_df.shape[0]} clips, {train_df.shape[0] / 8 / 2 / 1000} nodes, {train_df.shape[0] / 8 / 6 / 16} iters\"\n", ")" ] }, { "cell_type": "code", "execution_count": 158, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936268Z", "start_time": "2024-05-16T13:59:41.936260Z" }, "execution": { "iopub.execute_input": "2025-11-22T04:58:57.713789Z", "iopub.status.busy": "2025-11-22T04:58:57.713629Z", "iopub.status.idle": "2025-11-22T05:00:07.333736Z", "shell.execute_reply": "2025-11-22T05:00:07.333174Z", "shell.execute_reply.started": "2025-11-22T04:58:57.713773Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "t_data_memmap is set to: 12000\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4794/4794 [01:08<00:00, 69.74it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 4794 clips, 0 different prompts, 0 different tags, 0 different negative tags\n", "168 hours of False\n", "169 hours of True\n", "gen: 131.9 hours\n", "cover: 126.1 hours\n", "artist_consistency: 35.7 hours\n", "infill: 1.4 hours\n", "extend: 6.3 hours\n", "artist_cover: 25.0 hours\n", "underpainting: 1.5 hours\n", "stem_condition: 2.5 hours\n", "artist_extend: 1.8 hours\n", "playlist_condition: 4.3 hours\n", "overpainting: 1.3 hours\n", "\n", "--- Gender Distribution ---\n", " female: 452 (9.4%)\n", " male: 720 (15.0%)\n", " unspecified: 3,622 (75.6%)\n", "\n", "--- Negative Tags Usage ---\n", " has_neg_tags: 242 (5.0%)\n", " no_neg_tags: 4,552 (95.0%)\n", "\n", "--- Control Slider Usage ---\n", " has_control_slider: 1,472 (30.7% of clips)\n", " no_control_slider: 3,322 (69.3% of clips)\n", "Done\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "make_dataset(\n", " val_df, OUT_DATA_DIR, is_val=True, npz_dir=NPZ_DIR, t_data_memmap=N_TOKENS_AUDIO\n", ")" ] }, { "cell_type": "code", "execution_count": 159, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T05:00:07.334547Z", "iopub.status.busy": "2025-11-22T05:00:07.334381Z", "iopub.status.idle": "2025-11-22T05:00:07.689839Z", "shell.execute_reply": "2025-11-22T05:00:07.689219Z", "shell.execute_reply.started": "2025-11-22T05:00:07.334531Z" } }, "outputs": [], "source": [ "# test_npz = np.load(\"/app/suno/data/dpo/30b_npz/26d19085-18da-4701-af43-122684543891.npz\")\n", "# for k in test_npz.keys():\n", "# print(k)" ] }, { "cell_type": "code", "execution_count": 160, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936964Z", "start_time": "2024-05-16T13:59:41.936957Z" }, "execution": { "iopub.execute_input": "2025-11-22T05:00:07.690657Z", "iopub.status.busy": "2025-11-22T05:00:07.690497Z", "iopub.status.idle": "2025-11-22T06:32:46.347688Z", "shell.execute_reply": "2025-11-22T06:32:46.347065Z", "shell.execute_reply.started": "2025-11-22T05:00:07.690641Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "t_data_memmap is set to: 12000\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 474468/474468 [1:32:37<00:00, 85.38it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 474468 clips, 0 different prompts, 0 different tags, 0 different negative tags\n", "16,470 hours of False\n", "16,549 hours of True\n", "gen: 13299.5 hours\n", "artist_cover: 2156.3 hours\n", "cover: 12178.0 hours\n", "infill: 143.4 hours\n", "artist_consistency: 3062.5 hours\n", "extend: 840.2 hours\n", "artist_extend: 244.7 hours\n", "playlist_condition: 510.7 hours\n", "stem_condition: 241.8 hours\n", "overpainting: 209.5 hours\n", "underpainting: 132.2 hours\n", "\n", "--- Gender Distribution ---\n", " female: 45,006 (9.5%)\n", " male: 71,440 (15.1%)\n", " unspecified: 358,022 (75.5%)\n", "\n", "--- Negative Tags Usage ---\n", " has_neg_tags: 26,844 (5.7%)\n", " no_neg_tags: 447,624 (94.3%)\n", "\n", "--- Control Slider Usage ---\n", " has_control_slider: 145,520 (30.7% of clips)\n", " no_control_slider: 328,948 (69.3% of clips)\n", "Done\n" ] } ], "source": [ "make_dataset(\n", " train_df, OUT_DATA_DIR, is_val=False, npz_dir=NPZ_DIR, t_data_memmap=N_TOKENS_AUDIO\n", ")" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2024-01-29T19:46:47.549860Z", "start_time": "2024-01-29T19:46:47.548015Z" } }, "source": [ "# Validation" ] }, { "cell_type": "code", "execution_count": 161, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.937879Z", "start_time": "2024-05-16T13:59:41.937870Z" }, "execution": { "iopub.execute_input": "2025-11-22T06:32:46.348494Z", "iopub.status.busy": "2025-11-22T06:32:46.348328Z", "iopub.status.idle": "2025-11-22T06:32:47.369217Z", "shell.execute_reply": "2025-11-22T06:32:47.368644Z", "shell.execute_reply.started": "2025-11-22T06:32:46.348478Z" } }, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "test_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_val.jsonl\"))\n", "test_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_val.json\"))\n", "mm = mm.reshape(-1, N_TOKENS_AUDIO, 1)\n", "assert len(mm) == len(test_metas)\n", "assert mm[:100, :, 0].min() >= 0\n", "assert mm[:100, :, 0].max() <= 4000" ] }, { "cell_type": "code", "execution_count": 162, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T06:32:47.369955Z", "iopub.status.busy": "2025-11-22T06:32:47.369804Z", "iopub.status.idle": "2025-11-22T06:32:47.387888Z", "shell.execute_reply": "2025-11-22T06:32:47.387401Z", "shell.execute_reply.started": "2025-11-22T06:32:47.369939Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Counter({None: 2602, 'cover': 1300, 'artist_consistency': 350, 'artist_cover': 212, 'extend': 116, 'infill': 78, 'playlist_condition': 50, 'stem_condition': 34, 'artist_extend': 20, 'overpainting': 18, 'underpainting': 14})\n" ] } ], "source": [ "task_counts = Counter()\n", "for test_meta in test_metas:\n", " task_counts[test_meta.get(\"task\")] += 1\n", "print(task_counts)" ] }, { "cell_type": "code", "execution_count": 163, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.938629Z", "start_time": "2024-05-16T13:59:41.938621Z" }, "execution": { "iopub.execute_input": "2025-11-22T06:32:47.390068Z", "iopub.status.busy": "2025-11-22T06:32:47.389914Z", "iopub.status.idle": "2025-11-22T06:32:47.403379Z", "shell.execute_reply": "2025-11-22T06:32:47.402946Z", "shell.execute_reply.started": "2025-11-22T06:32:47.390054Z" } }, "outputs": [], "source": [ "# # randomly listen to some stuff\n", "# from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "# from suno_utils.tasks.dac_2c_12cb import (\n", "# encode as codec_encode,\n", "# decode_stream_to_full_audio as codec_decode,\n", "# EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", "# decode as decode\n", "# )\n", "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", "# _ = preload_codec_models(\"/app/suno/data/dpo/models/dac_2c_25x12.pt\", device=\"cuda\")\n", "# assert len(test_metas) == len(mm)\n", "# idx_list = list(range(len(test_metas)))\n", "# # random.shuffle(idx_list)\n", "# # idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "# print(len(mm))" ] }, { "cell_type": "code", "execution_count": 164, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939205Z", "start_time": "2024-05-16T13:59:41.939198Z" }, "execution": { "iopub.execute_input": "2025-11-22T06:32:47.404052Z", "iopub.status.busy": "2025-11-22T06:32:47.403915Z", "iopub.status.idle": "2025-11-22T06:32:47.415548Z", "shell.execute_reply": "2025-11-22T06:32:47.415123Z", "shell.execute_reply.started": "2025-11-22T06:32:47.404039Z" } }, "outputs": [], "source": [ "# import random\n", "# idx = random.choice(test_info[\"perference_0\"][\"idx_list\"])\n", "# assert \"original_duration_s\" in test_metas[idx]\n", "# # positive index should be shifted by 1\n", "# pos_idx = idx + 1\n", "# print(\n", "# \"tags:\",\n", "# test_metas[idx].get(\"tags\") == test_metas[pos_idx].get(\"tags\"),\n", "# test_metas[idx].get(\"tags\"),\n", "# )\n", "# arr = mm[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pos_arr = mm[pos_idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pad_idx_arr = np.where(arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pad_idx_arr) > 0:\n", "# arr = arr[: pad_idx_arr[0], :]\n", "# pos_pad_idx_arr = np.where(pos_arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pos_pad_idx_arr) > 0:\n", "# pos_arr = pos_arr[: pos_pad_idx_arr[0], :]\n", "# a = decode(arr)\n", "# print(\"\\n negative example \\n\", test_metas[idx])\n", "# a.play(compress=False)\n", "# pos_a = decode(pos_arr)\n", "# print(\"\\n positive example \\n\", test_metas[pos_idx])\n", "# pos_a.play(compress=False)\n", "# print(\n", "# \"text:\",\n", "# test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"),\n", "# test_metas[idx].get(\"text\"),\n", "# )" ] }, { "cell_type": "code", "execution_count": 165, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939977Z", "start_time": "2024-05-16T13:59:41.939969Z" }, "execution": { "iopub.execute_input": "2025-11-22T06:32:47.416156Z", "iopub.status.busy": "2025-11-22T06:32:47.416021Z", "iopub.status.idle": "2025-11-22T06:32:47.427316Z", "shell.execute_reply": "2025-11-22T06:32:47.426886Z", "shell.execute_reply.started": "2025-11-22T06:32:47.416140Z" } }, "outputs": [], "source": [ "# val_df[val_df[\"tags\"] == 'a vibrant blend of experimental jazz fusion, drum-and-bass and swagger fuzzed-out guitars']" ] }, { "cell_type": "code", "execution_count": 166, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.940610Z", "start_time": "2024-05-16T13:59:41.940603Z" }, "execution": { "iopub.execute_input": "2025-11-22T06:32:47.427901Z", "iopub.status.busy": "2025-11-22T06:32:47.427771Z", "iopub.status.idle": "2025-11-22T06:32:47.438983Z", "shell.execute_reply": "2025-11-22T06:32:47.438566Z", "shell.execute_reply.started": "2025-11-22T06:32:47.427888Z" } }, "outputs": [], "source": [ "# from collections import Counter\n", "# c = Counter()\n", "# for _, row in df_slice.iterrows():\n", "# # print(row[\"metadata\"])\n", "# for k in ast.literal_eval(row[\"metadata\"]).keys():\n", "# c[k] += 1" ] }, { "cell_type": "code", "execution_count": 167, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941167Z", "start_time": "2024-05-16T13:59:41.941159Z" }, "execution": { "iopub.execute_input": "2025-11-22T06:32:47.439591Z", "iopub.status.busy": "2025-11-22T06:32:47.439453Z", "iopub.status.idle": "2025-11-22T06:32:47.450877Z", "shell.execute_reply": "2025-11-22T06:32:47.450453Z", "shell.execute_reply.started": "2025-11-22T06:32:47.439577Z" } }, "outputs": [], "source": [ "# original_npz_path = f\"/app/suno/data/dpo/7b_npz/{test_metas[idx]['id']}.npz\"\n", "# original_npz_path = \"/app/suno/data/dpo/7b_npz/729c3011-f672-4ccd-8d82-1cbf2b52ff69.npz\"\n", "# original_arr = np.load(original_npz_path)[\"v2_raw\"]" ] }, { "cell_type": "code", "execution_count": 168, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941801Z", "start_time": "2024-05-16T13:59:41.941793Z" }, "execution": { "iopub.execute_input": "2025-11-22T06:32:47.451484Z", "iopub.status.busy": "2025-11-22T06:32:47.451351Z", "iopub.status.idle": "2025-11-22T06:32:47.468120Z", "shell.execute_reply": "2025-11-22T06:32:47.467661Z", "shell.execute_reply.started": "2025-11-22T06:32:47.451471Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2397 0\n" ] } ], "source": [ "def validation_on_metas(input_metas):\n", " total_bad = 0\n", " total_good = 0\n", " for idx in range(len(input_metas)):\n", " if idx % 2 == 0:\n", " pos_idx = idx + 1\n", " if input_metas[idx].get(\"tags\") != input_metas[pos_idx].get(\"tags\"):\n", " print(\n", " input_metas[idx].get(\"id\"),\n", " input_metas[idx].get(\"tags\"),\n", " input_metas[pos_idx].get(\"id\"),\n", " input_metas[pos_idx].get(\"tags\"),\n", " )\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " elif input_metas[idx].get(\"text\") != input_metas[pos_idx].get(\"text\"):\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " else:\n", " total_good += 1\n", " print(total_good, total_bad)\n", " return\n", "\n", "\n", "validation_on_metas(test_metas)" ] }, { "cell_type": "code", "execution_count": 169, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.942520Z", "start_time": "2024-05-16T13:59:41.942511Z" }, "execution": { "iopub.execute_input": "2025-11-22T06:32:47.468845Z", "iopub.status.busy": "2025-11-22T06:32:47.468713Z", "iopub.status.idle": "2025-11-22T06:32:47.516035Z", "shell.execute_reply": "2025-11-22T06:32:47.515579Z", "shell.execute_reply.started": "2025-11-22T06:32:47.468831Z" } }, "outputs": [], "source": [ "train_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_tr.json\"))" ] }, { "cell_type": "code", "execution_count": 170, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.943072Z", "start_time": "2024-05-16T13:59:41.943065Z" }, "execution": { "iopub.execute_input": "2025-11-22T06:32:47.516679Z", "iopub.status.busy": "2025-11-22T06:32:47.516544Z", "iopub.status.idle": "2025-11-22T06:32:47.546881Z", "shell.execute_reply": "2025-11-22T06:32:47.546435Z", "shell.execute_reply.started": "2025-11-22T06:32:47.516666Z" } }, "outputs": [], "source": [ "n_neg_tr = train_info[\"perference_0\"][\"idx_list\"]\n", "n_pos_tr = train_info[\"perference_1\"][\"idx_list\"]\n", "assert len(n_pos_tr) == len(n_neg_tr)\n", "# make sure they are offset by 1 and exactly 1\n", "for i, j in zip(n_neg_tr, n_pos_tr):\n", " assert i == j - 1" ] }, { "cell_type": "code", "execution_count": 171, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.944246Z", "start_time": "2024-05-16T13:59:41.944237Z" }, "execution": { "iopub.execute_input": "2025-11-22T06:32:47.547513Z", "iopub.status.busy": "2025-11-22T06:32:47.547377Z", "iopub.status.idle": "2025-11-22T06:32:47.559296Z", "shell.execute_reply": "2025-11-22T06:32:47.558851Z", "shell.execute_reply.started": "2025-11-22T06:32:47.547499Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total samples 474468 (474468, 223)\n" ] } ], "source": [ "total_iters = len(n_neg_tr) + len(n_pos_tr)\n", "print(\"total samples\", total_iters, train_df.shape)" ] }, { "cell_type": "code", "execution_count": 172, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T06:32:47.559917Z", "iopub.status.busy": "2025-11-22T06:32:47.559779Z", "iopub.status.idle": "2025-11-22T06:32:56.074856Z", "shell.execute_reply": "2025-11-22T06:32:56.074249Z", "shell.execute_reply.started": "2025-11-22T06:32:47.559903Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "237234 0\n" ] } ], "source": [ "metas_tr = read_jsonl(os.path.join(OUT_DATA_DIR, \"meta_tr.jsonl\"))\n", "validation_on_metas(metas_tr)" ] }, { "cell_type": "code", "execution_count": 173, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T06:32:56.075648Z", "iopub.status.busy": "2025-11-22T06:32:56.075478Z", "iopub.status.idle": "2025-11-22T06:32:56.091740Z", "shell.execute_reply": "2025-11-22T06:32:56.091289Z", "shell.execute_reply.started": "2025-11-22T06:32:56.075632Z" } }, "outputs": [], "source": [ "# new_metas_tr = []\n", "# for index, l in enumerate(metas_tr):\n", "# if index % 2 == 1:\n", "# last_l = new_metas_tr[-1]\n", "# if l[\"tags\"] != last_l[\"tags\"]:\n", "# print(l[\"tags\"], last_l[\"tags\"])\n", "# l[\"tags\"] = last_l[\"tags\"]\n", "# new_metas_tr.append(l)\n", "# validation_on_metas(new_metas_tr)\n", "# write_jsonl(new_metas_tr, os.path.join(OUT_DATA_DIR, \"meta_tr.jsonl\"))" ] }, { "cell_type": "code", "execution_count": 174, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945249Z", "start_time": "2024-05-16T13:59:41.945241Z" }, "execution": { "iopub.execute_input": "2025-11-22T06:32:56.092385Z", "iopub.status.busy": "2025-11-22T06:32:56.092244Z", "iopub.status.idle": "2025-11-22T06:32:56.106721Z", "shell.execute_reply": "2025-11-22T06:32:56.106280Z", "shell.execute_reply.started": "2025-11-22T06:32:56.092371Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1 epoch per batch 4, total 1853.390625\n", "1 epoch per batch 6, total 617.796875\n", "1 epoch per batch 8, total 463.34765625\n" ] } ], "source": [ "print(\"1 epoch per batch 4, total\", total_iters / 8 / 8 / 4)\n", "print(\"1 epoch per batch 6, total\", total_iters / 16 / 8 / 6)\n", "print(\"1 epoch per batch 8, total\", total_iters / 16 / 8 / 8)" ] }, { "cell_type": "code", "execution_count": 175, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T06:32:56.107318Z", "iopub.status.busy": "2025-11-22T06:32:56.107186Z", "iopub.status.idle": "2025-11-22T06:32:56.118421Z", "shell.execute_reply": "2025-11-22T06:32:56.117984Z", "shell.execute_reply.started": "2025-11-22T06:32:56.107306Z" } }, "outputs": [], "source": [ "# import time\n", "# time.sleep(60 * 60 * 1)" ] }, { "cell_type": "code", "execution_count": 176, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T06:32:56.119116Z", "iopub.status.busy": "2025-11-22T06:32:56.118980Z", "iopub.status.idle": "2025-11-22T06:32:56.130167Z", "shell.execute_reply": "2025-11-22T06:32:56.129751Z", "shell.execute_reply.started": "2025-11-22T06:32:56.119103Z" } }, "outputs": [], "source": [ "# df[\"control_tags\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 177, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945972Z", "start_time": "2024-05-16T13:59:41.945964Z" }, "execution": { "iopub.execute_input": "2025-11-22T06:32:56.130751Z", "iopub.status.busy": "2025-11-22T06:32:56.130622Z", "iopub.status.idle": "2025-11-22T06:32:56.142000Z", "shell.execute_reply": "2025-11-22T06:32:56.141574Z", "shell.execute_reply.started": "2025-11-22T06:32:56.130738Z" } }, "outputs": [], "source": [ "# !cd /home/tony/Work/tony/slurm/crow && sbatch sbatch_ipo_crow" ] }, { "cell_type": "code", "execution_count": 178, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T06:32:56.142613Z", "iopub.status.busy": "2025-11-22T06:32:56.142485Z", "iopub.status.idle": "2025-11-22T06:32:56.165344Z", "shell.execute_reply": "2025-11-22T06:32:56.164895Z", "shell.execute_reply.started": "2025-11-22T06:32:56.142601Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cache kept!\n" ] } ], "source": [ "import shutil\n", "\n", "# Basic file copy\n", "shutil.copy(\n", " \"/home/tony/Work/tony/Preference/make_dataset_crow_t1_dup3.ipynb\",\n", " os.path.join(OUT_DATA_DIR, \"make_dataset.ipynb\"),\n", ")\n", "print(\"Cache kept!\")" ] }, { "cell_type": "code", "execution_count": 179, "metadata": { "execution": { "iopub.execute_input": "2025-11-22T06:32:56.165943Z", "iopub.status.busy": "2025-11-22T06:32:56.165811Z", "iopub.status.idle": "2025-11-22T06:32:56.188446Z", "shell.execute_reply": "2025-11-22T06:32:56.187271Z", "shell.execute_reply.started": "2025-11-22T06:32:56.165930Z" } }, "outputs": [ { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[179], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mBREAK\u001b[49m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "BREAK" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# some gymathtics loading prev data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.946562Z", "start_time": "2024-05-16T13:59:41.946555Z" }, "execution": { "iopub.status.busy": "2025-11-22T06:32:56.188910Z", "iopub.status.idle": "2025-11-22T06:32:56.189087Z", "shell.execute_reply": "2025-11-22T06:32:56.189001Z", "shell.execute_reply.started": "2025-11-22T06:32:56.188993Z" } }, "outputs": [], "source": [ "# prev_v3_data = \"/app/suno/data/dpo/7v_v20_full/\"\n", "\n", "# test_val_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_val.jsonl\"))\n", "# test_tr_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_tr.jsonl\"))\n", "\n", "# all_ids = set()\n", "# for meta in test_val_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# for meta in test_tr_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# print(len(all_ids), len(test_val_metas) + len(test_tr_metas))\n", "\n", "# all_ids = list(all_ids)\n", "# with open(\"/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id.json\", \"w\") as fp:\n", "# json.dump(all_ids, fp)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.189594Z", "iopub.status.idle": "2025-11-22T06:32:56.189756Z", "shell.execute_reply": "2025-11-22T06:32:56.189676Z", "shell.execute_reply.started": "2025-11-22T06:32:56.189669Z" } }, "outputs": [], "source": [ "# x_data = train_df[train_df[\"preference\"]][\"similarity\"]\n", "# y_data = train_df[~train_df[\"preference\"]][\"similarity\"]\n", "# from matplotlib.colors import LogNorm\n", "\n", "# fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 10))\n", "\n", "# # 2D Histogram\n", "# h = ax1.hist2d(\n", "# x_data,\n", "# y_data,\n", "# bins=(50, 50),\n", "# cmap=\"coolwarm\",\n", "# range=[[0, 1], [0, 1]],\n", "# norm=LogNorm(),\n", "# )\n", "\n", "# ax1.set_xlabel(\"Semantic Distance (Preferred)\")\n", "# ax1.set_ylabel(\"Semantic Distance (Non-Preferred)\")\n", "# ax1.set_title(\n", "# \"2D Histogram of Semantic Distances: Preferred vs Non-Preferred (Log Scale)\"\n", "# )\n", "\n", "# cbar1 = plt.colorbar(h[3], ax=ax1)\n", "# cbar1.set_label(\"Number of Request IDs (Log Scale)\")\n", "\n", "# # Scatter plot\n", "# ax2.scatter(x_data, y_data, alpha=0.1, s=1)\n", "# ax2.set_xlabel(\"Semantic Distance (Preferred)\")\n", "# ax2.set_ylabel(\"Semantic Distance (Non-Preferred)\")\n", "# ax2.set_title(\"Scatter Plot of Semantic Distances: Preferred vs Non-Preferred\")\n", "# ax2.set_xlim(0, 1)\n", "# ax2.set_ylim(0, 1)\n", "\n", "# plt.tight_layout()\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.190134Z", "iopub.status.idle": "2025-11-22T06:32:56.190291Z", "shell.execute_reply": "2025-11-22T06:32:56.190217Z", "shell.execute_reply.started": "2025-11-22T06:32:56.190209Z" } }, "outputs": [], "source": [ "# train_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_tr.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.190744Z", "iopub.status.idle": "2025-11-22T06:32:56.190896Z", "shell.execute_reply": "2025-11-22T06:32:56.190825Z", "shell.execute_reply.started": "2025-11-22T06:32:56.190818Z" } }, "outputs": [], "source": [ "# train_info.keys()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.191448Z", "iopub.status.idle": "2025-11-22T06:32:56.191597Z", "shell.execute_reply": "2025-11-22T06:32:56.191530Z", "shell.execute_reply.started": "2025-11-22T06:32:56.191522Z" } }, "outputs": [], "source": [ "# import torch\n", "\n", "# a = torch.tensor([6.2500e-04, 3.9062e-05, 2.3462e-03])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.191907Z", "iopub.status.idle": "2025-11-22T06:32:56.192047Z", "shell.execute_reply": "2025-11-22T06:32:56.191980Z", "shell.execute_reply.started": "2025-11-22T06:32:56.191973Z" } }, "outputs": [], "source": [ "# a.mean()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.192452Z", "iopub.status.idle": "2025-11-22T06:32:56.192593Z", "shell.execute_reply": "2025-11-22T06:32:56.192526Z", "shell.execute_reply.started": "2025-11-22T06:32:56.192519Z" } }, "outputs": [], "source": [ "# import time\n", "# time.sleep(3600 * 3)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.193007Z", "iopub.status.idle": "2025-11-22T06:32:56.193161Z", "shell.execute_reply": "2025-11-22T06:32:56.193090Z", "shell.execute_reply.started": "2025-11-22T06:32:56.193082Z" } }, "outputs": [], "source": [ "# !cd /home/tony/Work/tony/slurm/diffusion && sbatch run_diffusion_infill.sh" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.193551Z", "iopub.status.idle": "2025-11-22T06:32:56.193699Z", "shell.execute_reply": "2025-11-22T06:32:56.193630Z", "shell.execute_reply.started": "2025-11-22T06:32:56.193622Z" } }, "outputs": [], "source": [ "def get_control_slider(metadata):\n", " if \"param_experiment\" in metadata:\n", " exp = metadata.get(\"param_experiment\", \"\")\n", " if exp:\n", " if exp == \"mask_control_slider\":\n", " if not metadata.get(\"control_sliders\", None):\n", " return False\n", " else:\n", " return True\n", " return None\n", "\n", "df[\"mask_control\"] = df.apply(\n", " lambda row: get_control_slider(row[\"metadata\"]), axis=1\n", ")\n", "masked_request_id = df[~df[\"mask_control\"].isna()][\"request_id\"].unique()\n", "subset_df = df[df[\"request_id\"].isin(masked_request_id)].copy()\n", "print(\"check subset shape\", df.shape, subset_df.shape)\n", "# control masked\n", "user_pref_pct = (\n", " subset_df[subset_df[\"mask_control\"] == True].groupby(\"user_id\")[\"preference\"]\n", " .apply(lambda x: x.mean())\n", ")\n", "print(user_pref_pct.head())\n", "# Plot a histogram of the user preference percentages\n", "plt.figure(figsize=(6, 4))\n", "plt.hist(user_pref_pct, bins=np.linspace(0, 1, 100), edgecolor=\"black\")\n", "plt.xlabel(\"Preference % for mask_control\")\n", "plt.ylabel(\"Number of Users\")\n", "plt.title(\"Histogram of User Preference for 'mask_control'\")\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.193970Z", "iopub.status.idle": "2025-11-22T06:32:56.194108Z", "shell.execute_reply": "2025-11-22T06:32:56.194044Z", "shell.execute_reply.started": "2025-11-22T06:32:56.194037Z" } }, "outputs": [], "source": [ "user_counts = subset_df.groupby(\"user_id\")[\"preference\"].count()\n", "eligible_users = user_counts[user_counts >= 4].index\n", "len(eligible_users)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.194523Z", "iopub.status.idle": "2025-11-22T06:32:56.194664Z", "shell.execute_reply": "2025-11-22T06:32:56.194598Z", "shell.execute_reply.started": "2025-11-22T06:32:56.194592Z" } }, "outputs": [], "source": [ "# control masked\n", "eligible_user_pref_pct = (\n", " subset_df[subset_df[\"user_id\"].isin(eligible_users) & (subset_df[\"mask_control\"] == True)].groupby(\"user_id\")[\"preference\"]\n", " .apply(lambda x: x.mean())\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.195197Z", "iopub.status.idle": "2025-11-22T06:32:56.195340Z", "shell.execute_reply": "2025-11-22T06:32:56.195272Z", "shell.execute_reply.started": "2025-11-22T06:32:56.195266Z" } }, "outputs": [], "source": [ "# Plot a histogram of the user preference percentages\n", "plt.figure(figsize=(6, 4))\n", "plt.hist(eligible_user_pref_pct, bins=np.linspace(0, 1, 20), edgecolor=\"black\")\n", "plt.xlabel(\"Preference % for mask_control\")\n", "plt.ylabel(\"Number of Users\")\n", "plt.title(\"Histogram of User Preference for 'mask_control'\")\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.195782Z", "iopub.status.idle": "2025-11-22T06:32:56.195934Z", "shell.execute_reply": "2025-11-22T06:32:56.195861Z", "shell.execute_reply.started": "2025-11-22T06:32:56.195854Z" } }, "outputs": [], "source": [ "eligible_user_pref_pct[172918] # kakermix" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## sliders" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.196376Z", "iopub.status.idle": "2025-11-22T06:32:56.196523Z", "shell.execute_reply": "2025-11-22T06:32:56.196453Z", "shell.execute_reply.started": "2025-11-22T06:32:56.196446Z" } }, "outputs": [], "source": [ "from typing import Any\n", "import pandas as pd\n", "\n", "def map_control_sliders_to_df(df: pd.DataFrame) -> pd.DataFrame:\n", " \"\"\"Extracts 'style_weight' and 'weirdness_constraint' from the 'control_sliders' dict in the 'metadata' column\n", " and adds them as new columns to the DataFrame.\n", "\n", " Args:\n", " df (pd.DataFrame): DataFrame with a 'metadata' column containing a 'control_sliders' dict.\n", "\n", " Returns:\n", " pd.DataFrame: DataFrame with added 'style_weight' and 'weirdness_constraint' columns.\n", "\n", " Raises:\n", " KeyError: If 'control_sliders', 'style_weight', or 'weirdness_constraint' are missing in any row.\n", " TypeError: If the extracted values are not floats.\n", "\n", " Example:\n", " >>> import pandas as pd\n", " >>> data = [{'metadata': {'control_sliders': {'style_weight': 0.89, 'weirdness_constraint': 0.8}}}]\n", " >>> df = pd.DataFrame(data)\n", " >>> df = map_control_sliders_to_df(df)\n", " >>> df[['style_weight', 'weirdness_constraint']].iloc[0].tolist()\n", " [0.89, 0.8]\n", " \"\"\"\n", " # Vectorized extraction for performance\n", " sliders = df[\"metadata\"].map(lambda m: m.get(\"control_sliders\", {}))\n", " style_weight = sliders.map(lambda s: s.get(\"style_weight\", None))\n", " weirdness_constraint = sliders.map(lambda s: s.get(\"weirdness_constraint\", None))\n", " audio_weight = sliders.map(lambda s: s.get(\"audio_weight\", None))\n", "\n", " df[\"style_weight\"] = style_weight\n", " df[\"weirdness_constraint\"] = weirdness_constraint\n", " df[\"audio_weight\"] = audio_weight\n", " return df\n", "\n", "def print_percentiles(\n", " data: np.ndarray,\n", " percentiles: list[float] = [5, 10, 20, 25, 50, 75, 80, 90, 95]\n", ") -> None:\n", " \"\"\"Prints specified percentiles of the data.\n", "\n", " Args:\n", " data (np.ndarray): Array of values to compute percentiles for.\n", " percentiles (list[float], optional): List of percentiles to print. Defaults to [5, 10, 20, 25, 50, 75, 80, 90, 95].\n", "\n", " Example:\n", " >>> print_percentiles(np.array([1, 2, 3, 4, 5]))\n", " \"\"\"\n", " data = data[~data.isna()]\n", " results = np.percentile(data, percentiles)\n", " print(\"Percentiles:\")\n", " for p, v in zip(percentiles, results):\n", " print(f\" {p:>3}%: {v:.4f}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.196999Z", "iopub.status.idle": "2025-11-22T06:32:56.197142Z", "shell.execute_reply": "2025-11-22T06:32:56.197074Z", "shell.execute_reply.started": "2025-11-22T06:32:56.197067Z" } }, "outputs": [], "source": [ "df_total = map_control_sliders_to_df(df_total)\n", "df_total[[\"style_weight\",\"weirdness_constraint\",\"audio_weight\"]].describe()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.197550Z", "iopub.status.idle": "2025-11-22T06:32:56.197693Z", "shell.execute_reply": "2025-11-22T06:32:56.197625Z", "shell.execute_reply.started": "2025-11-22T06:32:56.197618Z" } }, "outputs": [], "source": [ "plt.figure(figsize=(8, 5))\n", "df_total[\"style_weight\"].hist(bins=np.linspace(0, 1, 100), color=\"skyblue\", edgecolor=\"black\")\n", "print_percentiles(df_total[\"style_weight\"])\n", "plt.title(\"Distribution of Style Weight\")\n", "plt.xlabel(\"Style Weight\")\n", "plt.ylabel(\"Counts\")\n", "plt.grid(True, linestyle=\"--\", alpha=0.6)\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.198134Z", "iopub.status.idle": "2025-11-22T06:32:56.198289Z", "shell.execute_reply": "2025-11-22T06:32:56.198218Z", "shell.execute_reply.started": "2025-11-22T06:32:56.198211Z" } }, "outputs": [], "source": [ "plt.figure(figsize=(8, 5))\n", "df_total[\"weirdness_constraint\"].hist(bins=np.linspace(0, 1, 100), color=\"skyblue\", edgecolor=\"black\")\n", "print_percentiles(df_total[\"weirdness_constraint\"])\n", "plt.title(\"Distribution of Weirdness\")\n", "plt.xlabel(\"weirdness_constraint\")\n", "plt.ylabel(\"Counts\")\n", "plt.grid(True, linestyle=\"--\", alpha=0.6)\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.198752Z", "iopub.status.idle": "2025-11-22T06:32:56.198896Z", "shell.execute_reply": "2025-11-22T06:32:56.198829Z", "shell.execute_reply.started": "2025-11-22T06:32:56.198822Z" } }, "outputs": [], "source": [ "plt.figure(figsize=(8, 5))\n", "df_total[\"audio_weight\"].hist(bins=np.linspace(0, 1, 100), color=\"skyblue\", edgecolor=\"black\")\n", "print_percentiles(df_total[\"audio_weight\"])\n", "plt.title(\"Distribution of audio_weight\")\n", "plt.xlabel(\"audio_weight\")\n", "plt.ylabel(\"Counts\")\n", "plt.grid(True, linestyle=\"--\", alpha=0.6)\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Fetch other parameters" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.199381Z", "iopub.status.idle": "2025-11-22T06:32:56.199524Z", "shell.execute_reply": "2025-11-22T06:32:56.199457Z", "shell.execute_reply.started": "2025-11-22T06:32:56.199450Z" } }, "outputs": [], "source": [ "# df_total[df_total[\"task\"] == \"stem_condition\"][[\"s3_id\", \"preference\", \"request_id\", \"control_tags\"]].head(n=100)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.199979Z", "iopub.status.idle": "2025-11-22T06:32:56.200121Z", "shell.execute_reply": "2025-11-22T06:32:56.200052Z", "shell.execute_reply.started": "2025-11-22T06:32:56.200045Z" } }, "outputs": [], "source": [ "len(metas_tr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.200412Z", "iopub.status.idle": "2025-11-22T06:32:56.200559Z", "shell.execute_reply": "2025-11-22T06:32:56.200490Z", "shell.execute_reply.started": "2025-11-22T06:32:56.200483Z" } }, "outputs": [], "source": [ "c = Counter()\n", "for data_meta in metas_tr:\n", " c[data_meta.get(\"control_tags\", \"\") if isinstance(data_meta.get(\"control_tags\", \"\"), str) else \"\"] += 1\n", "print(c)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.200944Z", "iopub.status.idle": "2025-11-22T06:32:56.201085Z", "shell.execute_reply": "2025-11-22T06:32:56.201018Z", "shell.execute_reply.started": "2025-11-22T06:32:56.201011Z" } }, "outputs": [], "source": [ "# with open(\"/app2/suno/data/dpo/crow_t1_v3/dodo_t23_2025-08-28_01-30-09_cached_loss.json\", \"r\") as fp:\n", "# old_loss = json.load(fp)\n", "# with open(\"/app2/suno/data/dpo/crow_t1_v3/dodo_t23_2025-08-28_01-30-09_redo3_cached_loss.json\", \"r\") as fp:\n", "# new_loss = json.load(fp)\n", "\n", "# sem_losses = []\n", "# for curr_id, o_v in old_loss[\"train\"].items():\n", "# n_v = new_loss[\"train\"][curr_id]\n", "# for k in o_v:\n", "# assert o_v[k] == n_v[k]\n", "# if k == \"semantic_0\":\n", "# if o_v[\"orig_idx\"] % 2 == 1:\n", "# sem_losses.append(o_v[k])\n", "# # if o_v[k] > 4 and o_v[\"orig_idx\"] % 2 == 1:\n", "# # print(o_v, o_v[\"orig_idx\"] % 3360, o_v[\"orig_idx\"] % 3360 // 4)\n", "# sem_losses.sort()\n", "# plt.hist(sem_losses, bins=np.linspace(0, 10, 100))\n", "# plt.yscale(\"log\")\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.201519Z", "iopub.status.idle": "2025-11-22T06:32:56.201661Z", "shell.execute_reply": "2025-11-22T06:32:56.201594Z", "shell.execute_reply.started": "2025-11-22T06:32:56.201587Z" } }, "outputs": [], "source": [ "# df[df[\"is_public\"]].to_pickle(\"/home/tony/Data/Preference/crow_t1/interesting_clips_crow_t1_20251020_public_only.pkl\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.201927Z", "iopub.status.idle": "2025-11-22T06:32:56.202062Z", "shell.execute_reply": "2025-11-22T06:32:56.201997Z", "shell.execute_reply.started": "2025-11-22T06:32:56.201990Z" } }, "outputs": [], "source": [ "df[df[\"is_public\"] & (df[\"tags\"].str.len() < 1)]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.202521Z", "iopub.status.idle": "2025-11-22T06:32:56.202659Z", "shell.execute_reply": "2025-11-22T06:32:56.202594Z", "shell.execute_reply.started": "2025-11-22T06:32:56.202588Z" } }, "outputs": [], "source": [ "from suno_analytics.preference_data_selection import (\n", " plot_clip_distribution\n", ")\n", "plot_clip_distribution(df)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.203145Z", "iopub.status.idle": "2025-11-22T06:32:56.203295Z", "shell.execute_reply": "2025-11-22T06:32:56.203224Z", "shell.execute_reply.started": "2025-11-22T06:32:56.203217Z" } }, "outputs": [], "source": [ "plot_clip_distribution(df_total)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.203760Z", "iopub.status.idle": "2025-11-22T06:32:56.203904Z", "shell.execute_reply": "2025-11-22T06:32:56.203836Z", "shell.execute_reply.started": "2025-11-22T06:32:56.203829Z" } }, "outputs": [], "source": [ "plot_clip_distribution(df_total[df_total[\"source\"] == \"web\"])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.204344Z", "iopub.status.idle": "2025-11-22T06:32:56.204488Z", "shell.execute_reply": "2025-11-22T06:32:56.204420Z", "shell.execute_reply.started": "2025-11-22T06:32:56.204413Z" } }, "outputs": [], "source": [ "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "\n", "# Assuming df is your dataframe with 'created_at' column\n", "# df = pd.read_csv('your_data.csv')\n", "\n", "# Convert created_at to datetime if not already\n", "df['created_at'] = pd.to_datetime(df['created_at'])\n", "\n", "# Extract day and hour\n", "df['date'] = df['created_at'].dt.date\n", "df['hour'] = df['created_at'].dt.hour\n", "\n", "# Define your cut_frac function here\n", "def compute_cut_frac(group_df):\n", " \"\"\"\n", " Define your cut fraction logic here.\n", " \n", " Examples:\n", " - Fraction filtered out: (group_df['filtered'] == True).mean()\n", " - Fraction above threshold: (group_df['margin'] > 0.5).mean()\n", " - Fraction in bottom quantile: (group_df['score'] < group_df['score'].quantile(0.2)).mean()\n", " \"\"\"\n", " # CUSTOMIZE THIS:\n", " # Example: fraction of rows where margin > threshold\n", " threshold = 0.5\n", " cut_frac = (group_df[\"preference\"] & (group_df[\"norm_play_frac\"] > 5.1) \n", " & (group_df[\"norm_play_frac\"] >= group_df[\"\"] / 3)\n", " ).mean()\n", " \n", " return cut_frac\n", "\n", "# Group by date and hour\n", "daily_hourly_stats = df.groupby(['date', 'hour']).apply(compute_cut_frac).reset_index()\n", "daily_hourly_stats.columns = ['date', 'hour', 'cut_frac']\n", "\n", "# Also compute count per day-hour\n", "daily_hourly_counts = df.groupby(['date', 'hour']).size().reset_index(name='count')\n", "daily_hourly_stats = daily_hourly_stats.merge(daily_hourly_counts, on=['date', 'hour'])\n", "\n", "# Create plots\n", "fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(14, 10))\n", "\n", "# Plot 1: Line plot with one line per day\n", "for date in sorted(df['date'].unique()):\n", " date_data = daily_hourly_stats[daily_hourly_stats['date'] == date]\n", " ax1.plot(date_data['hour'], date_data['cut_frac'], \n", " marker='o', linewidth=2, markersize=4, label=str(date), alpha=0.7)\n", "\n", "ax1.set_xlabel('Hour of Day', fontsize=12)\n", "ax1.set_ylabel('Cut Fraction', fontsize=12)\n", "ax1.set_title('Cut Fraction by Day and Hour', fontsize=14, fontweight='bold')\n", "ax1.grid(True, alpha=0.3)\n", "ax1.set_xlim(-0.5, 23.5)\n", "ax1.set_xticks(range(0, 24, 2))\n", "ax1.legend(bbox_to_anchor=(1.05, 1), loc='upper left', title='Date')\n", "\n", "# Plot 2: Heatmap\n", "pivot_table = daily_hourly_stats.pivot(index='date', columns='hour', values='cut_frac')\n", "im = ax2.imshow(pivot_table, aspect='auto', cmap='viridis', interpolation='nearest')\n", "\n", "ax2.set_xlabel('Hour of Day', fontsize=12)\n", "ax2.set_ylabel('Date', fontsize=12)\n", "ax2.set_title('Cut Fraction Heatmap', fontsize=14, fontweight='bold')\n", "ax2.set_xticks(range(24))\n", "ax2.set_xticklabels(range(24))\n", "ax2.set_yticks(range(len(pivot_table)))\n", "ax2.set_yticklabels([str(d) for d in pivot_table.index])\n", "\n", "# Add colorbar\n", "cbar = plt.colorbar(im, ax=ax2)\n", "cbar.set_label('Cut Fraction', fontsize=12)\n", "\n", "fig.tight_layout()\n", "plt.show()\n", "\n", "# Print summary stats\n", "print(f\"\\nSummary Statistics:\")\n", "print(f\"Overall cut_frac: {daily_hourly_stats['cut_frac'].mean():.3f}\")\n", "print(f\"Min cut_frac: {daily_hourly_stats['cut_frac'].min():.3f}\")\n", "print(f\"Max cut_frac: {daily_hourly_stats['cut_frac'].max():.3f}\")\n", "print(f\"Total samples: {daily_hourly_stats['count'].sum()}\")\n", "print(f\"\\nPer-day averages:\")\n", "day_avg = daily_hourly_stats.groupby('date')['cut_frac'].mean()\n", "for date, avg in day_avg.items():\n", " print(f\" {date}: {avg:.3f}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-22T06:32:56.204739Z", "iopub.status.idle": "2025-11-22T06:32:56.204873Z", "shell.execute_reply": "2025-11-22T06:32:56.204810Z", "shell.execute_reply.started": "2025-11-22T06:32:56.204803Z" } }, "outputs": [], "source": [ "df['date'] = pd.to_datetime(df['created_at']).dt.date # adjust column name\n", "# Group by date and calculate mean play duration\n", "avg_play_duration_by_day = df[df[\"preference\"]].groupby('date')['reaction_play_count'].mean().reset_index()\n", "\n", "# Or with more stats\n", "daily_stats = df.groupby('date').agg({\n", " 'reaction_play_count': ['mean', 'median', 'count', 'std']\n", "}).reset_index()\n", "\n", "print(avg_play_duration_by_day)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 4 }