{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:29:21.983203Z", "iopub.status.busy": "2025-07-14T18:29:21.983064Z", "iopub.status.idle": "2025-07-14T18:29:21.996571Z", "shell.execute_reply": "2025-07-14T18:29:21.996132Z", "shell.execute_reply.started": "2025-07-14T18:29:21.983186Z" } }, "outputs": [], "source": [ "# setup autoload\n", "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.040680Z", "start_time": "2024-05-16T13:58:19.777010Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:29:21.997288Z", "iopub.status.busy": "2025-07-14T18:29:21.997154Z", "iopub.status.idle": "2025-07-14T18:29:24.468285Z", "shell.execute_reply": "2025-07-14T18:29:24.467743Z", "shell.execute_reply.started": "2025-07-14T18:29:21.997275Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The autoreload extension is already loaded. To reload it, use:\n", " %reload_ext autoreload\n" ] } ], "source": [ "import ast\n", "import os\n", "import shutil\n", "import sys\n", "from collections import defaultdict\n", "\n", "import json\n", "import numpy as np\n", "import pandas as pd\n", "from preference_data_preparation_auk import *\n", "from preference_helper import *\n", "from sklearn.model_selection import train_test_split\n", "from suno_utils.utils.s3 import download_s3_files\n", "from suno_utils.utils.text import read_json, read_jsonl, write_json, write_jsonl\n", "from tqdm import tqdm\n", "\n", "pd.set_option(\"display.max_rows\", 500)\n", "pd.set_option(\"display.max_columns\", 500)\n", "pd.set_option(\"display.width\", 1000)\n", "\n", "# setup autoload\n", "%load_ext autoreload\n", "%autoreload 2\n", "\n", "\n", "def custom_parse(x):\n", " try:\n", " return json.loads(x)\n", " except:\n", " return {}" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.082172Z", "start_time": "2024-05-16T13:58:21.041926Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:29:24.469029Z", "iopub.status.busy": "2025-07-14T18:29:24.468832Z", "iopub.status.idle": "2025-07-14T18:29:24.535029Z", "shell.execute_reply": "2025-07-14T18:29:24.534567Z", "shell.execute_reply.started": "2025-07-14T18:29:24.469014Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "N_TOKENS_AUDIO 12000\n" ] } ], "source": [ "OUT_DATA_DIR = \"/app2/suno/data/dpo/auk_t1_v48\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "shutil.copyfile(\n", " \"/app/suno/data/dpo/7v_v20_full/tokenizer_60k.json\",\n", " os.path.join(OUT_DATA_DIR, \"tokenizer_60k.json\"),\n", ")\n", "NPZ_DIR = \"/app2/suno/data/dpo/auk_t1_npz\"\n", "N_TOKENS_AUDIO = 25 * 8 * 60\n", "print(\"N_TOKENS_AUDIO\", N_TOKENS_AUDIO)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:29:24.535662Z", "iopub.status.busy": "2025-07-14T18:29:24.535525Z", "iopub.status.idle": "2025-07-14T18:29:24.547979Z", "shell.execute_reply": "2025-07-14T18:29:24.547556Z", "shell.execute_reply.started": "2025-07-14T18:29:24.535648Z" } }, "outputs": [], "source": [ "# import glob\n", "# import os\n", "# from tqdm import tqdm\n", "\n", "# # Find all pkl files in the directory and sort by creation time\n", "# # pkl_files = glob.glob(\"/home/tony/Data/Preference/auk_t1/interesting_clips_*.pkl\")\n", "# # pkl_files.sort(key=lambda x: os.path.getctime(x))\n", "# pkl_files = [\n", "# \"/home/tony/Data/Preference/auk_t1/interesting_clips_auk_t1_20250623.pkl\",\n", "# # \"/home/tony/Data/Preference/auk_t1/interesting_clips_auk_t1_20250613.pkl\"\n", "# ]\n", "# # Initialize empty dataframe\n", "# df = pd.read_pickle(\n", "# \"/home/tony/Data/Preference/auk_t1/fully_merged_auk_t1.pkl\"\n", "# )\n", "\n", "# # Load each file and handle duplicates\n", "# for pkl_file in tqdm(pkl_files, desc=\"Loading pickle files\"):\n", "# print(f\"\\nProcessing {os.path.basename(pkl_file)}\")\n", "# prev_size = len(df)\n", "# try:\n", "# temp_df = pd.read_pickle(pkl_file)\n", "# except Exception as e:\n", "# print(f\"Error reading {pkl_file}: {str(e)}\")\n", "# continue\n", "# new_size = len(temp_df)\n", "\n", "# # Convert datetime columns if they exist\n", "# for col in [\"created_at\", \"updated_at\"]:\n", "# if col in temp_df.columns:\n", "# temp_df[col] = pd.to_datetime(temp_df[col], utc=True)\n", "\n", "# # If this is the first file, just use it\n", "# if df.empty:\n", "# df = temp_df\n", "# print(f\"First file loaded. Size: {len(df)}\")\n", "# continue\n", "\n", "# # For subsequent files, handle duplicates based on id\n", "# if \"id\" in temp_df.columns:\n", "# # Since files are sorted by creation time, we can simply update existing records\n", "# df = pd.concat([df, temp_df], ignore_index=True)\n", "# df = df.drop_duplicates(subset=[\"id\"], keep=\"last\")\n", "# else:\n", "# # If no id, just append\n", "# df = pd.concat([df, temp_df], ignore_index=True)\n", "\n", "# # Print size statistics\n", "# current_size = len(df)\n", "# net_increase = current_size - prev_size\n", "# print(f\"Previous size: {prev_size:,}\")\n", "# print(f\"New input size: {new_size:,}\")\n", "# print(f\"Current total size: {current_size:,}\")\n", "# print(f\"Net increase: {net_increase:,}\")\n", "\n", "# print(\"\\nFinal dataframe shape:\", df.shape)\n", "# print(\n", "# \"Unique ids:\",\n", "# df[\"id\"].nunique() if \"id\" in df.columns else \"No id column\",\n", "# )\n", "# df.to_pickle(\"/home/tony/Data/Preference/auk_t1/fully_merged_auk_t1.pkl\")" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:29:24.549471Z", "iopub.status.busy": "2025-07-14T18:29:24.549319Z", "iopub.status.idle": "2025-07-14T18:30:27.091479Z", "shell.execute_reply": "2025-07-14T18:30:27.090882Z", "shell.execute_reply.started": "2025-07-14T18:29:24.549457Z" } }, "outputs": [], "source": [ "# df = pd.read_pickle(\n", "# \"/home/tony/Data/Preference/auk_t1/fully_merged_auk_t1.pkl\"\n", "# )\n", "df = pd.read_pickle(\n", " \"/home/tony/Data/Preference/auk_t1/interesting_clips_auk_t1_20250714.pkl\"\n", ")" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:30:27.092265Z", "iopub.status.busy": "2025-07-14T18:30:27.092101Z", "iopub.status.idle": "2025-07-14T18:30:27.912526Z", "shell.execute_reply": "2025-07-14T18:30:27.912000Z", "shell.execute_reply.started": "2025-07-14T18:30:27.092241Z" } }, "outputs": [], "source": [ "# # Load each file and handle duplicates\n", "# for pkl_file in tqdm([\"/home/tony/Data/Preference/auk_t1/interesting_clips_auk_t1_20250610.pkl\"], desc=\"Loading pickle files\"):\n", "# print(f\"\\nProcessing {os.path.basename(pkl_file)}\")\n", "# prev_size = len(df)\n", "# try:\n", "# temp_df = pd.read_pickle(pkl_file)\n", "# except Exception as e:\n", "# print(f\"Error reading {pkl_file}: {str(e)}\")\n", "# continue\n", "# new_size = len(temp_df)\n", "\n", "# # Convert datetime columns if they exist\n", "# for col in [\"created_at\", \"updated_at\"]:\n", "# if col in temp_df.columns:\n", "# temp_df[col] = pd.to_datetime(temp_df[col], utc=True)\n", "\n", "# # If this is the first file, just use it\n", "# if df.empty:\n", "# df = temp_df\n", "# print(f\"First file loaded. Size: {len(df)}\")\n", "# continue\n", "\n", "# # For subsequent files, handle duplicates based on id\n", "# if \"id\" in temp_df.columns:\n", "# # Since files are sorted by creation time, we can simply update existing records\n", "# df = pd.concat([df, temp_df], ignore_index=True)\n", "# df = df.drop_duplicates(subset=[\"id\"], keep=\"last\")\n", "# else:\n", "# # If no id, just append\n", "# df = pd.concat([df, temp_df], ignore_index=True)\n", "\n", "# # Print size statistics\n", "# current_size = len(df)\n", "# net_increase = current_size - prev_size\n", "# print(f\"Previous size: {prev_size:,}\")\n", "# print(f\"New input size: {new_size:,}\")\n", "# print(f\"Current total size: {current_size:,}\")\n", "# print(f\"Net increase: {net_increase:,}\")\n", "\n", "# print(\"\\nFinal dataframe shape:\", df.shape)\n", "# print(\n", "# \"Unique ids:\",\n", "# df[\"id\"].nunique() if \"id\" in df.columns else \"No id column\",\n", "# )\n", "# df.to_pickle(\"/home/tony/Data/Preference/auk_t1/fully_merged_auk_t1.pkl\")" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:30:27.913220Z", "iopub.status.busy": "2025-07-14T18:30:27.913076Z", "iopub.status.idle": "2025-07-14T18:30:27.927128Z", "shell.execute_reply": "2025-07-14T18:30:27.926668Z", "shell.execute_reply.started": "2025-07-14T18:30:27.913205Z" } }, "outputs": [], "source": [ "# df.to_pickle(\"/home/tony/Data/Preference/auk_t1/fully_merged_auk_t1.pkl\")" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:53.962528Z", "start_time": "2024-05-16T13:58:21.105919Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:30:27.927841Z", "iopub.status.busy": "2025-07-14T18:30:27.927698Z", "iopub.status.idle": "2025-07-14T18:30:27.939115Z", "shell.execute_reply": "2025-07-14T18:30:27.938699Z", "shell.execute_reply.started": "2025-07-14T18:30:27.927826Z" } }, "outputs": [], "source": [ "# df = pd.read_pickle(\n", "# \"/home/tony/Data/Preference/auk_t1/fully_merged_auk_t1.pkl\"\n", "# )\n", "# print(\"Preference data shape\", df.shape)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:30:27.939847Z", "iopub.status.busy": "2025-07-14T18:30:27.939714Z", "iopub.status.idle": "2025-07-14T18:30:27.950691Z", "shell.execute_reply": "2025-07-14T18:30:27.950277Z", "shell.execute_reply.started": "2025-07-14T18:30:27.939833Z" } }, "outputs": [], "source": [ "# df[\"created_at\"] = pd.to_datetime(df[\"created_at\"], utc=True)\n", "# cutoff_date = pd.to_datetime(\"2025-05-10\", utc=True)\n", "# # cutoff_date = pd.to_datetime(\"2025-04-17\", utc=True)\n", "# print(df.shape, df[df[\"created_at\"] > cutoff_date].shape)\n", "# df = df[(df[\"created_at\"] > cutoff_date)].copy()\n", "# print(\"after date cut\", df.shape)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:30:27.951276Z", "iopub.status.busy": "2025-07-14T18:30:27.951147Z", "iopub.status.idle": "2025-07-14T18:30:31.839035Z", "shell.execute_reply": "2025-07-14T18:30:31.838473Z", "shell.execute_reply.started": "2025-07-14T18:30:27.951263Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "after dropna (2593532, 85)\n" ] } ], "source": [ "df = df.dropna(axis=1, how=\"all\")\n", "print(\"after dropna\", df.shape)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.199480Z", "start_time": "2024-05-16T13:58:53.963687Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:30:31.839759Z", "iopub.status.busy": "2025-07-14T18:30:31.839604Z", "iopub.status.idle": "2025-07-14T18:34:47.350824Z", "shell.execute_reply": "2025-07-14T18:34:47.350288Z", "shell.execute_reply.started": "2025-07-14T18:30:31.839743Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "12184904\n", "12184904\n", "pre-downloaded df (2593532, 85)\n", "downloaded df (2593530, 85)\n" ] } ], "source": [ "converted_paths = os.listdir(NPZ_DIR)\n", "print(len(converted_paths))\n", "\n", "converted_paths = set([f.replace(\".npz\", \"\") for f in converted_paths])\n", "print(len(converted_paths))\n", "\n", "print(\"pre-downloaded df\", df.shape)\n", "df[df[\"s3_id\"].isin(converted_paths)].shape\n", "df = df[df[\"s3_id\"].isin(converted_paths)].copy()\n", "print(\"downloaded df\", df.shape)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.467253Z", "start_time": "2024-05-16T13:58:56.207647Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:34:47.351586Z", "iopub.status.busy": "2025-07-14T18:34:47.351349Z", "iopub.status.idle": "2025-07-14T18:34:47.484465Z", "shell.execute_reply": "2025-07-14T18:34:47.484030Z", "shell.execute_reply.started": "2025-07-14T18:34:47.351570Z" } }, "outputs": [ { "data": { "text/plain": [ "task\n", " 1672149\n", "cover 523978\n", "artist_consistency 212597\n", "artist_cover 91364\n", "extend 54254\n", "upload_extend 29740\n", "artist_extend 9426\n", "underpainting 16\n", "overpainting 6\n", "Name: count, dtype: int64" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"task\"].value_counts()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# LET's do the data prep" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.592883Z", "start_time": "2024-05-16T13:58:56.470781Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:34:47.485118Z", "iopub.status.busy": "2025-07-14T18:34:47.484980Z", "iopub.status.idle": "2025-07-14T18:34:50.122369Z", "shell.execute_reply": "2025-07-14T18:34:50.121807Z", "shell.execute_reply.started": "2025-07-14T18:34:47.485104Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "preference model_name \n", "False chirp-auk-t1 1296764\n", "True chirp-auk-t1 1296766\n", "Name: count, dtype: int64\n", "before filter on model name (2593530, 85)\n", "after filter on model name (2593530, 85)\n", "is_public\n", "False 2463027\n", "True 130503\n", "Name: count, dtype: int64\n" ] } ], "source": [ "## for 13b this is easy for now\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "print(\"before filter on model name\", df.shape)\n", "df = df[df[\"model_name\"].isin([\"chirp-auk-t1\"])]\n", "# df = df[df[\"model_name\"].isin([\"chirp-v3p5-engine-t-6\"])]\n", "print(\"after filter on model name\", df.shape)\n", "print(df[\"is_public\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.909539Z", "start_time": "2024-05-16T13:58:56.595736Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:34:50.123079Z", "iopub.status.busy": "2025-07-14T18:34:50.122931Z", "iopub.status.idle": "2025-07-14T18:34:55.128204Z", "shell.execute_reply": "2025-07-14T18:34:55.127586Z", "shell.execute_reply.started": "2025-07-14T18:34:50.123063Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "before filter on request id pairs (2593530, 85)\n", "after filter on request id pairs (2593528, 85)\n", "preference model_name \n", "False chirp-auk-t1 1296764\n", "True chirp-auk-t1 1296764\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(\"before filter on request id pairs\", df.shape)\n", "df = df[\n", " df[\"request_id\"].isin(\n", " df[\"request_id\"].value_counts().index[df[\"request_id\"].value_counts() == 2]\n", " )\n", "]\n", "print(\"after filter on request id pairs\", df.shape)\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "assert df.shape[0] == df[\"request_id\"].nunique() * 2" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:34:55.128974Z", "iopub.status.busy": "2025-07-14T18:34:55.128818Z", "iopub.status.idle": "2025-07-14T18:43:17.736513Z", "shell.execute_reply": "2025-07-14T18:43:17.735679Z", "shell.execute_reply.started": "2025-07-14T18:34:55.128959Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "unique_requests 1296764\n", "before removing duplicates (2593528, 186)\n", "after removing duplicates (2593528, 179)\n" ] } ], "source": [ "# Let's use the old selection for now -- for quality assurance\n", "# expand the metadata columns -- this takes forever...~ 6 mins\n", "# test_slice = df[\"metadata\"].apply(lambda x: ast.literal_eval(str(x)))\n", "# test_slice = df[\"metadata\"].apply(lambda x: custom_parse(x))\n", "test_slice = df[\"metadata\"] # .apply(lambda x: custom_parse(x))\n", "test_slice_series = test_slice.apply(pd.Series)\n", "df = pd.concat([df, test_slice_series], axis=1, join=\"inner\")\n", "print(\"unique_requests\", df[\"request_id\"].nunique())\n", "# remove the duplicates\n", "print(\"before removing duplicates\", df.shape)\n", "df = df.loc[:, ~df.columns.duplicated()].copy()\n", "print(\"after removing duplicates\", df.shape)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:43:17.737365Z", "iopub.status.busy": "2025-07-14T18:43:17.737195Z", "iopub.status.idle": "2025-07-14T18:43:25.110103Z", "shell.execute_reply": "2025-07-14T18:43:25.109479Z", "shell.execute_reply.started": "2025-07-14T18:43:17.737348Z" } }, "outputs": [ { "data": { "text/plain": [ "pos_diff_preference\n", "1.0 872191\n", "2.0 424573\n", "Name: count, dtype: int64" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = df.sort_values(by=[\"request_id\", \"preference\", \"diff_preference\"])\n", "df[\"pos_diff_preference\"] = df[\"diff_preference\"].diff()\n", "# df[\"cer_diff_preference\"] = df[\"cer\"].diff()\n", "df[df[\"preference\"]][\"pos_diff_preference\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:43:25.110988Z", "iopub.status.busy": "2025-07-14T18:43:25.110823Z", "iopub.status.idle": "2025-07-14T18:43:27.149523Z", "shell.execute_reply": "2025-07-14T18:43:27.148704Z", "shell.execute_reply.started": "2025-07-14T18:43:25.110971Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive param_experiment\n", "mask_control_slider 26848\n", "cfg_steps_240 21528\n", "n_tag_2 21426\n", "cfg_steps_60 21352\n", "temp_s_85 21223\n", "tag_cfg_1 21192\n", "n_tag_1 21190\n", "temp_s_80 21032\n", "temp_s_95 20956\n", "cfg_steps_10 20709\n", "tag_cfg_3 20404\n", "Name: count, dtype: int64\n" ] } ], "source": [ "try:\n", " print(\"positive\", df[df[\"preference\"]][\"param_experiment\"].value_counts())\n", "except:\n", " pass" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:43:42.025523Z", "iopub.status.busy": "2025-07-14T18:43:42.025121Z", "iopub.status.idle": "2025-07-14T18:44:23.898145Z", "shell.execute_reply": "2025-07-14T18:44:23.897403Z", "shell.execute_reply.started": "2025-07-14T18:43:42.025502Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Found 83910 duplicated prompts 41955 unique requests\n", "Found 19768 request_ids with duplicate prompts but not highest play counts in their group\n", "['17e29d69-1e92-4304-a9f5-d3a60c62b006', '3d19c8b1-edc1-4b0c-90e5-39dc4679cff2', '6deaeb48-8c9b-4964-af81-fd5ef0aaf20a', 'd23d6833-867c-482a-893c-8fb546590bfd', '1a68ecf7-416f-4050-9978-c8cc45261f4a', 'cd6c8c25-3705-4276-b77a-f0d6b24220cc', '39e11da3-7ff1-4ef7-8b3c-89924d1be3a5', 'd98333d7-9762-40ee-8622-5d966d2546cb', '80a711f8-c2e1-4508-bd95-a2e0289ca2a3', '839fcac3-4a41-49f5-a8e0-5b473963468c']\n", "Before dedup user gen requests 2593528\n", "After dedup user gen requests 2593528\n" ] } ], "source": [ "# Find duplicated prompts with count > 2\n", "duplicate_entries = df.groupby(\n", " [\"user_id\", \"prompt_text\", \"tags\", \"task\", \"edited_clip_id\"]\n", ").filter(lambda x: len(x) > 2)\n", "print(\n", " \"Found\",\n", " len(duplicate_entries),\n", " \"duplicated prompts\",\n", " len(duplicate_entries[\"request_id\"].unique()),\n", " \"unique requests\",\n", ")\n", "\n", "# Group by user_id, prompt_text, and tags to find duplicate prompt groups\n", "prompt_groups = duplicate_entries.groupby(\n", " [\"user_id\", \"prompt_text\", \"tags\", \"task\", \"edited_clip_id\"]\n", ")\n", "\n", "# For each prompt group, find the request_id with the highest total reaction_play_count\n", "low_play_count_request_ids = []\n", "for prompt_key, prompt_group in prompt_groups:\n", " # Get the sum of reaction_play_count for each request_id in this group\n", " request_play_counts = prompt_group.groupby(\"request_id\")[\n", " \"reaction_play_count\"\n", " ].sum()\n", "\n", " # Find the max play count in this group\n", " max_play_count = request_play_counts.max()\n", "\n", " # Add request_ids that don't have the max play count to our filter list\n", " lower_play_count_request_ids = request_play_counts[\n", " request_play_counts < max_play_count\n", " ].index.tolist()\n", " low_play_count_request_ids.extend(lower_play_count_request_ids)\n", "\n", "# Display the filtered request IDs\n", "print(\n", " f\"Found {len(low_play_count_request_ids)} request_ids with duplicate prompts but not highest play counts in their group\"\n", ")\n", "print(\n", " low_play_count_request_ids[:10]\n", " if len(low_play_count_request_ids) > 10\n", " else low_play_count_request_ids\n", ")\n", "print(\"Before dedup user gen requests\", df.shape[0])\n", "# df = df[~df[\"request_id\"].isin(low_play_count_request_ids)]\n", "print(\"After dedup user gen requests\", df.shape[0])" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:40.799375Z", "start_time": "2024-05-16T13:59:36.394236Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:44:23.899786Z", "iopub.status.busy": "2025-07-14T18:44:23.899281Z", "iopub.status.idle": "2025-07-14T18:44:39.457052Z", "shell.execute_reply": "2025-07-14T18:44:39.456312Z", "shell.execute_reply.started": "2025-07-14T18:44:23.899766Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "35704\n", "good_continue_at\n", "True 2588434\n", "False 5094\n", "Name: count, dtype: int64\n", "\n", " Check some basics... \n", " preference\n", "False 1296764\n", "True 1296764\n", "Name: count, dtype: int64 model_name\n", "chirp-auk-t1 2593528\n", "Name: count, dtype: int64 preference model_name \n", "False chirp-auk-t1 1296764\n", "True chirp-auk-t1 1296764\n", "Name: count, dtype: int64\n", "task\n", " 1672148\n", "cover 523978\n", "artist_consistency 212596\n", "artist_cover 91364\n", "extend 54254\n", "upload_extend 29740\n", "artist_extend 9426\n", "underpainting 16\n", "overpainting 6\n", "Name: count, dtype: int64\n" ] } ], "source": [ "df[\"id\"] = df[\"str_id\"]\n", "# get the original duration of the clips, if they are concacted\n", "df[\"original_duration_s\"] = df[\"total_start_s\"] + df[\"duration\"]\n", "# classify the continue at behavoirs by the duration choice\n", "audio_prompt_id_to_continue_at = {}\n", "\n", "for _, row in df[~df[\"continued_parent\"].isna()].iterrows():\n", " audio_prompt_id = row[\"continued_parent\"]\n", " if audio_prompt_id not in audio_prompt_id_to_continue_at:\n", " audio_prompt_id_to_continue_at[audio_prompt_id] = row[\"continue_at\"]\n", " else:\n", " # pick the max\n", " audio_prompt_id = max(\n", " audio_prompt_id_to_continue_at[audio_prompt_id], row[\"continue_at\"]\n", " )\n", "print(len(audio_prompt_id_to_continue_at))\n", "df[\"has_continue_and_start_continue_at\"] = df[\"id\"].apply(\n", " lambda x: audio_prompt_id_to_continue_at.get(x)\n", ")\n", "# we want continue at to be at most of the clip...\n", "df[\"good_continue_at\"] = (\n", " (df[\"has_continue_and_start_continue_at\"] / df[\"duration\"]) > 0.9\n", ") | df[\"has_continue_and_start_continue_at\"].isna()\n", "print(df[\"good_continue_at\"].value_counts())\n", "\n", "\n", "print(\n", " \"\\n Check some basics... \\n\",\n", " df[\"preference\"].value_counts(),\n", " df[\"model_name\"].value_counts(),\n", " df.groupby([\"preference\"])[\"model_name\"].value_counts(),\n", ")\n", "\n", "df = df.sort_values(by=[\"request_id\", \"preference\"])\n", "df[\"duration_rel_diff\"] = df[\"duration\"].diff()\n", "df[\"play_rel_diff\"] = df[\"reaction_play_count\"].diff()\n", "print(df[\"task\"].value_counts())\n", "\n", "# df[\"post_infill_duration\"] = (\n", "# df[\"duration\"]\n", "# + df[\"infill_context_end_s\"]\n", "# - df[\"infill_context_start_s\"]\n", "# - df[\"include_future_s\"]\n", "# - df[\"include_history_s\"]\n", "# - df[\"infill_dur_s\"]\n", "# )" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.035167Z", "start_time": "2024-05-16T13:59:40.801098Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:44:39.458063Z", "iopub.status.busy": "2025-07-14T18:44:39.457888Z", "iopub.status.idle": "2025-07-14T18:44:49.197188Z", "shell.execute_reply": "2025-07-14T18:44:49.196545Z", "shell.execute_reply.started": "2025-07-14T18:44:39.458046Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "after duration 0.999470219716155\n", "after infill duration 1.0\n", "neg_filter_reaction_play_count 1.0\n", "neg_filter_upvote_count 0.9902\n", "neg_filter_norm_play_frac 1.0\n", "neg_filter_continues 1.0\n", "----------------\n", "pos_filter_continues 0.9961\n", "pos_filter_reaction_play_count 1.0\n", "pos_filter_relative_play_count 0.9776\n", "pos_filter_cer_diff_preference 1.0\n", "pos_filter_bad_flags 0.9998\n", "after filter on play counts 0.7096\n", "after filter on higher quality 0.2986\n", "----------------\n", "negative 1283323 positive 297414\n", "----------------\n", "total pair requests 1296764 --> selected pair requests 294407 frac 0.227 --> total intitial users 202586\n" ] } ], "source": [ "normal_pos_play_count = 5\n", "# this is lower, cause a concat is probably already ensuring that it is good\n", "concat_pos_play_count = 2\n", "# this is a filter on the concated clip\n", "concat_total_play_count = 5\n", "\n", "all_fitlers = (df[\"duration\"] >= 10) & (df[\"duration\"] <= 480)\n", "print(\"after duration\", all_fitlers.sum() / df.shape[0])\n", "infill_duration_filter = ~df[\"task\"].isin(\n", " [\n", " \"infill\",\n", " \"infill_intro\",\n", " \"infill_outro\",\n", " ]\n", ") # | (df[\"post_infill_duration\"] <= 239)\n", "print(\"after infill duration\", infill_duration_filter.sum() / df.shape[0])\n", "# negative fitlers\n", "total_negative = df[~df[\"preference\"]].shape[0]\n", "neg_filter_reaction_play_count = (~df[\"preference\"]) & (df[\"reaction_play_count\"] >= 1)\n", "print(\n", " \"neg_filter_reaction_play_count\",\n", " round(neg_filter_reaction_play_count.sum() / total_negative, 4),\n", ")\n", "neg_filter_upvote_count = (~df[\"preference\"]) & (df[\"upvote_count\"] == 0)\n", "print(\n", " \"neg_filter_upvote_count\",\n", " round(neg_filter_upvote_count.sum() / total_negative, 4),\n", ")\n", "neg_filter_norm_play_frac = (~df[\"preference\"]) & (df[\"norm_play_frac\"] <= 3.1)\n", "print(\n", " \"neg_filter_norm_play_frac\",\n", " round(neg_filter_norm_play_frac.sum() / total_negative, 4),\n", ")\n", "neg_filter_continues = (~df[\"preference\"]) & (\n", " df[\"has_continue_and_start_continue_at\"].isna()\n", ")\n", "print(\n", " \"neg_filter_continues\",\n", " round(neg_filter_continues.sum() / total_negative, 4),\n", ")\n", "\n", "neg_filter_selection_mask = (\n", " all_fitlers\n", " & infill_duration_filter\n", " & neg_filter_reaction_play_count\n", " & neg_filter_upvote_count\n", " & neg_filter_norm_play_frac\n", " & neg_filter_continues\n", ")\n", "\n", "print(\"----------------\")\n", "total_positive = df[df[\"preference\"]].shape[0]\n", "assert total_positive == total_negative\n", "pos_filter_continues = (df[\"preference\"]) & (df[\"good_continue_at\"])\n", "print(\"pos_filter_continues\", round(pos_filter_continues.sum() / total_positive, 4))\n", "pos_filter_reaction_play_count = (df[\"preference\"]) & (df[\"reaction_play_count\"] >= 1)\n", "print(\n", " \"pos_filter_reaction_play_count\",\n", " round(pos_filter_reaction_play_count.sum() / total_positive, 4),\n", ")\n", "pos_filter_relative_play_count = (df[\"preference\"]) & (df[\"play_rel_diff\"] >= 0)\n", "print(\n", " \"pos_filter_relative_play_count\",\n", " round(pos_filter_relative_play_count.sum() / total_positive, 4),\n", ")\n", "pos_filter_cer_diff_preference = (\n", " df[\n", " \"preference\"\n", " ] # & (df[\"pos_diff_preference\"] == 2) # & (df[\"cer_diff_preference\"] < 0.5) & (df[\"cer\"] < 0.99)\n", ")\n", "print(\n", " \"pos_filter_cer_diff_preference\",\n", " round(pos_filter_cer_diff_preference.sum() / total_positive, 4),\n", ")\n", "pos_filter_bad_flags = (\n", " (df[\"preference\"]) & (df[\"flag_count\"] == 0) & (df[\"dislike_count\"] == 0)\n", ")\n", "print(\n", " \"pos_filter_bad_flags\",\n", " round(pos_filter_bad_flags.sum() / total_positive, 4),\n", ")\n", "pos_filter_play_counts = (df[\"preference\"]) & (\n", " (\n", " (df[\"part_of_concat\"])\n", " & (df[\"reaction_play_count\"] >= concat_pos_play_count)\n", " & (df[\"concat_play_counts\"] >= concat_total_play_count)\n", " )\n", " | (\n", " (~df[\"part_of_concat\"]) & (df[\"reaction_play_count\"] >= normal_pos_play_count)\n", " # & (df[\"norm_play_frac\"] >= 2.1) # this is a bit of a luxury cut...\n", " )\n", " | (df[\"task\"].isin([\"infill\", \"infill_intro\", \"infill_outro\"]))\n", ")\n", "print(\n", " \"after filter on play counts\",\n", " round(pos_filter_play_counts.sum() / total_positive, 4),\n", ")\n", "high_quality_tasks_filter = (\n", " (\n", " df[\"task\"].isin(\n", " [\n", " \"cover\",\n", " \"upload_extend\",\n", " \"cover_extend\",\n", " \"artist_cover\",\n", " \"extend\",\n", " \"artist_consistency\",\n", " \"artist_extend\",\n", " \"\",\n", " ]\n", " )\n", " )\n", " & (\n", " (df[\"upvote_count\"] >= 1) # (df[\"upvote_count\"] >= 1)\n", " | (df[\"reaction_play_count\"] >= 5)\n", " | (df[\"concat_play_counts\"] >= 5)\n", " )\n", " & (\n", " (df[\"part_of_concat\"])\n", " | (\n", " (~df[\"part_of_concat\"])\n", " & (df[\"norm_play_frac\"] >= 5.1) # this is a bit of a luxury cut...\n", " & (\n", " df[\"norm_play_frac\"] >= df[\"reaction_play_count\"] / 3\n", " ) # play duration is not low on average\n", " )\n", " )\n", ")\n", "medium_quality_tasks_filter = (\n", " df[\"task\"].isin(\n", " [\n", " \"infill\",\n", " \"infill_intro\",\n", " \"infill_outro\",\n", " ]\n", " )\n", ") & ( # let more infill through only in this case...\n", " (\n", " df[\"upvote_count\"] >= 0\n", " ) # (df[\"upvote_count\"] >= 1) (df[\"pos_diff_preference\"] == 2)\n", " | (df[\"reaction_play_count\"] >= 1)\n", " | (df[\"concat_play_counts\"] >= 1)\n", ")\n", "pos_filter_higher_quality = (df[\"preference\"]) & (\n", " high_quality_tasks_filter | medium_quality_tasks_filter\n", ")\n", "print(\n", " \"after filter on higher quality\",\n", " round(pos_filter_higher_quality.sum() / total_positive, 4),\n", ")\n", "\n", "user_gen_filter = (\n", " df[\"user_n_clips\"] >= 40\n", ") # user needs to have genereated at least 100 over the time period\n", "\n", "print(\"----------------\")\n", "pos_filter_selectin_mask = (\n", " (df[\"preference\"]) # get basics aligned\n", " & all_fitlers\n", " & infill_duration_filter\n", " & pos_filter_continues\n", " & pos_filter_reaction_play_count\n", " & pos_filter_relative_play_count\n", " & pos_filter_cer_diff_preference\n", " & pos_filter_bad_flags\n", " & pos_filter_play_counts\n", " & pos_filter_higher_quality\n", " & user_gen_filter\n", ")\n", "print(\n", " \"negative\",\n", " sum(neg_filter_selection_mask),\n", " \"positive\",\n", " sum(pos_filter_selectin_mask),\n", ")\n", "\n", "neg_filter_requests = df[neg_filter_selection_mask][\"request_id\"].unique()\n", "pos_filter_requests = df[pos_filter_selectin_mask][\"request_id\"].unique()\n", "# looking for very strong signal here:\n", "# listen to the positive/negative more than once\n", "# disliked one of the clips\n", "unique_requests = set(pos_filter_requests).intersection(neg_filter_requests)\n", "print(\"----------------\")\n", "print(\n", " \"total pair requests\",\n", " df[\"request_id\"].nunique(),\n", " \" --> selected pair requests\",\n", " len(unique_requests),\n", " f\"frac {len(unique_requests) / df['request_id'].nunique():.3f}\",\n", " \" --> total intitial users\",\n", " df[\"user_id\"].nunique(),\n", ")" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.250737Z", "start_time": "2024-05-16T13:59:41.036434Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:44:49.198016Z", "iopub.status.busy": "2025-07-14T18:44:49.197848Z", "iopub.status.idle": "2025-07-14T18:44:52.105246Z", "shell.execute_reply": "2025-07-14T18:44:52.104627Z", "shell.execute_reply.started": "2025-07-14T18:44:49.197999Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "auk_t1_v48 requests 294407 clips 588814 total khrs 29.992; N gpus for 1000 iters 36.801; 4 gpus for x iters 9200.219; n unique users 80991 n pro users 78825\n" ] } ], "source": [ "df_slice = df[df[\"request_id\"].isin(set(unique_requests))].copy()\n", "print(\n", " f\"{os.path.basename(OUT_DATA_DIR)} requests\",\n", " df_slice[\"request_id\"].nunique(),\n", " \"clips\",\n", " df_slice.shape[0],\n", " f\"total khrs {sum(df_slice['duration'] / 3600 / 1000):.3f};\",\n", " f\"N gpus for 1000 iters {df_slice.shape[0] / 8 / 2 / 1000:.3f};\",\n", " f\"4 gpus for x iters {df_slice.shape[0] / 8 / 2 / 4:.3f};\",\n", " f\"n unique users {df_slice['user_id'].nunique()}\",\n", " f\"n pro users {df_slice[df_slice['is_pro_user']]['user_id'].nunique()}\",\n", ")\n", "# auk_mix_t1_v2 requests 102002 clips 204004 total khrs 9.191; N gpus for 1000 iters 12.750; 4 gpus for x iters 3187.562; n unique users 36408 n pro users 34038\n", "# auk_t1_v1 requests 9179 clips 18358 total khrs 0.854; N gpus for 1000 iters 1.147; 4 gpus for x iters 286.844; n unique users 6288 n pro users 6275\n", "# auk_t1_v2 requests 40903 clips 81806 total khrs 3.864; N gpus for 1000 iters 5.113; 4 gpus for x iters 1278.219; n unique users 21079 n pro users 20966\n", "# auk_t1_v3 requests 102015 clips 204030 total khrs 9.703; N gpus for 1000 iters 12.752; 4 gpus for x iters 3187.969; n unique users 42837 n pro users 42462\n", "# auk_t1_v4 requests 211452 clips 422904 total khrs 20.216; N gpus for 1000 iters 26.431; 4 gpus for x iters 6607.875; n unique users 71182 n pro users 70059\n", "# auk_t1_v9 requests 547743 clips 1095486 total khrs 56.567; N gpus for 1000 iters 68.468; 4 gpus for x iters 17116.969; n unique users 129354 n pro users 125126\n", "# auk_t1_v13 requests 627646 clips 1255292 total khrs 64.530; N gpus for 1000 iters 78.456; 4 gpus for x iters 19613.938; n unique users 139873 n pro users 134738\n", "# auk_t1_v17 requests 494902 clips 989804 total khrs 51.445; N gpus for 1000 iters 61.863; 4 gpus for x iters 15465.688; n unique users 114240 n pro users 109708\n", "# auk_t1_v19 requests 740881 clips 1481762 total khrs 76.133; N gpus for 1000 iters 92.610; 4 gpus for x iters 23152.531; n unique users 154570 n pro users 147261\n", "# auk_t1_v24 requests 717745 clips 1435490 total khrs 74.515; N gpus for 1000 iters 89.718; 4 gpus for x iters 22429.531; n unique users 139344 n pro users 130740\n", "# auk_t1_v29 requests 1097586 clips 2195172 total khrs 112.614; N gpus for 1000 iters 137.198; 4 gpus for x iters 34299.562; n unique users 193138 n pro users 179429\n", "# auk_t1_v30 requests 1224422 clips 2448844 total khrs 125.330; N gpus for 1000 iters 153.053; 4 gpus for x iters 38263.188; n unique users 203138 n pro users 186758\n", "# auk_t1_v31 requests 389265 clips 778530 total khrs 40.761; N gpus for 1000 iters 48.658; 4 gpus for x iters 12164.531; n unique users 85198 n pro users 78445\n", "# auk_t1_v33 requests 1285261 clips 2570522 total khrs 131.585; N gpus for 1000 iters 160.658; 4 gpus for x iters 40164.406; n unique users 208932 n pro users 191380\n", "# auk_t1_v33 requests 1086639 clips 2173278 total khrs 110.641; N gpus for 1000 iters 135.830; 4 gpus for x iters 33957.469; n unique users 197293 n pro users 180968 -- play dur from /3 to /2\n", "# auk_t1_v33 requests 806877 clips 1613754 total khrs 81.964; N gpus for 1000 iters 100.860; 4 gpus for x iters 25214.906; n unique users 156679 n pro users 144253 -- filter to web\n", "# auk_t1_v37 requests 1339132 clips 2678264 total khrs 137.055; N gpus for 1000 iters 167.392; 4 gpus for x iters 41847.875; n unique users 214313 n pro users 196815\n", "# auk_t1_v38 requests 979451 clips 1958902 total khrs 102.038; N gpus for 1000 iters 122.431; 4 gpus for x iters 30607.844; n unique users 170773 n pro users 156737\n", "# auk_t1_v43 requests 184775 clips 369550 total khrs 19.089; N gpus for 1000 iters 23.097; 4 gpus for x iters 5774.219; n unique users 60652 n pro users 59126\n", "# auk_t1_v45 requests 1176216 clips 2352432 total khrs 122.102; N gpus for 1000 iters 147.027; 4 gpus for x iters 36756.750; n unique users 176751 n pro users 163097\n", "# auk_t1_v48 requests 294407 clips 588814 total khrs 29.992; N gpus for 1000 iters 36.801; 4 gpus for x iters 9200.219; n unique users 80991 n pro users 78825" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:44:52.106744Z", "iopub.status.busy": "2025-07-14T18:44:52.106487Z", "iopub.status.idle": "2025-07-14T18:44:52.249518Z", "shell.execute_reply": "2025-07-14T18:44:52.249052Z", "shell.execute_reply.started": "2025-07-14T18:44:52.106726Z" } }, "outputs": [ { "data": { "text/plain": [ "source\n", "web 461388\n", "android 75382\n", "ios 52044\n", "Name: count, dtype: int64" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_slice[\"source\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.277006Z", "start_time": "2024-05-16T13:59:41.252105Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:44:52.250187Z", "iopub.status.busy": "2025-07-14T18:44:52.250040Z", "iopub.status.idle": "2025-07-14T18:44:52.503749Z", "shell.execute_reply": "2025-07-14T18:44:52.503166Z", "shell.execute_reply.started": "2025-07-14T18:44:52.250172Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive in playlist (87291, 185)\n", "task\n", " 392074\n", "cover 99270\n", "artist_consistency 51500\n", "artist_cover 20804\n", "extend 17038\n", "upload_extend 4848\n", "artist_extend 3280\n", "Name: count, dtype: int64\n" ] } ], "source": [ "test_mask = (df_slice[\"preference\"]) & (\n", " (df_slice[\"is_in_playlist\"]) | (df_slice[\"concat_in_playlist\"])\n", ")\n", "print(\"positive in playlist\", df_slice[test_mask].shape)\n", "print(df_slice[\"task\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:44:52.504518Z", "iopub.status.busy": "2025-07-14T18:44:52.504360Z", "iopub.status.idle": "2025-07-14T18:44:52.523689Z", "shell.execute_reply": "2025-07-14T18:44:52.523201Z", "shell.execute_reply.started": "2025-07-14T18:44:52.504503Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "is_public\n", "False 536153\n", "True 52661\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(df_slice[\"is_public\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:44:52.524408Z", "iopub.status.busy": "2025-07-14T18:44:52.524266Z", "iopub.status.idle": "2025-07-14T18:44:52.677286Z", "shell.execute_reply": "2025-07-14T18:44:52.676729Z", "shell.execute_reply.started": "2025-07-14T18:44:52.524393Z" } }, "outputs": [], "source": [ "df_slice[\"npz_path\"] = df_slice[\"s3_id\"].map(lambda x: f\"{NPZ_DIR}/{x}.npz\")" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:44:52.678017Z", "iopub.status.busy": "2025-07-14T18:44:52.677867Z", "iopub.status.idle": "2025-07-14T18:44:52.693257Z", "shell.execute_reply": "2025-07-14T18:44:52.692787Z", "shell.execute_reply.started": "2025-07-14T18:44:52.678002Z" } }, "outputs": [], "source": [ "# tr_metas_t1_v7 = read_jsonl(\n", "# os.path.join(\"/app2/suno/data/dpo/auk_t1_v33\", f\"meta_tr.jsonl\")\n", "# )\n", "# tr_metas_t1_v6 = read_jsonl(\n", "# os.path.join(\"/app2/suno/data/dpo/auk_t1_v34\", f\"meta_tr.jsonl\")\n", "# )\n", "# tr_metas_t1_v19 = read_jsonl(\n", "# os.path.join(\"/app2/suno/data/dpo/auk_t1_v35\", f\"meta_tr.jsonl\")\n", "# )\n", "# known_train_ids = set()\n", "# for prev_tr_meta in tr_metas_t1_v7:\n", "# known_train_ids.add(prev_tr_meta[\"id\"])\n", "# for prev_tr_meta in tr_metas_t1_v6:\n", "# known_train_ids.add(prev_tr_meta[\"id\"])\n", "# for prev_tr_meta in tr_metas_t1_v19:\n", "# known_train_ids.add(prev_tr_meta[\"id\"])\n", "# print(len(known_train_ids))\n", "# print(df_slice.shape)\n", "# df_slice = df_slice[~df_slice[\"id\"].isin(known_train_ids)].copy()\n", "# print(df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:44:52.693916Z", "iopub.status.busy": "2025-07-14T18:44:52.693774Z", "iopub.status.idle": "2025-07-14T18:44:54.385745Z", "shell.execute_reply": "2025-07-14T18:44:54.385133Z", "shell.execute_reply.started": "2025-07-14T18:44:52.693901Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(588814, 186)\n" ] } ], "source": [ "df_total = df_slice.copy()\n", "print(df_total.shape)" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:44:54.386525Z", "iopub.status.busy": "2025-07-14T18:44:54.386364Z", "iopub.status.idle": "2025-07-14T18:44:54.404003Z", "shell.execute_reply": "2025-07-14T18:44:54.403528Z", "shell.execute_reply.started": "2025-07-14T18:44:54.386508Z" } }, "outputs": [], "source": [ "# df_total.to_pickle(\"/home/tony/Data/Preference/auk_t1/fully_merged_auk_t1_slice0704.pkl\")" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:44:54.404666Z", "iopub.status.busy": "2025-07-14T18:44:54.404517Z", "iopub.status.idle": "2025-07-14T18:44:54.418522Z", "shell.execute_reply": "2025-07-14T18:44:54.418099Z", "shell.execute_reply.started": "2025-07-14T18:44:54.404651Z" } }, "outputs": [], "source": [ "# df_total[\"created_at\"] = pd.to_datetime(df_total[\"created_at\"], utc=True)\n", "# start_cutoff_date = pd.to_datetime(\"2025-06-01\", utc=True)\n", "# end_cutoff_date = pd.to_datetime(\"2025-06-30\", utc=True)\n", "# date_mask = (df_total[\"created_at\"] < end_cutoff_date) & (df_total[\"created_at\"] >= start_cutoff_date)\n", "# print(df_total.shape, df_total[date_mask].shape)\n", "# df_slice = df_total[date_mask].copy()\n", "# print(\"after date cut\", df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:44:54.419130Z", "iopub.status.busy": "2025-07-14T18:44:54.418995Z", "iopub.status.idle": "2025-07-14T18:44:54.430741Z", "shell.execute_reply": "2025-07-14T18:44:54.430324Z", "shell.execute_reply.started": "2025-07-14T18:44:54.419116Z" } }, "outputs": [], "source": [ "# print(\n", "# \"Before filtering by user_id and task\",\n", "# df_slice.shape[0],\n", "# \"user_id unique:\",\n", "# df_slice[\"user_id\"].nunique(),\n", "# )\n", "\n", "# # Create a copy to avoid fragmentation warning\n", "# df_slice = df_slice.copy()\n", "\n", "# # Calculate score for each row: reaction_play_count + 5 if preference is True, else 0\n", "# score_values = (\n", "# df_slice[\"reaction_play_count\"] + (5 * df_slice[\"upvote_count\"].astype(int))\n", "# ) * df_slice[\"preference\"].astype(int)\n", "\n", "# # Use pd.concat to add the score column efficiently\n", "# df_slice = pd.concat(\n", "# [df_slice, pd.DataFrame({\"score\": score_values}, index=df_slice.index)], axis=1\n", "# )\n", "\n", "# # Group by user_id and task, then for each group find the request_id with highest score\n", "# best_request_ids = []\n", "# for (user_id, task), group in tqdm(\n", "# df_slice.groupby([\"user_id\", \"task\"]), desc=\"Processing user_id and task groups\"\n", "# ):\n", "# # Get the request_id with the highest score in this group\n", "# best_request_id = group.loc[group[\"score\"].idxmax(), \"request_id\"]\n", "# best_request_ids.append(best_request_id)\n", "\n", "# # Filter df_slice to keep only the best request_ids for each user_id, task combination\n", "# df_slice = df_slice[df_slice[\"request_id\"].isin(best_request_ids)].copy()\n", "\n", "# print(\n", "# \"After filtering by user_id and task\",\n", "# df_slice.shape[0],\n", "# \"user_id unique:\",\n", "# df_slice[\"user_id\"].nunique(),\n", "# )" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T14:00:20.866354Z", "start_time": "2024-05-16T14:00:12.443344Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:44:54.444164Z", "iopub.status.busy": "2025-07-14T18:44:54.443866Z", "iopub.status.idle": "2025-07-14T18:44:54.501490Z", "shell.execute_reply": "2025-07-14T18:44:54.500669Z", "shell.execute_reply.started": "2025-07-14T18:44:54.444148Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(588814, 186)\n", "task\n", " 392074\n", "cover 99270\n", "artist_consistency 51500\n", "artist_cover 20804\n", "extend 17038\n", "upload_extend 4848\n", "artist_extend 3280\n", "Name: count, dtype: int64\n" ] }, { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[32], line 6\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_slice\u001b[38;5;241m.\u001b[39mshape)\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_slice[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mvalue_counts())\n\u001b[0;32m----> 6\u001b[0m \u001b[43mBREAK\u001b[49m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "# df_slice.to_pickle(\n", "# \"/home/tony/Data/Preference/30b_v6/interesting_clips_v4_h_t_6_20250426_full_long_slice.pkl\"\n", "# )\n", "print(df_slice.shape)\n", "print(df_slice[\"task\"].value_counts())\n", "BREAK" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Need to kick out the ones has gpt prompt -- these are pairs with different text inputs" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:45:21.245884Z", "iopub.status.busy": "2025-07-14T18:45:21.245471Z", "iopub.status.idle": "2025-07-14T18:45:23.399323Z", "shell.execute_reply": "2025-07-14T18:45:23.398703Z", "shell.execute_reply.started": "2025-07-14T18:45:21.245864Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(588814, 186)\n", "(588814, 186)\n", "(588814, 186)\n" ] } ], "source": [ "print(df_slice.shape)\n", "df_slice = df_slice[df_slice[\"request_id\"].apply(lambda x: len(x) > 3)]\n", "print(df_slice.shape)\n", "# df_slice = df_slice[df_slice[\"is_pro_user\"]].copy()\n", "print(df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932296Z", "start_time": "2024-05-16T13:59:41.932287Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:45:23.400389Z", "iopub.status.busy": "2025-07-14T18:45:23.400215Z", "iopub.status.idle": "2025-07-14T18:45:23.438281Z", "shell.execute_reply": "2025-07-14T18:45:23.437723Z", "shell.execute_reply.started": "2025-07-14T18:45:23.400373Z" } }, "outputs": [], "source": [ "# don't have continue at\n", "df_slice[\"request_id\"] = df_slice[\"request_id\"].astype(str)\n", "# df_slice[\"npz_path\"] = df_slice[\"npz_path\"].apply(lambda x: str(x).replace(\"_npz\", \"_npz/\"))\n", "# df_slice[df_slice[\"continue_at\"].isna()][\"request_id\"].nunique(), df_slice[\"request_id\"].nunique()" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932966Z", "start_time": "2024-05-16T13:59:41.932957Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:45:23.438975Z", "iopub.status.busy": "2025-07-14T18:45:23.438832Z", "iopub.status.idle": "2025-07-14T18:45:23.537757Z", "shell.execute_reply": "2025-07-14T18:45:23.537180Z", "shell.execute_reply.started": "2025-07-14T18:45:23.438960Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "294407\n" ] } ], "source": [ "final_filtered_requests = df_slice[\"request_id\"].astype(str).unique()\n", "# final_filtered_requests = df_slice[df_slice[\"is_pro_user\"]][\"request_id\"].astype(str).unique()\n", "print(len(final_filtered_requests))" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934277Z", "start_time": "2024-05-16T13:59:41.934268Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:45:23.538989Z", "iopub.status.busy": "2025-07-14T18:45:23.538828Z", "iopub.status.idle": "2025-07-14T18:45:27.964243Z", "shell.execute_reply": "2025-07-14T18:45:27.963613Z", "shell.execute_reply.started": "2025-07-14T18:45:23.538973Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "291462 2945\n", "(582924, 186) (5890, 186)\n" ] } ], "source": [ "train_requests, val_requests = train_test_split(\n", " sorted(list(final_filtered_requests)), test_size=0.01, random_state=42\n", ")\n", "print(len(train_requests), len(val_requests))\n", "\n", "train_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(train_requests))].copy()\n", "val_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(val_requests))].copy()\n", "train_df = train_df.sort_values(by=[\"request_id\", \"preference\"])\n", "train_df = train_df # .reset_index()\n", "val_df = val_df.sort_values(by=[\"request_id\", \"preference\"])\n", "val_df = val_df # .reset_index()\n", "train_df = train_df.reset_index(drop=True)\n", "val_df = val_df.reset_index(drop=True)\n", "print(train_df.shape, val_df.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Actually make" ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.935620Z", "start_time": "2024-05-16T13:59:41.935613Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:45:27.965047Z", "iopub.status.busy": "2025-07-14T18:45:27.964887Z", "iopub.status.idle": "2025-07-14T18:45:48.761362Z", "shell.execute_reply": "2025-07-14T18:45:48.760713Z", "shell.execute_reply.started": "2025-07-14T18:45:27.965031Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 582924/582924 [00:19<00:00, 29260.73it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "29,694 hours of 582924 clips, 36.43275 nodes, 569.26171875 iters\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "total_duration = 0\n", "for i, row in tqdm(train_df.iterrows(), total=len(train_df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " try:\n", " assert row[\"preference\"] == (i % 2 == 1)\n", " total_duration += row[\"duration\"]\n", " except Exception as E:\n", " print(i, row)\n", " print(E)\n", " raise ValueError()\n", "\n", "print(\n", " f\"{round(total_duration / 60 / 60):,} hours of {train_df.shape[0]} clips, {train_df.shape[0] / 8 / 2 / 1000} nodes, {train_df.shape[0] / 8 / 8 / 16} iters\"\n", ")" ] }, { "cell_type": "code", "execution_count": 38, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936268Z", "start_time": "2024-05-16T13:59:41.936260Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:45:48.762318Z", "iopub.status.busy": "2025-07-14T18:45:48.762040Z", "iopub.status.idle": "2025-07-14T18:46:51.622225Z", "shell.execute_reply": "2025-07-14T18:46:51.621703Z", "shell.execute_reply.started": "2025-07-14T18:45:48.762301Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "t_data_memmap is set to: 12000\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5890/5890 [01:02<00:00, 93.77it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 5890 clips, 1 different prompts, 0 different tags\n", "196 hours of False\n", "197 hours of True\n", "gen: 202.9 hours\n", "cover: 98.9 hours\n", "artist_consistency: 50.6 hours\n", "extend: 12.4 hours\n", "artist_cover: 25.0 hours\n", "artist_extend: 3.6 hours\n", "Done\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "make_dataset(\n", " val_df, OUT_DATA_DIR, is_val=True, npz_dir=NPZ_DIR, t_data_memmap=N_TOKENS_AUDIO\n", ")" ] }, { "cell_type": "code", "execution_count": 39, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T18:46:51.622928Z", "iopub.status.busy": "2025-07-14T18:46:51.622773Z", "iopub.status.idle": "2025-07-14T18:46:52.474151Z", "shell.execute_reply": "2025-07-14T18:46:52.473678Z", "shell.execute_reply.started": "2025-07-14T18:46:51.622913Z" } }, "outputs": [], "source": [ "# test_npz = np.load(\"/app/suno/data/dpo/30b_npz/26d19085-18da-4701-af43-122684543891.npz\")\n", "# for k in test_npz.keys():\n", "# print(k)" ] }, { "cell_type": "code", "execution_count": 40, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936964Z", "start_time": "2024-05-16T13:59:41.936957Z" }, "execution": { "iopub.execute_input": "2025-07-14T18:46:52.474900Z", "iopub.status.busy": "2025-07-14T18:46:52.474649Z", "iopub.status.idle": "2025-07-14T20:29:35.955467Z", "shell.execute_reply": "2025-07-14T20:29:35.954852Z", "shell.execute_reply.started": "2025-07-14T18:46:52.474885Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "t_data_memmap is set to: 12000\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 582924/582924 [1:42:42<00:00, 94.60it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 582924 clips, 71 different prompts, 2 different tags\n", "19,468 hours of False\n", "19,515 hours of True\n", "cover: 9515.6 hours\n", "gen: 20282.5 hours\n", "artist_consistency: 5212.6 hours\n", "artist_cover: 2347.8 hours\n", "extend: 1287.1 hours\n", "artist_extend: 337.9 hours\n", "Done\n" ] } ], "source": [ "make_dataset(\n", " train_df, OUT_DATA_DIR, is_val=False, npz_dir=NPZ_DIR, t_data_memmap=N_TOKENS_AUDIO\n", ")" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2024-01-29T19:46:47.549860Z", "start_time": "2024-01-29T19:46:47.548015Z" } }, "source": [ "# Validation" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.937879Z", "start_time": "2024-05-16T13:59:41.937870Z" }, "execution": { "iopub.execute_input": "2025-07-14T20:29:35.956259Z", "iopub.status.busy": "2025-07-14T20:29:35.956094Z", "iopub.status.idle": "2025-07-14T20:29:36.930278Z", "shell.execute_reply": "2025-07-14T20:29:36.929712Z", "shell.execute_reply.started": "2025-07-14T20:29:35.956243Z" } }, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "test_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_val.jsonl\"))\n", "test_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_val.json\"))\n", "mm = mm.reshape(-1, N_TOKENS_AUDIO, 1)\n", "assert len(mm) == len(test_metas)\n", "assert mm[:100, :, 0].min() >= 0\n", "assert mm[:100, :, 0].max() <= 4000" ] }, { "cell_type": "code", "execution_count": 42, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T20:29:36.932110Z", "iopub.status.busy": "2025-07-14T20:29:36.931941Z", "iopub.status.idle": "2025-07-14T20:29:36.949371Z", "shell.execute_reply": "2025-07-14T20:29:36.948884Z", "shell.execute_reply.started": "2025-07-14T20:29:36.932093Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Counter({None: 3898, 'cover': 1034, 'artist_consistency': 492, 'artist_cover': 224, 'extend': 208, 'artist_extend': 34})\n" ] } ], "source": [ "task_counts = Counter()\n", "for test_meta in test_metas:\n", " task_counts[test_meta.get(\"task\")] += 1\n", "print(task_counts)" ] }, { "cell_type": "code", "execution_count": 43, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.938629Z", "start_time": "2024-05-16T13:59:41.938621Z" }, "execution": { "iopub.execute_input": "2025-07-14T20:29:36.950031Z", "iopub.status.busy": "2025-07-14T20:29:36.949888Z", "iopub.status.idle": "2025-07-14T20:29:36.963766Z", "shell.execute_reply": "2025-07-14T20:29:36.963328Z", "shell.execute_reply.started": "2025-07-14T20:29:36.950016Z" } }, "outputs": [], "source": [ "# # randomly listen to some stuff\n", "# from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "# from suno_utils.tasks.dac_2c_12cb import (\n", "# encode as codec_encode,\n", "# decode_stream_to_full_audio as codec_decode,\n", "# EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", "# decode as decode\n", "# )\n", "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", "# _ = preload_codec_models(\"/app/suno/data/dpo/models/dac_2c_25x12.pt\", device=\"cuda\")\n", "# assert len(test_metas) == len(mm)\n", "# idx_list = list(range(len(test_metas)))\n", "# # random.shuffle(idx_list)\n", "# # idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "# print(len(mm))" ] }, { "cell_type": "code", "execution_count": 44, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939205Z", "start_time": "2024-05-16T13:59:41.939198Z" }, "execution": { "iopub.execute_input": "2025-07-14T20:29:36.964403Z", "iopub.status.busy": "2025-07-14T20:29:36.964266Z", "iopub.status.idle": "2025-07-14T20:29:36.976881Z", "shell.execute_reply": "2025-07-14T20:29:36.976442Z", "shell.execute_reply.started": "2025-07-14T20:29:36.964389Z" } }, "outputs": [], "source": [ "# import random\n", "# idx = random.choice(test_info[\"perference_0\"][\"idx_list\"])\n", "# assert \"original_duration_s\" in test_metas[idx]\n", "# # positive index should be shifted by 1\n", "# pos_idx = idx + 1\n", "# print(\n", "# \"tags:\",\n", "# test_metas[idx].get(\"tags\") == test_metas[pos_idx].get(\"tags\"),\n", "# test_metas[idx].get(\"tags\"),\n", "# )\n", "# arr = mm[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pos_arr = mm[pos_idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pad_idx_arr = np.where(arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pad_idx_arr) > 0:\n", "# arr = arr[: pad_idx_arr[0], :]\n", "# pos_pad_idx_arr = np.where(pos_arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pos_pad_idx_arr) > 0:\n", "# pos_arr = pos_arr[: pos_pad_idx_arr[0], :]\n", "# a = decode(arr)\n", "# print(\"\\n negative example \\n\", test_metas[idx])\n", "# a.play(compress=False)\n", "# pos_a = decode(pos_arr)\n", "# print(\"\\n positive example \\n\", test_metas[pos_idx])\n", "# pos_a.play(compress=False)\n", "# print(\n", "# \"text:\",\n", "# test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"),\n", "# test_metas[idx].get(\"text\"),\n", "# )" ] }, { "cell_type": "code", "execution_count": 45, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939977Z", "start_time": "2024-05-16T13:59:41.939969Z" }, "execution": { "iopub.execute_input": "2025-07-14T20:29:36.977458Z", "iopub.status.busy": "2025-07-14T20:29:36.977323Z", "iopub.status.idle": "2025-07-14T20:29:36.989144Z", "shell.execute_reply": "2025-07-14T20:29:36.988711Z", "shell.execute_reply.started": "2025-07-14T20:29:36.977445Z" } }, "outputs": [], "source": [ "# val_df[val_df[\"tags\"] == 'a vibrant blend of experimental jazz fusion, drum-and-bass and swagger fuzzed-out guitars']" ] }, { "cell_type": "code", "execution_count": 46, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.940610Z", "start_time": "2024-05-16T13:59:41.940603Z" }, "execution": { "iopub.execute_input": "2025-07-14T20:29:36.989753Z", "iopub.status.busy": "2025-07-14T20:29:36.989619Z", "iopub.status.idle": "2025-07-14T20:29:37.001189Z", "shell.execute_reply": "2025-07-14T20:29:37.000762Z", "shell.execute_reply.started": "2025-07-14T20:29:36.989739Z" } }, "outputs": [], "source": [ "# from collections import Counter\n", "# c = Counter()\n", "# for _, row in df_slice.iterrows():\n", "# # print(row[\"metadata\"])\n", "# for k in ast.literal_eval(row[\"metadata\"]).keys():\n", "# c[k] += 1" ] }, { "cell_type": "code", "execution_count": 47, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941167Z", "start_time": "2024-05-16T13:59:41.941159Z" }, "execution": { "iopub.execute_input": "2025-07-14T20:29:37.001807Z", "iopub.status.busy": "2025-07-14T20:29:37.001668Z", "iopub.status.idle": "2025-07-14T20:29:37.013022Z", "shell.execute_reply": "2025-07-14T20:29:37.012593Z", "shell.execute_reply.started": "2025-07-14T20:29:37.001793Z" } }, "outputs": [], "source": [ "# original_npz_path = f\"/app/suno/data/dpo/7b_npz/{test_metas[idx]['id']}.npz\"\n", "# original_npz_path = \"/app/suno/data/dpo/7b_npz/729c3011-f672-4ccd-8d82-1cbf2b52ff69.npz\"\n", "# original_arr = np.load(original_npz_path)[\"v2_raw\"]" ] }, { "cell_type": "code", "execution_count": 48, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941801Z", "start_time": "2024-05-16T13:59:41.941793Z" }, "execution": { "iopub.execute_input": "2025-07-14T20:29:37.013644Z", "iopub.status.busy": "2025-07-14T20:29:37.013502Z", "iopub.status.idle": "2025-07-14T20:29:37.030528Z", "shell.execute_reply": "2025-07-14T20:29:37.030071Z", "shell.execute_reply.started": "2025-07-14T20:29:37.013630Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2945 0\n" ] } ], "source": [ "def validation_on_metas(input_metas):\n", " total_bad = 0\n", " total_good = 0\n", " for idx in range(len(input_metas)):\n", " if idx % 2 == 0:\n", " pos_idx = idx + 1\n", " if input_metas[idx].get(\"tags\") != input_metas[pos_idx].get(\"tags\"):\n", " print(input_metas[idx].get(\"id\"), input_metas[idx].get(\"tags\"),input_metas[pos_idx].get(\"id\"), input_metas[pos_idx].get(\"tags\"), )\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " elif input_metas[idx].get(\"text\") != input_metas[pos_idx].get(\"text\"):\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " else:\n", " total_good += 1\n", " print(total_good, total_bad)\n", " return\n", "\n", "\n", "validation_on_metas(test_metas)" ] }, { "cell_type": "code", "execution_count": 49, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.942520Z", "start_time": "2024-05-16T13:59:41.942511Z" }, "execution": { "iopub.execute_input": "2025-07-14T20:29:37.031155Z", "iopub.status.busy": "2025-07-14T20:29:37.031019Z", "iopub.status.idle": "2025-07-14T20:29:37.084194Z", "shell.execute_reply": "2025-07-14T20:29:37.083739Z", "shell.execute_reply.started": "2025-07-14T20:29:37.031142Z" } }, "outputs": [], "source": [ "train_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_tr.json\"))" ] }, { "cell_type": "code", "execution_count": 50, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.943072Z", "start_time": "2024-05-16T13:59:41.943065Z" }, "execution": { "iopub.execute_input": "2025-07-14T20:29:37.085002Z", "iopub.status.busy": "2025-07-14T20:29:37.084863Z", "iopub.status.idle": "2025-07-14T20:29:37.120258Z", "shell.execute_reply": "2025-07-14T20:29:37.119812Z", "shell.execute_reply.started": "2025-07-14T20:29:37.084988Z" } }, "outputs": [], "source": [ "n_neg_tr = train_info[\"perference_0\"][\"idx_list\"]\n", "n_pos_tr = train_info[\"perference_1\"][\"idx_list\"]\n", "assert len(n_pos_tr) == len(n_neg_tr)\n", "# make sure they are offset by 1 and exactly 1\n", "for i, j in zip(n_neg_tr, n_pos_tr):\n", " assert i == j - 1" ] }, { "cell_type": "code", "execution_count": 51, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.944246Z", "start_time": "2024-05-16T13:59:41.944237Z" }, "execution": { "iopub.execute_input": "2025-07-14T20:29:37.120880Z", "iopub.status.busy": "2025-07-14T20:29:37.120744Z", "iopub.status.idle": "2025-07-14T20:29:37.132811Z", "shell.execute_reply": "2025-07-14T20:29:37.132354Z", "shell.execute_reply.started": "2025-07-14T20:29:37.120866Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total samples 582924 (582924, 186)\n" ] } ], "source": [ "total_iters = len(n_neg_tr) + len(n_pos_tr)\n", "print(\"total samples\", total_iters, train_df.shape)" ] }, { "cell_type": "code", "execution_count": 52, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T20:29:37.133438Z", "iopub.status.busy": "2025-07-14T20:29:37.133303Z", "iopub.status.idle": "2025-07-14T20:29:46.371534Z", "shell.execute_reply": "2025-07-14T20:29:46.370960Z", "shell.execute_reply.started": "2025-07-14T20:29:37.133424Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "291462 0\n" ] } ], "source": [ "metas_tr = read_jsonl(os.path.join(OUT_DATA_DIR, \"meta_tr.jsonl\"))\n", "validation_on_metas(metas_tr)" ] }, { "cell_type": "code", "execution_count": 53, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T20:29:46.372276Z", "iopub.status.busy": "2025-07-14T20:29:46.372125Z", "iopub.status.idle": "2025-07-14T20:29:46.387709Z", "shell.execute_reply": "2025-07-14T20:29:46.387256Z", "shell.execute_reply.started": "2025-07-14T20:29:46.372261Z" } }, "outputs": [], "source": [ "# new_metas_tr = []\n", "# for index, l in enumerate(metas_tr):\n", "# if index % 2 == 1:\n", "# last_l = new_metas_tr[-1]\n", "# if l[\"tags\"] != last_l[\"tags\"]:\n", "# print(l[\"tags\"], last_l[\"tags\"])\n", "# l[\"tags\"] = last_l[\"tags\"]\n", "# new_metas_tr.append(l)\n", "# validation_on_metas(new_metas_tr)\n", "# write_jsonl(new_metas_tr, os.path.join(OUT_DATA_DIR, \"meta_tr.jsonl\"))" ] }, { "cell_type": "code", "execution_count": 54, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945249Z", "start_time": "2024-05-16T13:59:41.945241Z" }, "execution": { "iopub.execute_input": "2025-07-14T20:29:46.388339Z", "iopub.status.busy": "2025-07-14T20:29:46.388198Z", "iopub.status.idle": "2025-07-14T20:29:46.402207Z", "shell.execute_reply": "2025-07-14T20:29:46.401750Z", "shell.execute_reply.started": "2025-07-14T20:29:46.388325Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1 epoch per batch 4, total 569.26171875\n" ] } ], "source": [ "print(\"1 epoch per batch 4, total\", total_iters / 16 / 8 / 8)" ] }, { "cell_type": "code", "execution_count": 55, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T20:29:46.402832Z", "iopub.status.busy": "2025-07-14T20:29:46.402696Z", "iopub.status.idle": "2025-07-14T20:29:46.414405Z", "shell.execute_reply": "2025-07-14T20:29:46.413964Z", "shell.execute_reply.started": "2025-07-14T20:29:46.402819Z" } }, "outputs": [], "source": [ "# import time\n", "# time.sleep(60 * 60 * 1)" ] }, { "cell_type": "code", "execution_count": 56, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945972Z", "start_time": "2024-05-16T13:59:41.945964Z" }, "execution": { "iopub.execute_input": "2025-07-14T20:29:46.415016Z", "iopub.status.busy": "2025-07-14T20:29:46.414880Z", "iopub.status.idle": "2025-07-14T20:29:46.426389Z", "shell.execute_reply": "2025-07-14T20:29:46.425964Z", "shell.execute_reply.started": "2025-07-14T20:29:46.415002Z" } }, "outputs": [], "source": [ "# !cd /home/tony/Work/tony/slurm/bluejay && sbatch sbatch_ipo_bluejay" ] }, { "cell_type": "code", "execution_count": 57, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T20:29:46.426957Z", "iopub.status.busy": "2025-07-14T20:29:46.426829Z", "iopub.status.idle": "2025-07-14T20:29:46.448642Z", "shell.execute_reply": "2025-07-14T20:29:46.448184Z", "shell.execute_reply.started": "2025-07-14T20:29:46.426944Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cache kept!\n" ] } ], "source": [ "import shutil\n", "\n", "# Basic file copy\n", "shutil.copy(\n", " \"/home/tony/Work/tony/Preference/make_dataset_auk_t1.ipynb\",\n", " os.path.join(OUT_DATA_DIR, \"make_dataset.ipynb\"),\n", ")\n", "print(\"Cache kept!\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# some gymathtics loading prev data" ] }, { "cell_type": "code", "execution_count": 58, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.946562Z", "start_time": "2024-05-16T13:59:41.946555Z" }, "execution": { "iopub.execute_input": "2025-07-14T20:29:46.449406Z", "iopub.status.busy": "2025-07-14T20:29:46.449272Z", "iopub.status.idle": "2025-07-14T20:29:46.460524Z", "shell.execute_reply": "2025-07-14T20:29:46.460099Z", "shell.execute_reply.started": "2025-07-14T20:29:46.449393Z" } }, "outputs": [], "source": [ "# prev_v3_data = \"/app/suno/data/dpo/7v_v20_full/\"\n", "\n", "# test_val_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_val.jsonl\"))\n", "# test_tr_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_tr.jsonl\"))\n", "\n", "# all_ids = set()\n", "# for meta in test_val_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# for meta in test_tr_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# print(len(all_ids), len(test_val_metas) + len(test_tr_metas))\n", "\n", "# all_ids = list(all_ids)\n", "# with open(\"/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id.json\", \"w\") as fp:\n", "# json.dump(all_ids, fp)" ] }, { "cell_type": "code", "execution_count": 59, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T20:29:46.461117Z", "iopub.status.busy": "2025-07-14T20:29:46.460984Z", "iopub.status.idle": "2025-07-14T20:29:46.472617Z", "shell.execute_reply": "2025-07-14T20:29:46.472193Z", "shell.execute_reply.started": "2025-07-14T20:29:46.461104Z" } }, "outputs": [], "source": [ "# x_data = train_df[train_df[\"preference\"]][\"similarity\"]\n", "# y_data = train_df[~train_df[\"preference\"]][\"similarity\"]\n", "# from matplotlib.colors import LogNorm\n", "\n", "# fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 10))\n", "\n", "# # 2D Histogram\n", "# h = ax1.hist2d(\n", "# x_data,\n", "# y_data,\n", "# bins=(50, 50),\n", "# cmap=\"coolwarm\",\n", "# range=[[0, 1], [0, 1]],\n", "# norm=LogNorm(),\n", "# )\n", "\n", "# ax1.set_xlabel(\"Semantic Distance (Preferred)\")\n", "# ax1.set_ylabel(\"Semantic Distance (Non-Preferred)\")\n", "# ax1.set_title(\n", "# \"2D Histogram of Semantic Distances: Preferred vs Non-Preferred (Log Scale)\"\n", "# )\n", "\n", "# cbar1 = plt.colorbar(h[3], ax=ax1)\n", "# cbar1.set_label(\"Number of Request IDs (Log Scale)\")\n", "\n", "# # Scatter plot\n", "# ax2.scatter(x_data, y_data, alpha=0.1, s=1)\n", "# ax2.set_xlabel(\"Semantic Distance (Preferred)\")\n", "# ax2.set_ylabel(\"Semantic Distance (Non-Preferred)\")\n", "# ax2.set_title(\"Scatter Plot of Semantic Distances: Preferred vs Non-Preferred\")\n", "# ax2.set_xlim(0, 1)\n", "# ax2.set_ylim(0, 1)\n", "\n", "# plt.tight_layout()\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": 60, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T20:29:46.474846Z", "iopub.status.busy": "2025-07-14T20:29:46.474695Z", "iopub.status.idle": "2025-07-14T20:29:46.486313Z", "shell.execute_reply": "2025-07-14T20:29:46.485875Z", "shell.execute_reply.started": "2025-07-14T20:29:46.474832Z" } }, "outputs": [], "source": [ "# train_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_tr.jsonl\"))" ] }, { "cell_type": "code", "execution_count": 61, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T20:29:46.486918Z", "iopub.status.busy": "2025-07-14T20:29:46.486780Z", "iopub.status.idle": "2025-07-14T20:29:46.499393Z", "shell.execute_reply": "2025-07-14T20:29:46.498976Z", "shell.execute_reply.started": "2025-07-14T20:29:46.486904Z" } }, "outputs": [ { "data": { "text/plain": [ "dict_keys(['perference_0', 'perference_1'])" ] }, "execution_count": 61, "metadata": {}, "output_type": "execute_result" } ], "source": [ "train_info.keys()" ] }, { "cell_type": "code", "execution_count": 62, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T20:29:46.499995Z", "iopub.status.busy": "2025-07-14T20:29:46.499860Z", "iopub.status.idle": "2025-07-14T20:29:49.316863Z", "shell.execute_reply": "2025-07-14T20:29:49.316248Z", "shell.execute_reply.started": "2025-07-14T20:29:46.499982Z" } }, "outputs": [], "source": [ "import torch\n", "a = torch.tensor([6.2500e-04, 3.9062e-05, 2.3462e-03])" ] }, { "cell_type": "code", "execution_count": 63, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T20:29:49.317630Z", "iopub.status.busy": "2025-07-14T20:29:49.317475Z", "iopub.status.idle": "2025-07-14T20:29:49.359200Z", "shell.execute_reply": "2025-07-14T20:29:49.358756Z", "shell.execute_reply.started": "2025-07-14T20:29:49.317614Z" } }, "outputs": [ { "data": { "text/plain": [ "tensor(0.0010)" ] }, "execution_count": 63, "metadata": {}, "output_type": "execute_result" } ], "source": [ "a.mean()" ] }, { "cell_type": "code", "execution_count": 64, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T20:29:49.359854Z", "iopub.status.busy": "2025-07-14T20:29:49.359706Z", "iopub.status.idle": "2025-07-14T20:29:49.376815Z", "shell.execute_reply": "2025-07-14T20:29:49.376368Z", "shell.execute_reply.started": "2025-07-14T20:29:49.359839Z" } }, "outputs": [], "source": [ "# import time\n", "# time.sleep(3600 * 3)" ] }, { "cell_type": "code", "execution_count": 65, "metadata": { "execution": { "iopub.execute_input": "2025-07-14T20:29:49.377454Z", "iopub.status.busy": "2025-07-14T20:29:49.377317Z", "iopub.status.idle": "2025-07-14T20:29:49.393912Z", "shell.execute_reply": "2025-07-14T20:29:49.393478Z", "shell.execute_reply.started": "2025-07-14T20:29:49.377440Z" } }, "outputs": [], "source": [ "# !cd /home/tony/Work/tony/slurm/diffusion && sbatch run_diffusion_infill.sh" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env_dev", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 4 }