{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:01:55.493035Z", "start_time": "2024-03-03T01:01:54.489755Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_1756727/2161522484.py:1: DeprecationWarning: \n", "Pyarrow will become a required dependency of pandas in the next major release of pandas (pandas 3.0),\n", "(to allow more performant data types, such as the Arrow string type, and better interoperability with other libraries)\n", "but was not found to be installed on your system.\n", "If this would cause problems for you,\n", "please provide us feedback at https://github.com/pandas-dev/pandas/issues/54466\n", " \n", " import pandas as pd\n" ] } ], "source": [ "import pandas as pd\n", "import numpy as np\n", "import os\n", "from tqdm import tqdm\n", "from sklearn.model_selection import train_test_split\n", "from suno_utils.utils.s3 import download_s3_files\n", "import sys\n", "from collections import defaultdict\n", "from suno_utils.utils.text import (\n", " write_jsonl,\n", " read_jsonl,\n", " write_json,\n", " read_json,\n", ")\n", "import ast\n", "\n", "sys.path.insert(0, \"/home/tony/Work/glockenspiel/sunoGPT/scripts/\")\n", "\n", "from data_preparation_7b import *\n", "import numpy as np" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:01:55.496406Z", "start_time": "2024-03-03T01:01:55.494247Z" } }, "outputs": [], "source": [ "OUT_DATA_DIR = \"/app/suno/data/dpo/7v_v0_mix/\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "NPZ_DIR = \"/app/suno/data/dpo/v3_npz\"" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:02:01.987764Z", "start_time": "2024-03-03T01:01:55.497933Z" } }, "outputs": [ { "data": { "text/plain": [ "(376932, 37)" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = pd.read_csv(\"/home/tony/Data/Preference/7b_v0/interesting_clips.csv\")\n", "df.shape\n", "# v0: (68746, 31)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:02:02.072671Z", "start_time": "2024-03-03T01:02:01.988898Z" } }, "outputs": [], "source": [ "# download all clips; 883k \n", "from suno_utils.utils.s3 import download_s3_files\n", "\n", "s3_ids = df[\"s3_id\"].values\n", "s3_paths = [f\"s3://suno-data-uploads/studio/uploads/{s3_id}.npz\" for s3_id in s3_ids]\n", "local_paths = [f\"{NPZ_DIR}/{s3_id}.npz\" for s3_id in s3_ids]" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:02:02.587667Z", "start_time": "2024-03-03T01:02:02.073973Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "jobs to be done 0 0\n" ] } ], "source": [ "finished_paths = os.listdir(NPZ_DIR)\n", "finished_paths_set = set(finished_paths)\n", "unfinished_s3_paths = [path for path in s3_paths if os.path.basename(path) not in finished_paths_set]\n", "unfinished_paths = [path for path in local_paths if os.path.basename(path) not in finished_paths_set]\n", "print(\"jobs to be done\", len(unfinished_paths), len(unfinished_s3_paths))" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:02:02.590533Z", "start_time": "2024-03-03T01:02:02.588897Z" } }, "outputs": [], "source": [ "_ = download_s3_files(unfinished_s3_paths, unfinished_paths, n_cores=32)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:02:03.204791Z", "start_time": "2024-03-03T01:02:02.623909Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "jobs to be done 0 0\n" ] } ], "source": [ "# # deleted files are at: deleted; try to get them as well\n", "finished_paths = os.listdir(NPZ_DIR)\n", "finished_paths_set = set(finished_paths)\n", "unfinished_s3_paths = [path for path in s3_paths if os.path.basename(path) not in finished_paths_set]\n", "unfinished_paths = [path for path in local_paths if os.path.basename(path) not in finished_paths_set]\n", "print(\"jobs to be done\", len(unfinished_paths), len(unfinished_s3_paths))\n", "unfinished_deleted_s3_paths = [path.replace(\"/uploads/\", \"/deleted/\") for path in unfinished_s3_paths]\n", "_ = download_s3_files(unfinished_deleted_s3_paths, unfinished_paths, n_cores=32)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:02:03.207852Z", "start_time": "2024-03-03T01:02:03.206070Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Finish downloads\n" ] } ], "source": [ "print(\"Finish downloads\")" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:02:03.365297Z", "start_time": "2024-03-03T01:02:03.208819Z" } }, "outputs": [], "source": [ "converted_paths = os.listdir(NPZ_DIR)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:02:03.368455Z", "start_time": "2024-03-03T01:02:03.366949Z" } }, "outputs": [], "source": [ "# # don't run this unless you kill some job accidentally\n", "# for processed_file in tqdm.tqdm(converted_paths):\n", "# processed_file_path = os.path.join(\"/app/suno/data/dpo/7b_npz\", processed_file)\n", "# # could have been removed already\n", "# if os.path.exists(processed_file_path):\n", "# file_size = os.stat(processed_file_path).st_size\n", "# # print(processed_file_path, file_size)\n", "# # break\n", "# if file_size < 1000:\n", "# print(processed_file_path, file_size)\n", "# os.remove(processed_file_path)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:02:03.531766Z", "start_time": "2024-03-03T01:02:03.370899Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "414710\n" ] } ], "source": [ "converted_paths = set([f.replace(\".npz\", \"\") for f in converted_paths])\n", "print(len(converted_paths))" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:02:04.114229Z", "start_time": "2024-03-03T01:02:03.532994Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "pre-downloaded df (376932, 37)\n", "downloaded df (376932, 37)\n" ] } ], "source": [ "print(\"pre-downloaded df\", df.shape)\n", "df[df[\"s3_id\"].isin(converted_paths)].shape\n", "df = df[df[\"s3_id\"].isin(converted_paths)].copy()\n", "print(\"downloaded df\", df.shape)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:02:41.969500Z", "start_time": "2024-03-03T01:02:04.115434Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_1756727/435958546.py:1: DtypeWarning: Columns (3,10,13,16,17,19,25,28,29,30,31,32,33,34) have mixed types. Specify dtype option on import or set low_memory=False.\n", " df_v2 = pd.read_csv(\"/home/tony/Data/Preference/v4/interesting_clips.csv\")\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "(1926120, 35)\n" ] } ], "source": [ "df_v2 = pd.read_csv(\"/home/tony/Data/Preference/v4/interesting_clips.csv\")\n", "print(df_v2.shape)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:04:43.458314Z", "start_time": "2024-03-03T01:02:41.970699Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3519009\n", "pre-downloaded df (1926120, 35)\n", "downloaded df (1926091, 35)\n" ] } ], "source": [ "converted_paths_v2 = os.listdir(\"/app/suno/data/dpo/7b_npz\")\n", "converted_paths_v2 = set([f.replace(\".npz\", \"\") for f in converted_paths_v2])\n", "print(len(converted_paths_v2))\n", "print(\"pre-downloaded df\", df_v2.shape)\n", "df_v2 = df_v2[df_v2[\"s3_id\"].isin(converted_paths_v2)].copy()\n", "print(\"downloaded df\", df_v2.shape)" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:04:45.266201Z", "start_time": "2024-03-03T01:04:43.459519Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "merged df (2303023, 37)\n" ] } ], "source": [ "df = pd.concat([df, df_v2])\n", "print(\"merged df\", df.shape)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:04:45.646473Z", "start_time": "2024-03-03T01:04:45.267537Z" } }, "outputs": [ { "data": { "text/plain": [ "is_7b\n", "False 1926091\n", "True 376932\n", "Name: count, dtype: int64" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"is_7b\"] = df[\"model_name\"].str.contains(\"v3\")\n", "df[\"is_7b\"].value_counts()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# LET's do the data prep" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:04:48.151316Z", "start_time": "2024-03-03T01:04:45.647642Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(2303023, 38)\n", "(2303022, 38)\n" ] } ], "source": [ "print(df.shape)\n", "df = df[df['request_id'].isin(df['request_id'].value_counts().index[df['request_id'].value_counts() == 2])]\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:04:48.528835Z", "start_time": "2024-03-03T01:04:48.152480Z" } }, "outputs": [], "source": [ "assert df.shape[0] == df[\"request_id\"].nunique() * 2" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:04:48.554264Z", "start_time": "2024-03-03T01:04:48.531265Z" } }, "outputs": [ { "data": { "text/plain": [ "play_count\n", "3.0 539772\n", "1.0 358952\n", "2.0 358239\n", "4.0 323017\n", "5.0 201833\n", " ... \n", "401.0 1\n", "246.0 1\n", "3034.0 1\n", "201.0 1\n", "333.0 1\n", "Name: count, Length: 401, dtype: int64" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"play_count\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:13:14.032440Z", "start_time": "2024-03-03T01:04:48.555325Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1151511\n" ] } ], "source": [ "# expand the metadata columns\n", "test_slice = df[\"metadata\"].apply(lambda x: pd.Series(ast.literal_eval(x)))\n", "df = pd.concat([df, test_slice], axis=1, join=\"inner\")\n", "print(df[\"request_id\"].nunique())" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:13:16.117259Z", "start_time": "2024-03-03T01:13:14.036704Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1151509\n" ] } ], "source": [ "# double check we removed the gpt prompted ones for now\n", "df = df[df[\"has_gpt_prompt\"] == False]\n", "print(df[\"request_id\"].nunique())" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:13:56.442908Z", "start_time": "2024-03-03T01:13:16.120090Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "378022\n", "good_continue_at\n", "True 2299289\n", "False 3729\n", "Name: count, dtype: int64\n" ] } ], "source": [ "audio_prompt_id_to_continue_at = {}\n", "for _, row in df[~df[\"audio_prompt_id\"].isna()].iterrows():\n", " audio_prompt_id = row[\"audio_prompt_id\"]\n", " if audio_prompt_id not in audio_prompt_id_to_continue_at:\n", " audio_prompt_id_to_continue_at[audio_prompt_id] = row[\"continue_at\"]\n", " else:\n", " # pick the max\n", " audio_prompt_id = max(\n", " audio_prompt_id_to_continue_at[audio_prompt_id], row[\"continue_at\"]\n", " )\n", "print(len(audio_prompt_id_to_continue_at))\n", "df[\"has_continue_and_start_continue_at\"] = df[\"id\"].apply(\n", " lambda x: audio_prompt_id_to_continue_at.get(x)\n", ")\n", "# we want continue at to be at least half of the clip...\n", "df[\"good_continue_at\"] = (\n", " (df[\"has_continue_and_start_continue_at\"] / df[\"duration\"]) > 0.85\n", ") | df[\"has_continue_and_start_continue_at\"].isna()\n", "print(df[\"good_continue_at\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:13:56.531330Z", "start_time": "2024-03-03T01:13:56.444081Z" } }, "outputs": [ { "data": { "text/plain": [ "(preference\n", " True 1151509\n", " False 1151509\n", " Name: count, dtype: int64,\n", " is_7b\n", " False 1926086\n", " True 376932\n", " Name: count, dtype: int64)" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"preference\"].value_counts(), df[\"is_7b\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:13:56.692011Z", "start_time": "2024-03-03T01:13:56.532386Z" } }, "outputs": [ { "data": { "text/plain": [ "model_name\n", "chirp-v2-xxl-alpha 1908154\n", "chirp-v3-engine-d 202944\n", "chirp-v3-engine-v0 173976\n", "chirp-v2-engine-v12 12086\n", "chirp-v2-engine-v8 4116\n", "chirp-v2-engine-v0 1092\n", "chirp-v2-engine-jingle 628\n", "chirp-v3-alpha 11\n", "chirp-v2-engine-v13 6\n", "chirp-v2-engine-dev 4\n", "chirp-v3-engine-exp 1\n", "Name: count, dtype: int64" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"model_name\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:13:59.040642Z", "start_time": "2024-03-03T01:13:56.693073Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "negative 911377 positive 214456\n", "total 1151509 selected 190966 0.16583978067040728\n" ] } ], "source": [ "neg_filter_selection_mask = (\n", " (df[\"preference\"] == False)\n", " & (df[\"play_count\"] >= 1)\n", " & (df[\"duration\"] > 5)\n", " & (df[\"duration\"] <= 120)\n", " & (\n", " ((df[\"is_7b\"] == False) & (df[\"play_count\"] >= 2) & (df[\"duration\"] >= 10))\n", " | ((df[\"is_7b\"] == True) & (df[\"play_count\"] >= 1))\n", " )\n", ")\n", "pos_filter_selectin_mask = (df[\"preference\"] == True) & (\n", " (\n", " ((df[\"play_count\"] >= 3) & (df[\"is_7b\"] == True))\n", " | ((df[\"play_count\"] >= 10) & (df[\"is_7b\"] == False))\n", " )\n", " & (df[\"duration\"] > 5)\n", " & (df[\"duration\"] <= 120)\n", " & (df[\"good_continue_at\"])\n", ")\n", "print(\n", " \"negative\",\n", " sum(neg_filter_selection_mask),\n", " \"positive\",\n", " sum(pos_filter_selectin_mask),\n", ")\n", "\n", "neg_filter_requests = df[neg_filter_selection_mask][\"request_id\"].unique()\n", "pos_filter_requests = df[pos_filter_selectin_mask][\"request_id\"].unique()\n", "# looking for very strong signal here:\n", "# listen to the positive/negative more than once\n", "# disliked one of the clips\n", "unique_requests = set(pos_filter_requests).intersection(neg_filter_requests)\n", "print(\n", " \"total\",\n", " df[\"request_id\"].nunique(),\n", " \"selected\",\n", " len(unique_requests),\n", " len(unique_requests) / df[\"request_id\"].nunique(),\n", ")" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:14:00.170078Z", "start_time": "2024-03-03T01:13:59.041907Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "190966\n" ] } ], "source": [ "df_slice = df[df[\"request_id\"].isin(set(unique_requests))].copy()\n", "print(df_slice[\"request_id\"].nunique())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Need to kick out the ones has gpt prompt -- these are pairs with different text inputs" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:14:00.408109Z", "start_time": "2024-03-03T01:14:00.171307Z" } }, "outputs": [ { "data": { "text/plain": [ "137044" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# don't have continue at\n", "df_slice[df_slice[\"continue_at\"].isna()][\"request_id\"].nunique()" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:14:00.471434Z", "start_time": "2024-03-03T01:14:00.409284Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "190966\n" ] } ], "source": [ "final_filtered_requests = df_slice[\"request_id\"].unique()\n", "print(len(final_filtered_requests))" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:14:01.921435Z", "start_time": "2024-03-03T01:14:00.472682Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "189056 1910\n", "(378112, 65) (3820, 65)\n" ] } ], "source": [ "train_requests, val_requests = train_test_split(\n", " sorted(list(final_filtered_requests)), test_size=0.01, random_state=42\n", ")\n", "print(len(train_requests), len(val_requests))\n", "\n", "train_df = df_slice[df_slice[\"request_id\"].isin(set(train_requests))].copy()\n", "val_df = df_slice[df_slice[\"request_id\"].isin(set(val_requests))].copy()\n", "train_df = train_df.sort_values(by=[\"request_id\", \"preference\"])\n", "train_df = train_df.reset_index()\n", "val_df = val_df.sort_values(by=[\"request_id\", \"preference\"])\n", "val_df = val_df.reset_index()\n", "\n", "print(train_df.shape, val_df.shape)" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:14:01.942677Z", "start_time": "2024-03-03T01:14:01.922697Z" } }, "outputs": [], "source": [ "def reshift(arr):\n", " sem_start_idx = 0\n", " sem_end_idx = len(arr) - 1\n", " semantic_arr = arr[:, :SEMANTIC_N_CODEBOOKS]\n", " coarse_arr = arr[:, SEMANTIC_N_CODEBOOKS:]\n", "\n", " coarse_start_idx = int(round(sem_start_idx * COARSE_RATE_HZ / SEMANTIC_RATE_HZ))\n", " coarse_end_idx = int(round(sem_end_idx * COARSE_RATE_HZ / SEMANTIC_RATE_HZ))\n", " assert sem_end_idx >= 0 and coarse_start_idx >= 0\n", " assert not (sem_end_idx > len(semantic_arr) or coarse_end_idx > len(coarse_arr))\n", "\n", " # get array segments\n", " arr_s = semantic_arr[sem_start_idx:sem_end_idx, :].copy()\n", " arr_c = coarse_arr[coarse_start_idx:coarse_end_idx, :].copy()\n", " assert arr_s.max() <= SEMANTIC_PAD_TOKEN\n", " assert arr_c.max() <= COARSE_PAD_TOKEN\n", " assert len(arr_s) == len(arr_c)\n", " # concat and stack\n", " if len(arr_c) < N_TOKENS_AUDIO:\n", " arr_c = np.pad(\n", " arr_c,\n", " ((0, N_TOKENS_AUDIO - len(arr_c)), (0, 0)),\n", " constant_values=COARSE_PAD_TOKEN,\n", " mode=\"constant\",\n", " )\n", " arr_s = np.pad(\n", " arr_s,\n", " ((0, N_TOKENS_AUDIO - len(arr_s)), (0, 0)),\n", " constant_values=SEMANTIC_PAD_TOKEN,\n", " mode=\"constant\",\n", " )\n", " arr = np.concatenate([arr_s, arr_c], axis=-1)\n", " arr = arr.astype(np.uint16)\n", " assert arr.shape == (N_TOKENS_AUDIO, SEMANTIC_N_CODEBOOKS + COARSE_N_CODEBOOKS)\n", " return arr\n", "\n", "\n", "def make_dataset(df, is_val=False):\n", " dset_type = \"val\" if is_val else \"tr\"\n", " out_mmap_path = os.path.join(OUT_DATA_DIR, f\"data_{dset_type}.bin\")\n", " out_metas_path = os.path.join(OUT_DATA_DIR, f\"meta_{dset_type}.jsonl\")\n", " out_info_filepath = os.path.join(OUT_DATA_DIR, f\"info_{dset_type}.json\")\n", "\n", " # gather the data\n", " _ = np.memmap(out_mmap_path, dtype=np.uint16, mode=\"w+\", shape=(1,))\n", " n_offs = 0\n", " tot_duration_dict = defaultdict(int)\n", " datasets_info = defaultdict(dict)\n", " n = 0\n", " for i, row in tqdm.tqdm(df.iterrows(), total=len(df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " assert row[\"preference\"] == (i % 2 == 1)\n", " # make mmap -- two different paths\n", " local_path = (\n", " f\"{NPZ_DIR}/{row['s3_id']}.npz\"\n", " if row[\"is_7b\"]\n", " else f\"/app/suno/data/dpo/7b_npz/{row['s3_id']}.npz\"\n", " )\n", " if not os.path.exists(local_path):\n", " # print(row, local_path)\n", " raise ValueError()\n", " # print(local_path)\n", " # print( np.load(local_path))\n", " try:\n", " arr = (\n", " np.load(local_path)[\"v3.0_raw\"]\n", " if row[\"is_7b\"]\n", " else np.load(local_path)[\"v2_raw\"]\n", " )\n", " except Exception as e:\n", " print(local_path)\n", " raise e \n", " assert arr.shape[0] <= 3000\n", " assert arr.shape[1] == 13\n", " arr_duration = arr.shape[0] / 25\n", " # print(arr.shape)\n", " arr = reshift(arr)\n", " # print(\"after shift and pad\", arr.shape)\n", " arr = arr.reshape(\n", " -1,\n", " )\n", " # print(arr.shape)\n", " out_mm = np.memmap(\n", " out_mmap_path,\n", " dtype=np.uint16,\n", " mode=\"r+\",\n", " shape=(n_offs + arr.size,),\n", " )\n", " out_mm[n_offs : n_offs + arr.size] = arr\n", " # print(f\"offset is: {n_offs}\")\n", " # break\n", " # write it once\n", " out_mm.flush()\n", " del out_mm\n", " add_metas = []\n", " add_meta = {\n", " \"dataset\": f\"perference_{int(row['preference'])}\",\n", " \"id\": row[\"s3_id\"], # this is the row s3_id\n", " \"start_s\": row[\"total_start_s\"] if row[\"total_start_s\"] else 0,\n", " \"end_s\": (\n", " None if not row[\"total_clip_s\"] else row[\"total_clip_s\"]\n", " ), # for full clips we do know it has an edding, other wise, we don't know\n", " \"original_duration_s\": arr_duration,\n", " \"vocal_start_s\": None, # these are unfortunately missing for now\n", " \"vocal_end_s\": None, # these are unfortunately missing for now\n", " \"tags\": [\n", " row[\"tags\"] if row[\"tags\"] else \"\"\n", " ], # tags is a list, do you know :)\n", " \"text\": row[\"prompt\"] if row[\"prompt\"] else \"\",\n", " }\n", " add_metas.append(add_meta)\n", " tot_duration_dict[row[\"preference\"]] += arr_duration\n", " write_jsonl(\n", " add_metas,\n", " os.path.join(out_metas_path),\n", " do_append=bool(n_offs != 0),\n", " )\n", " if \"idx_list\" not in datasets_info[add_meta[\"dataset\"]]:\n", " datasets_info[add_meta[\"dataset\"]][\"idx_list\"] = [n]\n", " else:\n", " datasets_info[add_meta[\"dataset\"]][\"idx_list\"].append(n)\n", " n += 1\n", " n_offs += arr.size\n", "\n", " write_json(datasets_info, out_info_filepath)\n", " print(f\"Total {n} clips\")\n", " for k, v in tot_duration_dict.items():\n", " print(f\"{round(v / 60 / 60):,} hours of {k}\")\n", " print(f\"Done\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Actually make" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:14:02.013290Z", "start_time": "2024-03-03T01:14:01.943786Z" } }, "outputs": [], "source": [ "# val_df[[\"request_id\", \"metadata\", \"updated_at\", \"user_id\", \"preference\"]].head()" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:14:19.279675Z", "start_time": "2024-03-03T01:14:02.014699Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 378112/378112 [00:17<00:00, 21984.47it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "6,918 hours of 378112 clips, 15.754666666666667 nodes\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "total_duration = 0\n", "for i, row in tqdm.tqdm(train_df.iterrows(), total=len(train_df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " try:\n", " assert row[\"preference\"] == (i % 2 == 1)\n", " except:\n", " print(i, row)\n", " total_duration += row[\"duration\"]\n", "print(f\"{round(total_duration / 60 / 60):,} hours of {train_df.shape[0]} clips, {train_df.shape[0] / 8 / 2 / 1500} nodes\")" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T01:15:15.011844Z", "start_time": "2024-03-03T01:14:19.280825Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 3820/3820 [00:55<00:00, 68.96it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 3820 clips\n", "35 hours of False\n", "35 hours of True\n", "Done\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "make_dataset(val_df, is_val=True)" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T02:46:08.366798Z", "start_time": "2024-03-03T01:15:15.014051Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████| 378112/378112 [1:30:42<00:00, 69.47it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 378112 clips\n", "3,472 hours of False\n", "3,443 hours of True\n", "Done\n" ] } ], "source": [ "make_dataset(train_df, is_val=False)" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2024-01-29T19:46:47.549860Z", "start_time": "2024-01-29T19:46:47.548015Z" } }, "source": [ "# Validation" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T02:46:08.412957Z", "start_time": "2024-03-03T02:46:08.371288Z" } }, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "test_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_val.jsonl\"))\n", "test_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_val.json\"))\n", "mm = mm.reshape(-1, 3008, 13)\n", "assert len(mm) == len(test_metas)\n", "assert mm[:100, :, 0].min() >= 0\n", "assert mm[:100, :, 0].max() <= 4000\n", "assert mm[:100, :, 1:].min() >= 0\n", "assert mm[:100, :, 1:].max() <= 2048" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T02:46:14.553093Z", "start_time": "2024-03-03T02:46:08.415156Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3820\n" ] } ], "source": [ "# randomly listen to some stuff\n", "from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "from suno_utils.tasks.dac_2c_12cb import (\n", " encode as codec_encode,\n", " decode_stream_to_full_audio as codec_decode,\n", " EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", ")\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"2\"\n", "_ = preload_codec_models(\"/app/suno/tony/v3/dac_2c_25x12.pt\", device=\"cuda\")\n", "assert len(test_metas) == len(mm)\n", "idx_list = list(range(len(test_metas)))\n", "# random.shuffle(idx_list)\n", "# idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "print(len(mm))" ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T02:46:17.330454Z", "start_time": "2024-03-03T02:46:14.555575Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tags: True ['rock gritty rebellious']\n", "negative example\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "positive example\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "text: True [Verse] \n", "Ridin' on the wild side, through the neon lights\n", "Rebel hearts beating, in the city nights \n", "Syka blyat, idi nahoi, eblan \n", "Marchin' to the rhythm, of the streets we claim (oh, claim) \n", "No fear, no surrender, in this wild game\n", "\n", "[Chorus]\n", "Syka blyat, idi nahoi eblan\n", "Scream it loud, let the world know we stand\n", "Syka blyat, idi nahoi eblan (yeah, yeah)\n", "We won't be silenced, we won't be tamed\n", "Syka blyat, idi nahoi eblan (woah-oh-oh)\n", "Breakin' free, livin' life unchained\n", "Syka blyat, idi nahoi eblan (oh, oh)\n", "This is our anthem, we won't be restrained\n" ] } ], "source": [ "idx = random.choice(test_info[\"perference_0\"][\"idx_list\"])\n", "assert \"original_duration_s\" in test_metas[idx]\n", "# positive index should be shifted by 1\n", "pos_idx = idx + 1\n", "print(\n", " \"tags:\",\n", " test_metas[idx].get(\"tags\") == test_metas[pos_idx].get(\"tags\"),\n", " test_metas[idx].get(\"tags\"),\n", ")\n", "arr = mm[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "pos_arr = mm[pos_idx, 1:].copy().astype(np.int16)[:, 1:]\n", "pad_idx_arr = np.where(arr == COARSE_PAD_TOKEN)[0]\n", "if len(pad_idx_arr) > 0:\n", " arr = arr[: pad_idx_arr[0], :]\n", "pos_pad_idx_arr = np.where(pos_arr == COARSE_PAD_TOKEN)[0]\n", "if len(pos_pad_idx_arr) > 0:\n", " pos_arr = pos_arr[: pos_pad_idx_arr[0], :]\n", "a = codec_decode(arr)\n", "print(\"negative example\")\n", "a.play(compress=False)\n", "pos_a = codec_decode(pos_arr)\n", "print(\"positive example\")\n", "pos_a.play(compress=False)\n", "print(\n", " \"text:\",\n", " test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"),\n", " test_metas[idx].get(\"text\"),\n", ")" ] }, { "cell_type": "code", "execution_count": 38, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T02:46:58.651022Z", "start_time": "2024-03-03T02:46:17.332268Z" } }, "outputs": [], "source": [ "from collections import Counter\n", "c = Counter()\n", "for _, row in df_slice.iterrows():\n", " # print(row[\"metadata\"])\n", " for k in ast.literal_eval(row[\"metadata\"]).keys():\n", " c[k] += 1" ] }, { "cell_type": "code", "execution_count": 39, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T02:46:58.655473Z", "start_time": "2024-03-03T02:46:58.652967Z" } }, "outputs": [], "source": [ "# original_npz_path = f\"/app/suno/data/dpo/7b_npz/{test_metas[idx]['id']}.npz\"\n", "# original_npz_path = \"/app/suno/data/dpo/7b_npz/729c3011-f672-4ccd-8d82-1cbf2b52ff69.npz\"\n", "# original_arr = np.load(original_npz_path)[\"v2_raw\"]" ] }, { "cell_type": "code", "execution_count": 40, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T02:46:58.723058Z", "start_time": "2024-03-03T02:46:58.657440Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1910 0\n" ] } ], "source": [ "total_bad = 0\n", "total_good = 0\n", "for idx in range(len(test_metas)):\n", " if idx % 2 == 0:\n", " pos_idx = idx + 1\n", " if test_metas[idx].get(\"tags\") != test_metas[pos_idx].get(\"tags\"):\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " else:\n", " total_good += 1\n", "print(total_good, total_bad)" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T02:46:58.820761Z", "start_time": "2024-03-03T02:46:58.724944Z" } }, "outputs": [], "source": [ "train_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_tr.json\"))" ] }, { "cell_type": "code", "execution_count": 42, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T02:46:58.863707Z", "start_time": "2024-03-03T02:46:58.822513Z" } }, "outputs": [], "source": [ "n_neg_tr = train_info[\"perference_0\"][\"idx_list\"]\n", "n_pos_tr = train_info[\"perference_1\"][\"idx_list\"]\n", "assert len(n_pos_tr) == len(n_neg_tr)" ] }, { "cell_type": "code", "execution_count": 43, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T02:46:58.933288Z", "start_time": "2024-03-03T02:46:58.865375Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total samples 378112 (378112, 65)\n" ] } ], "source": [ "total_iters = (len(n_neg_tr) + len(n_pos_tr))\n", "print(\"total samples\", total_iters, train_df.shape)" ] }, { "cell_type": "code", "execution_count": 47, "metadata": { "ExecuteTime": { "end_time": "2024-03-03T03:28:25.423484Z", "start_time": "2024-03-03T03:28:25.420872Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1 epoch per batch 4, total 5908.0\n" ] } ], "source": [ "print(\"1 epoch per batch 4, total\", total_iters / 8 / 4 / 2)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.13" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 2 }