{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from suno_utils.utils.text import read_jsonl, read_json, write_jsonl, write_json\n",
    "import pandas as pd\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1",
   "metadata": {},
   "outputs": [],
   "source": [
    "remixes = read_jsonl(\"/home/sara/task_data/trax_beat_who_discogs_combined_1117.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2",
   "metadata": {},
   "outputs": [],
   "source": [
    "remix_df = pd.DataFrame(remixes)\n",
    "print(len(remix_df))\n",
    "print(remixes[0].keys())\n",
    "remix_df.sample(n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import unicodedata\n",
    "import numpy as np\n",
    "\n",
    "# --- Normalization helpers ---\n",
    "\n",
    "def normalize_text(s: str) -> str:\n",
    "    \"\"\"Lowercase, strip, remove accents and non-alphanumeric chars (except spaces).\"\"\"\n",
    "    if not isinstance(s, str):\n",
    "        return \"\"\n",
    "    s = s.strip().lower()\n",
    "    # Normalize unicode accents\n",
    "    s = unicodedata.normalize(\"NFKD\", s)\n",
    "    s = \"\".join(ch for ch in s if not unicodedata.combining(ch))\n",
    "    # Keep alnum and space\n",
    "    cleaned = []\n",
    "    for ch in s:\n",
    "        if ch.isalnum() or ch.isspace():\n",
    "            cleaned.append(ch)\n",
    "        # normalize common separators to space\n",
    "        elif ch in [\"-\", \"_\", \"/\", \"&\", \",\", \";\", \":\"]:\n",
    "            cleaned.append(\" \")\n",
    "        # else drop\n",
    "    s = \"\".join(cleaned)\n",
    "    # collapse multiple spaces\n",
    "    s = re.sub(r\"\\s+\", \" \", s).strip()\n",
    "    return s\n",
    "\n",
    "\n",
    "def normalize_year(value):\n",
    "    \"\"\"Extract a plausible year (int) from release_date-like values.\n",
    "\n",
    "    Accept years between 1500 and the current year; otherwise return np.nan.\n",
    "    \"\"\"\n",
    "    if pd.isna(value):\n",
    "        return np.nan\n",
    "    # If it's already an int-like year\n",
    "    try:\n",
    "        year = int(str(value)[:4])\n",
    "    except (ValueError, TypeError):\n",
    "        return np.nan\n",
    "    from datetime import datetime\n",
    "\n",
    "    current_year = datetime.utcnow().year\n",
    "    if 1500 <= year <= current_year:\n",
    "        return year\n",
    "    return np.nan\n",
    "\n",
    "\n",
    "def normalize_artists(artists):\n",
    "    \"\"\"Normalize artists field to a frozenset of normalized names.\n",
    "\n",
    "    Handles lists of strings or a single string with separators.\n",
    "    \"\"\"\n",
    "    # Explicitly handle missing values without triggering array truth-value issues\n",
    "    if artists is None:\n",
    "        return frozenset()\n",
    "    if isinstance(artists, float) and pd.isna(artists):\n",
    "        return frozenset()\n",
    "\n",
    "    names = []\n",
    "    if isinstance(artists, (list, tuple, set, np.ndarray)):\n",
    "        names = list(artists)\n",
    "    elif isinstance(artists, str):\n",
    "        # Split on common separators for multi-artist strings\n",
    "        parts = re.split(r\"[,/&]|feat\\.|ft\\.\", artists, flags=re.IGNORECASE)\n",
    "        names = parts\n",
    "    else:\n",
    "        return frozenset()\n",
    "\n",
    "    norm_names = [normalize_text(n) for n in names if isinstance(n, str) and normalize_text(n)]\n",
    "    return frozenset(norm_names)\n",
    "\n",
    "\n",
    "# --- Create normalized columns on remix_df ---\n",
    "\n",
    "remix_df[\"norm_title\"] = remix_df[\"title\"].apply(normalize_text)\n",
    "remix_df[\"norm_album_name\"] = remix_df[\"album_name\"].apply(normalize_text)\n",
    "remix_df[\"norm_label\"] = remix_df[\"label\"].apply(normalize_text)\n",
    "remix_df[\"norm_release_year\"] = remix_df[\"release_date\"].apply(normalize_year)\n",
    "remix_df[\"norm_artists\"] = remix_df[\"artists\"].apply(normalize_artists)\n",
    "\n",
    "# Quick sanity check\n",
    "remix_df[[\"title\", \"release_date\", \"norm_title\", \"norm_release_year\"]].head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Identify duplicate candidate groups ---\n",
    "\n",
    "# Filter out rows missing BOTH title and release_date; these will be dropped entirely\n",
    "has_title = remix_df[\"norm_title\"].astype(bool)\n",
    "has_year = remix_df[\"norm_release_year\"].notna()\n",
    "\n",
    "# Per plan: rows missing both title AND release date are removed from final output\n",
    "valid_for_grouping = has_title | has_year\n",
    "base_df = remix_df[valid_for_grouping].copy()\n",
    "\n",
    "# For grouping, if year is missing, use 0 as dummy per plan\n",
    "base_df[\"group_year\"] = base_df[\"norm_release_year\"].fillna(0).astype(int)\n",
    "\n",
    "# Group by normalized title and group_year\n",
    "group_cols = [\"norm_title\", \"group_year\"]\n",
    "\n",
    "# We'll mark candidate duplicates via a flag column\n",
    "base_df[\"dup_candidate\"] = False\n",
    "\n",
    "# duration tolerance in seconds\n",
    "DUR_TOL = 5\n",
    "\n",
    "# Compute representative duration and flag rows within ±5 seconds or with null duration\n",
    "for (title, year), grp_idx in base_df.groupby(group_cols).groups.items():\n",
    "    grp = base_df.loc[grp_idx]\n",
    "    durations = grp[\"duration\"].dropna().astype(float)\n",
    "    if durations.empty:\n",
    "        # if no durations, treat all as candidates (they at least share title/year)\n",
    "        base_df.loc[grp_idx, \"dup_candidate\"] = len(grp) > 1\n",
    "        continue\n",
    "    rep_dur = durations.median()\n",
    "    within_tol = grp[\"duration\"].sub(rep_dur).abs() <= DUR_TOL\n",
    "    null_dur = grp[\"duration\"].isna()\n",
    "    candidates_mask = within_tol | null_dur\n",
    "    if candidates_mask.sum() > 1:\n",
    "        base_df.loc[grp_idx, \"dup_candidate\"] = candidates_mask\n",
    "\n",
    "# Keep only groups with at least two dup_candidate rows\n",
    "candidate_df = base_df[base_df[\"dup_candidate\"]].copy()\n",
    "\n",
    "# For inspection: number of candidate groups and example\n",
    "num_groups = candidate_df.groupby(group_cols).ngroups\n",
    "num_rows = len(candidate_df)\n",
    "num_original = len(remix_df)\n",
    "print(f\"Original rows: {num_original}\")\n",
    "print(f\"Rows considered for grouping: {len(base_df)}\")\n",
    "print(f\"Duplicate candidate rows: {num_rows} across {num_groups} groups\")\n",
    "\n",
    "candidate_df[[\"title\", \"release_date\", \"duration\", \"norm_title\", \"group_year\"]].head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Compute match statistics per duplicate group ---\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "core_fields = [\"title\", \"artists\", \"album_name\", \"label\", \"release_date\", \"duration\"]\n",
    "\n",
    "\n",
    "def summarize_group(grp: pd.DataFrame):\n",
    "    summary = {\n",
    "        \"group_size\": len(grp),\n",
    "        \"indices\": grp.index.tolist(),\n",
    "        \"field_modes\": {},\n",
    "        \"field_mode_counts\": {},\n",
    "    }\n",
    "    for col in core_fields:\n",
    "        vals = grp[col].tolist()\n",
    "        # Treat lists as tuples for hashing in Counter\n",
    "        processed_vals = []\n",
    "        for v in vals:\n",
    "            if isinstance(v, list):\n",
    "                processed_vals.append(tuple(v))\n",
    "            else:\n",
    "                processed_vals.append(v)\n",
    "        # Drop NaNs for mode computation\n",
    "        non_null = [v for v in processed_vals if not (isinstance(v, float) and pd.isna(v))]\n",
    "        if not non_null:\n",
    "            mode_val, mode_count = None, 0\n",
    "        else:\n",
    "            cnt = Counter(non_null)\n",
    "            mode_val, mode_count = cnt.most_common(1)[0]\n",
    "            # Convert tuples back to lists for readability\n",
    "            if isinstance(mode_val, tuple):\n",
    "                mode_val = list(mode_val)\n",
    "        summary[\"field_modes\"][col] = mode_val\n",
    "        summary[\"field_mode_counts\"][col] = mode_count\n",
    "    return summary\n",
    "\n",
    "\n",
    "grouped = candidate_df.groupby(group_cols, group_keys=False)\n",
    "match_summaries = []\n",
    "for (title, year), grp in grouped:\n",
    "    if len(grp) < 2:\n",
    "        continue\n",
    "    s = summarize_group(grp)\n",
    "    s[\"norm_title\"] = title\n",
    "    s[\"group_year\"] = int(year)\n",
    "    match_summaries.append(s)\n",
    "\n",
    "match_summary_df = pd.DataFrame(match_summaries)\n",
    "print(\"Match summary groups:\", len(match_summary_df))\n",
    "match_summary_df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Merge duplicate candidate groups into single rows ---\n",
    "\n",
    "list_like_cols = [\"artists\", \"source_artists\", \"source_ids\", \"source_votes\"]\n",
    "id_like_cols = [\"output_id\", \"id\", \"s3_filepath\"]\n",
    "\n",
    "\n",
    "def most_frequent_non_null(series: pd.Series):\n",
    "    values = series.tolist()\n",
    "    processed_vals = []\n",
    "    for v in values:\n",
    "        if isinstance(v, list):\n",
    "            processed_vals.append(tuple(v))\n",
    "        else:\n",
    "            processed_vals.append(v)\n",
    "    non_null = [v for v in processed_vals if not (isinstance(v, float) and pd.isna(v))]\n",
    "    if not non_null:\n",
    "        return np.nan\n",
    "    cnt = Counter(non_null)\n",
    "    mode_val, _ = cnt.most_common(1)[0]\n",
    "    if isinstance(mode_val, tuple):\n",
    "        mode_val = list(mode_val)\n",
    "    return mode_val\n",
    "\n",
    "\n",
    "def merge_group(grp: pd.DataFrame):\n",
    "    merged = {}\n",
    "    # Start from core schema\n",
    "    for col in remix_df.columns:\n",
    "        if col in list_like_cols or col in id_like_cols:\n",
    "            continue\n",
    "        if col == \"duration\":\n",
    "            # respect ±5s tolerance around group median\n",
    "            durations = grp[\"duration\"].dropna().astype(float)\n",
    "            if durations.empty:\n",
    "                merged[col] = np.nan\n",
    "            else:\n",
    "                rep = durations.median()\n",
    "                within_tol = grp[\"duration\"].sub(rep).abs() <= DUR_TOL\n",
    "                cand_durs = grp.loc[within_tol, \"duration\"].dropna().astype(float)\n",
    "                if cand_durs.empty:\n",
    "                    merged[col] = rep\n",
    "                else:\n",
    "                    merged[col] = most_frequent_non_null(cand_durs)\n",
    "        else:\n",
    "            merged[col] = most_frequent_non_null(grp[col])\n",
    "\n",
    "    # List-like / multi-valued columns: union of elements\n",
    "    for col in list_like_cols:\n",
    "        if col not in grp.columns:\n",
    "            continue\n",
    "        union_vals = []\n",
    "        for v in grp[col]:\n",
    "            # Robust NaN / missing check without ambiguous array truth values\n",
    "            is_missing = False\n",
    "            if v is None:\n",
    "                is_missing = True\n",
    "            elif isinstance(v, float) and pd.isna(v):\n",
    "                is_missing = True\n",
    "            if is_missing:\n",
    "                continue\n",
    "            if isinstance(v, list):\n",
    "                union_vals.extend(v)\n",
    "            else:\n",
    "                union_vals.append(v)\n",
    "        # Normalize strings inside\n",
    "        norm_union = []\n",
    "        for v in union_vals:\n",
    "            if isinstance(v, str):\n",
    "                nv = v.strip()\n",
    "                if nv:\n",
    "                    norm_union.append(nv)\n",
    "            else:\n",
    "                norm_union.append(v)\n",
    "        # Drop duplicates preserving order\n",
    "        seen = set()\n",
    "        deduped = []\n",
    "        for v in norm_union:\n",
    "            key = v\n",
    "            try:\n",
    "                if key in seen:\n",
    "                    continue\n",
    "                seen.add(key)\n",
    "            except TypeError:\n",
    "                # unhashable (e.g., list); just append\n",
    "                pass\n",
    "            deduped.append(v)\n",
    "        merged[col] = deduped if deduped else np.nan\n",
    "\n",
    "    # Identifier-like columns: most frequent non-null value, plus optional merged list\n",
    "    for col in id_like_cols:\n",
    "        if col not in grp.columns:\n",
    "            continue\n",
    "        merged[col] = most_frequent_non_null(grp[col])\n",
    "        all_vals = [v for v in grp[col].tolist() if not (isinstance(v, float) and pd.isna(v))]\n",
    "        merged[f\"merged_{col}\"] = list(dict.fromkeys(all_vals)) if all_vals else []\n",
    "\n",
    "    # Attach metadata about merge\n",
    "    merged[\"merged_from_indices\"] = grp.index.tolist()\n",
    "    merged[\"merged_group_size\"] = len(grp)\n",
    "\n",
    "    return pd.Series(merged)\n",
    "\n",
    "\n",
    "merged_rows = []\n",
    "for (title, year), grp in candidate_df.groupby(group_cols, group_keys=False):\n",
    "    if len(grp) < 2:\n",
    "        continue\n",
    "    merged_row = merge_group(grp)\n",
    "    merged_row[\"norm_title\"] = title\n",
    "    merged_row[\"group_year\"] = int(year)\n",
    "    merged_rows.append(merged_row)\n",
    "\n",
    "merged_df = pd.DataFrame(merged_rows)\n",
    "print(\"Merged duplicate rows:\", len(merged_df))\n",
    "merged_df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Build final deduplicated DataFrame ---\n",
    "\n",
    "# Indices that were part of merged duplicate groups\n",
    "merged_indices = set()\n",
    "for idx_list in merged_df[\"merged_from_indices\"]:\n",
    "    merged_indices.update(idx_list)\n",
    "\n",
    "# Rows that did not participate in any merge\n",
    "singleton_df = remix_df.drop(index=list(merged_indices))\n",
    "\n",
    "# Combine merged rows with untouched singletons\n",
    "remix_dedup_df = pd.concat([singleton_df, merged_df], ignore_index=True, sort=False)\n",
    "\n",
    "print(\"Original rows:\", len(remix_df))\n",
    "print(\"After deduplication:\", len(remix_dedup_df))\n",
    "print(\"Merged groups:\", len(merged_df))\n",
    "\n",
    "# Simple sanity checks\n",
    "remix_dedup_df.sample(n=5)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8",
   "metadata": {},
   "outputs": [],
   "source": [
    "remix_dedup_df.drop(columns=[\"norm_title\", \"norm_album_name\", \"norm_label\", \"norm_artists\", \"norm_release_year\"], inplace=True)\n",
    "remix_dedup_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9",
   "metadata": {},
   "outputs": [],
   "source": [
    "remix_dedup_df.drop(columns=[\"merged_output_id\", \"merged_id\", \"merged_s3_filepath\", \"merged_from_indices\", \"merged_group_size\", \"group_year\"], inplace=True)\n",
    "remix_dedup_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "def extract_first_source_id(val):\n",
    "    \"\"\"Safely extract a single source_id from source_ids.\n",
    "\n",
    "    - If list-like, use the first element (if present)\n",
    "    - If missing/NaN/None or empty, return np.nan\n",
    "    - Otherwise, pass the value through\n",
    "    \"\"\"\n",
    "    if val is None:\n",
    "        return np.nan\n",
    "    if isinstance(val, float) and pd.isna(val):\n",
    "        return np.nan\n",
    "    if isinstance(val, list):\n",
    "        return val[0] if val else np.nan\n",
    "    return val\n",
    "\n",
    "\n",
    "# Create a robust scalar source_id column and drop the original list-like column\n",
    "remix_dedup_df[\"source_id\"] = remix_dedup_df[\"source_ids\"].apply(extract_first_source_id)\n",
    "remix_dedup_df.drop(columns=[\"source_ids\"], inplace=True)\n",
    "remix_dedup_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11",
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Save deduplicated remixes to JSONL ---\n",
    "\n",
    "output_path = \"/home/sara/task_data/trax_beat_who_discogs_combined_1117_dedupe.jsonl\"\n",
    "print(\"Saving to:\", output_path)\n",
    "\n",
    "# remix_dedup_df was built above; convert rows to plain dicts\n",
    "records = remix_dedup_df.to_dict(orient=\"records\")\n",
    "write_jsonl(records, output_path)\n",
    "\n",
    "print(\"Wrote\", len(records), \"rows to\", output_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12",
   "metadata": {},
   "outputs": [],
   "source": [
    "records[0].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13",
   "metadata": {},
   "outputs": [],
   "source": [
    "remix_dedup_df.sample(n=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14",
   "metadata": {},
   "outputs": [],
   "source": [
    "bundle_metas = read_jsonl(\"/app2/suno/data/diffusion/sfx/v6/combined_v3_w_extreme_metas_opus.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15",
   "metadata": {},
   "outputs": [],
   "source": [
    "for meta in bundle_metas:\n",
    "    s3_path = meta[\"s3_filepath\"]\n",
    "    if \"truncated\" not in s3_path:\n",
    "        print(meta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(bundle_metas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17",
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "from botocore.exceptions import ClientError\n",
    "\n",
    "# Prefix we care about\n",
    "OPUS_TRUNCATED_PREFIX = \"s3://suno-data/datasets/bundles/v5/sfx_all_processed/opus_audio_truncated/\"\n",
    "\n",
    "s3 = boto3.client(\"s3\")\n",
    "\n",
    "\n",
    "def parse_s3_url(url: str):\n",
    "    \"\"\"Split an s3:// URL into (bucket, key).\"\"\"\n",
    "    assert url.startswith(\"s3://\"), f\"Not an s3 url: {url}\"\n",
    "    bucket, key = url[5:].split(\"/\", 1)\n",
    "    return bucket, key\n",
    "\n",
    "\n",
    "# Build list of objects whose *filename* contains spaces\n",
    "space_renames = []  # (bucket, old_key, new_key)\n",
    "\n",
    "for meta in bundle_metas:\n",
    "    s3_path = meta.get(\"s3_filepath\")\n",
    "    if not isinstance(s3_path, str):\n",
    "        continue\n",
    "    if not s3_path.startswith(OPUS_TRUNCATED_PREFIX):\n",
    "        continue\n",
    "\n",
    "    bucket, key = parse_s3_url(s3_path)\n",
    "    if \"/\" in key:\n",
    "        prefix_part, fname = key.rsplit(\"/\", 1)\n",
    "    else:\n",
    "        prefix_part, fname = \"\", key\n",
    "\n",
    "    if \" \" in fname:\n",
    "        new_fname = fname.replace(\" \", \"_\")\n",
    "        new_key = f\"{prefix_part}/{new_fname}\" if prefix_part else new_fname\n",
    "        space_renames.append((bucket, key, new_key))\n",
    "\n",
    "print(f\"Found {len(space_renames)} objects with spaces in filename under opus_audio_truncated.\")\n",
    "space_renames[:10]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18",
   "metadata": {},
   "outputs": [],
   "source": [
    "# DRY RUN: inspect a few planned renames without changing S3\n",
    "for bucket, old_key, new_key in space_renames[:20]:\n",
    "    print(f\"s3://{bucket}/{old_key}  ->  s3://{bucket}/{new_key}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ACTUAL RENAME (PARALLEL): copy each object to the new key, then delete the old\n",
    "# Note: S3 has no native rename; this is copy + delete.\n",
    "\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
    "from botocore.exceptions import ClientError\n",
    "\n",
    "# Tune this based on how aggressive you want to be and S3 limits\n",
    "MAX_WORKERS = 32\n",
    "\n",
    "def rename_one(triple):\n",
    "    bucket, old_key, new_key = triple\n",
    "    old_url = f\"s3://{bucket}/{old_key}\"\n",
    "    new_url = f\"s3://{bucket}/{new_key}\"\n",
    "    try:\n",
    "        s3.copy_object(\n",
    "            Bucket=bucket,\n",
    "            CopySource={\"Bucket\": bucket, \"Key\": old_key},\n",
    "            Key=new_key,\n",
    "        )\n",
    "        s3.delete_object(Bucket=bucket, Key=old_key)\n",
    "        return (old_url, new_url, None)\n",
    "    except ClientError as e:\n",
    "        return (old_url, new_url, e)\n",
    "\n",
    "total = len(space_renames)\n",
    "print(f\"Starting parallel rename of {total} objects with {MAX_WORKERS} workers...\")\n",
    "\n",
    "done = 0\n",
    "errors = 0\n",
    "\n",
    "with ThreadPoolExecutor(max_workers=MAX_WORKERS) as ex:\n",
    "    futures = [ex.submit(rename_one, triple) for triple in space_renames]\n",
    "    for fut in as_completed(futures):\n",
    "        old_url, new_url, err = fut.result()\n",
    "        done += 1\n",
    "        if err is not None:\n",
    "            errors += 1\n",
    "            print(f\"ERROR renaming {old_url} -> {new_url}: {err}\")\n",
    "        if done % 1000 == 0:\n",
    "            print(f\"{done}/{total} renames done (errors so far: {errors})\")\n",
    "\n",
    "print(f\"Finished renaming. Total: {done}, errors: {errors}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20",
   "metadata": {},
   "outputs": [],
   "source": [
    "# OPTIONAL: update bundle_metas in-memory to point to the new S3 paths\n",
    "\n",
    "old_to_new = {}\n",
    "for bucket, old_key, new_key in space_renames:\n",
    "    old_to_new[f\"s3://{bucket}/{old_key}\"] = f\"s3://{bucket}/{new_key}\"\n",
    "\n",
    "updated = 0\n",
    "for meta in bundle_metas:\n",
    "    s3_path = meta.get(\"s3_filepath\")\n",
    "    if s3_path in old_to_new:\n",
    "        meta[\"s3_filepath\"] = old_to_new[s3_path]\n",
    "        updated += 1\n",
    "\n",
    "print(f\"Updated {updated} bundle_metas entries to new s3_filepath values.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21",
   "metadata": {},
   "outputs": [],
   "source": [
    "write_jsonl(bundle_metas, \"/app2/suno/data/diffusion/sfx/v6/combined_v3_w_extreme_metas_opus_name_norm.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22",
   "metadata": {},
   "outputs": [],
   "source": [
    "bundle_metas[0].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23",
   "metadata": {},
   "outputs": [],
   "source": [
    "bundle_metas[100]['tags']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24",
   "metadata": {},
   "outputs": [],
   "source": [
    "for b in bundle_metas:\n",
    "    all_tags = \", \".join(b['tags'])\n",
    "    if \"tempo:\" in all_tags and \"key:\" in all_tags:\n",
    "        print(b['tags'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = read_jsonl(\"/app2/suno/data/diffusion/sfx/v2/metas_val.jsonl\")\n",
    "for b in test:\n",
    "    all_tags = \", \".join(b['tags'])\n",
    "    if \"tempo:\" in all_tags and \"key:\" in all_tags:\n",
    "        print(b['tags'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_json(\"/home/sara/sara/ytm_tagged_metas.jsonl\", lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "rows = []\n",
    "path = \"ytm_tagged_metas.jsonl\"\n",
    "\n",
    "with open(path, \"r\") as f:\n",
    "    for i, line in enumerate(f, start=1):\n",
    "        line = line.strip()\n",
    "        if not line:\n",
    "            continue\n",
    "        try:\n",
    "            rows.append(json.loads(line))\n",
    "        except json.JSONDecodeError as e:\n",
    "            print(f\"Skipping bad line {i}: {e}\")\n",
    "\n",
    "df = pd.DataFrame(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29",
   "metadata": {},
   "outputs": [],
   "source": [
    "bot_warnings = []\n",
    "other_warnings = []\n",
    "for r in rows:\n",
    "    msg = r['fail_message']\n",
    "    if msg is not None:\n",
    "        if \"sign in\" in msg.lower():\n",
    "            bot_warnings.append(r)\n",
    "        elif \"available\" not in msg.lower():\n",
    "            other_warnings.append(r)\n",
    "\n",
    "print(len(bot_warnings))\n",
    "print(len(rows))\n",
    "print(len(other_warnings))\n",
    "print(len(bot_warnings) / len(rows))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30",
   "metadata": {},
   "outputs": [],
   "source": [
    "for r in other_warnings:\n",
    "    print(r['fail_message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31",
   "metadata": {},
   "outputs": [],
   "source": [
    "parsed_data = read_jsonl(\"test_remix_source_1118_v1.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32",
   "metadata": {},
   "outputs": [],
   "source": [
    "parsed_data[0].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33",
   "metadata": {},
   "outputs": [],
   "source": [
    "missed = 0\n",
    "total_found = 0\n",
    "\n",
    "for parse in parsed_data:\n",
    "    dataset = parse['data_source']\n",
    "    if \"discogs\" not in dataset.lower():\n",
    "        if not parse[\"sources_parsed\"]:\n",
    "            confidence = parse.get(\"parsing_confidence\")\n",
    "            parse[\"sources_parsed\"] = True\n",
    "            total_found += 1\n",
    "            if confidence < 0.5:\n",
    "                missed += 1\n",
    "\n",
    "print(missed)\n",
    "print(total_found)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34",
   "metadata": {},
   "outputs": [],
   "source": [
    "missing = 0\n",
    "for parse in parsed_data:\n",
    "    if not parse[\"sources_parsed\"]:\n",
    "        missing += 1\n",
    "\n",
    "print(missing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35",
   "metadata": {},
   "outputs": [],
   "source": [
    "write_jsonl(parsed_data, \"/home/sara/task_data/trax_beat_who_discogs_combined_1118_some_missing_sources.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36",
   "metadata": {},
   "outputs": [],
   "source": [
    "for parse in parsed_data:\n",
    "    if not parse[\"sources_parsed\"]:\n",
    "        print(parse)\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_clean",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}