{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from suno_utils.utils.text import read_json, read_jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1",
   "metadata": {},
   "outputs": [],
   "source": [
    "whosampled = read_json(\"/home/sara/whosampled.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2",
   "metadata": {},
   "outputs": [],
   "source": [
    "whosampled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3",
   "metadata": {},
   "outputs": [],
   "source": [
    "id_to_related = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4",
   "metadata": {},
   "outputs": [],
   "source": [
    "for k,v in whosampled.items():\n",
    "    if k not in id_to_related:\n",
    "        id_to_related[k] = []\n",
    "    for k2,v2 in v.items():\n",
    "        id_to_related[k].append(v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_id_to_data = {}\n",
    "\n",
    "for k,v in id_to_related.items():\n",
    "    for x in v:\n",
    "        track_id = str(x['track']['unique_url'])\n",
    "        unique_id_to_data[track_id] = dict(\n",
    "            track_name=x['track']['name'],\n",
    "            artists=[xx['name'] for xx in x['by']],\n",
    "            year=x['track']['year'],\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_id_to_variations = {}\n",
    "variation_types = set()\n",
    "\n",
    "for k,v in id_to_related.items():\n",
    "    for x in v:\n",
    "        variation_type = x['action_name']\n",
    "        track_id = str(x['track']['unique_url'])\n",
    "        other_track_id = str(x['other_track']['unique_url'])\n",
    "        variation_types.add(variation_type)\n",
    "\n",
    "        if variation_type == \"was remixed in\":\n",
    "            if track_id not in unique_id_to_variations:\n",
    "                unique_id_to_variations[track_id] = []\n",
    "            unique_id_to_variations[track_id].append((\"remix\", other_track_id))\n",
    "        elif variation_type == \"was covered in\":\n",
    "            if track_id not in unique_id_to_variations:\n",
    "                unique_id_to_variations[track_id] = []\n",
    "            unique_id_to_variations[track_id].append((\"cover\", other_track_id))\n",
    "        elif variation_type == \"was sampled in\":\n",
    "            if track_id not in unique_id_to_variations:\n",
    "                unique_id_to_variations[track_id] = []\n",
    "            unique_id_to_variations[track_id].append((\"sampled\", other_track_id))\n",
    "\n",
    "print(variation_types)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Process unique_id_to_variations into separate dicts by type\n",
    "unique_id_to_remix = {}\n",
    "unique_id_to_cover = {}\n",
    "unique_id_to_sampled = {}\n",
    "\n",
    "for track_id, variations in unique_id_to_variations.items():\n",
    "    for variation_type, other_track_id in variations:\n",
    "        if variation_type == \"remix\":\n",
    "            if track_id not in unique_id_to_remix:\n",
    "                unique_id_to_remix[track_id] = []\n",
    "            unique_id_to_remix[track_id].append(other_track_id)\n",
    "        elif variation_type == \"cover\":\n",
    "            if track_id not in unique_id_to_cover:\n",
    "                unique_id_to_cover[track_id] = []\n",
    "            unique_id_to_cover[track_id].append(other_track_id)\n",
    "        elif variation_type == \"sampled\":\n",
    "            if track_id not in unique_id_to_sampled:\n",
    "                unique_id_to_sampled[track_id] = []\n",
    "            unique_id_to_sampled[track_id].append(other_track_id)\n",
    "\n",
    "print(f\"Separate dictionaries created:\")\n",
    "print(f\"  unique_id_to_remix: {len(unique_id_to_remix)} tracks\")\n",
    "print(f\"  unique_id_to_cover: {len(unique_id_to_cover)} tracks\")\n",
    "print(f\"  unique_id_to_sampled: {len(unique_id_to_sampled)} tracks\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot_variation_histogram(data_dict, variation_name, ylabel_desc):\n",
    "    \"\"\"Plot histogram for a variation type with three subplots for different ranges.\"\"\"\n",
    "    \n",
    "    # Get the counts\n",
    "    counts = [len(v) for v in data_dict.values()]\n",
    "    \n",
    "    # Split into three groups: 0-10, 10-1000, 1000+\n",
    "    counts_0_to_10 = [count for count in counts if count < 10]\n",
    "    counts_10_to_1000 = [count for count in counts if 10 <= count < 1000]\n",
    "    counts_1000_plus = [count for count in counts if count >= 1000]\n",
    "    \n",
    "    # Create three subplots\n",
    "    fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(20, 6))\n",
    "    fig.suptitle(variation_name.upper(), fontsize=16, fontweight='bold')\n",
    "    \n",
    "    # Plot 1: Songs with 0-10\n",
    "    ax1.hist(counts_0_to_10, bins=10, edgecolor='black', alpha=0.7, color='steelblue')\n",
    "    ax1.set_xlabel(f'{ylabel_desc} per Track')\n",
    "    ax1.set_ylabel('Frequency')\n",
    "    ax1.set_title(f'Songs with 0-10 {variation_name} (n={len(counts_0_to_10)})')\n",
    "    ax1.grid(axis='y', alpha=0.3)\n",
    "    \n",
    "    # Plot 2: Songs with 10-1000\n",
    "    ax2.hist(counts_10_to_1000, bins=50, edgecolor='black', alpha=0.7, color='coral')\n",
    "    ax2.set_xlabel(f'{ylabel_desc} per Track')\n",
    "    ax2.set_ylabel('Frequency')\n",
    "    ax2.set_title(f'Songs with 10-1000 {variation_name} (n={len(counts_10_to_1000)})')\n",
    "    ax2.grid(axis='y', alpha=0.3)\n",
    "    \n",
    "    # Plot 3: Songs with 1000+\n",
    "    if counts_1000_plus:\n",
    "        ax3.hist(counts_1000_plus, bins=20, edgecolor='black', alpha=0.7, color='lightgreen')\n",
    "        ax3.set_xlabel(f'{ylabel_desc} per Track')\n",
    "        ax3.set_ylabel('Frequency')\n",
    "        ax3.set_title(f'Songs with 1000+ {variation_name} (n={len(counts_1000_plus)})')\n",
    "        ax3.grid(axis='y', alpha=0.3)\n",
    "    else:\n",
    "        ax3.text(0.5, 0.5, f'No songs with\\n1000+ {variation_name}', \n",
    "                 ha='center', va='center', fontsize=14, transform=ax3.transAxes)\n",
    "        ax3.set_title(f'Songs with 1000+ {variation_name} (n=0)')\n",
    "    \n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "    \n",
    "    # Print statistics\n",
    "    print(\"=\" * 60)\n",
    "    print(f\"{variation_name.upper()} - SONGS WITH 0-10:\")\n",
    "    print(f\"  Total tracks: {len(counts_0_to_10)}\")\n",
    "    if counts_0_to_10:\n",
    "        print(f\"  Mean {variation_name}: {sum(counts_0_to_10) / len(counts_0_to_10):.2f}\")\n",
    "        print(f\"  Max {variation_name}: {max(counts_0_to_10)}\")\n",
    "        print(f\"  Min {variation_name}: {min(counts_0_to_10)}\")\n",
    "    \n",
    "    print(\"\\n\" + \"=\" * 60)\n",
    "    print(f\"{variation_name.upper()} - SONGS WITH 10-1000:\")\n",
    "    print(f\"  Total tracks: {len(counts_10_to_1000)}\")\n",
    "    if counts_10_to_1000:\n",
    "        print(f\"  Mean {variation_name}: {sum(counts_10_to_1000) / len(counts_10_to_1000):.2f}\")\n",
    "        print(f\"  Max {variation_name}: {max(counts_10_to_1000)}\")\n",
    "        print(f\"  Min {variation_name}: {min(counts_10_to_1000)}\")\n",
    "    \n",
    "    print(\"\\n\" + \"=\" * 60)\n",
    "    print(f\"{variation_name.upper()} - SONGS WITH 1000+:\")\n",
    "    print(f\"  Total tracks: {len(counts_1000_plus)}\")\n",
    "    if counts_1000_plus:\n",
    "        print(f\"  Mean {variation_name}: {sum(counts_1000_plus) / len(counts_1000_plus):.2f}\")\n",
    "        print(f\"  Max {variation_name}: {max(counts_1000_plus)}\")\n",
    "        print(f\"  Min {variation_name}: {min(counts_1000_plus)}\")\n",
    "    \n",
    "    print(\"\\n\" + \"=\" * 60)\n",
    "    print(f\"{variation_name.upper()} - OVERALL:\")\n",
    "    print(f\"  Total tracks: {len(counts)}\")\n",
    "    if counts:\n",
    "        print(f\"  Mean {variation_name}: {sum(counts) / len(counts):.2f}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_variation_histogram(unique_id_to_variations, \"references\", \"Number of References\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_variation_histogram(unique_id_to_remix, \"remixes\", \"Number of Remixes\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_variation_histogram(unique_id_to_cover, \"covers\", \"Number of Covers\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_variation_histogram(unique_id_to_sampled, \"samples\", \"Number of Times Sampled\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13",
   "metadata": {},
   "outputs": [],
   "source": [
    "victor_who_sampled_remix = read_jsonl(\"/home/sara/who_sampled_victor/who_sampled/data/remix.jsonl\")\n",
    "victor_who_sampled_cover = read_jsonl(\"/home/sara/who_sampled_victor/who_sampled/data/cover.jsonl\")\n",
    "victor_who_sampled_sampled = read_jsonl(\"/home/sara/who_sampled_victor/who_sampled/data/sample.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14",
   "metadata": {},
   "outputs": [],
   "source": [
    "victor_who_sampled_remix[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15",
   "metadata": {},
   "outputs": [],
   "source": [
    "victor_who_sampled_sampled[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def parse_timings(timings_str):\n",
    "    \"\"\"\n",
    "    Parse timing information from a string.\n",
    "    \n",
    "    Args:\n",
    "        timings_str: String containing timestamps like \"0:34, 2:46 and 3:31 (and throughout)\"\n",
    "    \n",
    "    Returns:\n",
    "        tuple: (list of timestamps in seconds as floats, boolean if 'throughout' is present)\n",
    "    \n",
    "    Example:\n",
    "        >>> parse_timings(\"Sample appears at\\n\\n0:34\\n, 2:46\\nand 3:31\\n\\n(and throughout)\")\n",
    "        ([34.0, 166.0, 211.0], True)\n",
    "    \"\"\"\n",
    "    # Find all timestamps in format H:MM:SS, MM:SS, or M:SS\n",
    "    timestamp_pattern = r'(\\d+):(\\d+)(?::(\\d+))?'\n",
    "    matches = re.findall(timestamp_pattern, timings_str)\n",
    "    \n",
    "    # Convert timestamps to seconds\n",
    "    timestamps = []\n",
    "    for match in matches:\n",
    "        hours_or_minutes = int(match[0])\n",
    "        seconds_or_minutes = int(match[1])\n",
    "        seconds = match[2]\n",
    "        \n",
    "        if seconds:  # Format is H:MM:SS\n",
    "            total_seconds = hours_or_minutes * 3600 + seconds_or_minutes * 60 + int(seconds)\n",
    "        else:  # Format is MM:SS or M:SS\n",
    "            total_seconds = hours_or_minutes * 60 + seconds_or_minutes\n",
    "        \n",
    "        timestamps.append(float(total_seconds))\n",
    "    \n",
    "    # Check if \"throughout\" is present (case-insensitive)\n",
    "    throughout = 'throughout' in timings_str.lower()\n",
    "    \n",
    "    return timestamps, throughout\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analyze is_throughout and output_timestamps lengths\n",
    "from collections import Counter\n",
    "real_samples = []\n",
    "mashups = []\n",
    "\n",
    "total_samples = len(victor_who_sampled_sampled)\n",
    "is_throughout_count = 0\n",
    "timestamp_length_counts = Counter()\n",
    "\n",
    "for sample in victor_who_sampled_sampled:\n",
    "    output = sample[0]\n",
    "    name = output['song']\n",
    "    output_timestamps, is_throughout = parse_timings(output['timings'])\n",
    "\n",
    "    if re.search(r'\\bmash[-\\s]?up\\b', name, re.IGNORECASE):\n",
    "        mashups.append(sample)\n",
    "    elif len(output_timestamps) >=4 or is_throughout:\n",
    "        real_samples.append(sample)\n",
    "\n",
    "    if is_throughout:\n",
    "        is_throughout_count += 1\n",
    "    \n",
    "    timestamp_length_counts[len(output_timestamps)] += 1\n",
    "\n",
    "# Report results\n",
    "print(\"=\" * 70)\n",
    "print(\"ANALYSIS OF victor_who_sampled_sampled\")\n",
    "print(\"=\" * 70)\n",
    "print(f\"\\nTotal samples: {total_samples}\")\n",
    "print()\n",
    "\n",
    "# is_throughout statistics\n",
    "print(f\"Outputs with is_throughout=True: {is_throughout_count}\")\n",
    "print(f\"Percentage: {100 * is_throughout_count / total_samples:.2f}%\")\n",
    "print()\n",
    "\n",
    "# output_timestamps length statistics\n",
    "print(\"Output timestamps length distribution:\")\n",
    "print(\"-\" * 70)\n",
    "print(f\"{'Length':<15} {'Count':<15} {'Percentage':<15}\")\n",
    "print(\"-\" * 70)\n",
    "\n",
    "for length in sorted(timestamp_length_counts.keys()):\n",
    "    count = timestamp_length_counts[length]\n",
    "    percentage = 100 * count / total_samples\n",
    "    print(f\"{length:<15} {count:<15} {percentage:<15.2f}%\")\n",
    "\n",
    "print(\"-\" * 70)\n",
    "print(f\"{'Total':<15} {sum(timestamp_length_counts.values()):<15} {100.0:<15.2f}%\")\n",
    "print(\"=\" * 70)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(real_samples) / total_samples)\n",
    "print(len(real_samples))\n",
    "print(len(mashups) / total_samples)\n",
    "print(len(mashups))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19",
   "metadata": {},
   "outputs": [],
   "source": [
    "for m in mashups:\n",
    "    print(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_id_to_sampled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_id_to_data['/Alton-McClain-%26-Destiny/My-Destiny/']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_id_to_data['/Alton-McClain-%26-Destiny/My-Destiny/']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_clean",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}