{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import tempfile\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from suno_utils.utils.text import read_jsonl\n",
    "from suno_utils.utils.s3 import read_from_s3, upload_s3_files\n",
    "import emoji"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_metas = read_from_s3(\"s3://suno-data/datasets/harvest/freesound/segment_metas.jsonl\", read_f=read_jsonl)\n",
    "print(f\"{len(raw_metas):,} tracks with {sum([m['duration_s'] for m in raw_metas])/60/60:,.1f}h total\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.Series([m['duration_s'] for m in raw_metas]).clip(upper=250).hist(bins=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_raw = pd.DataFrame.from_dict(raw_metas)\n",
    "df_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def strip_text(text):\n",
    "    if text is None:\n",
    "        return text\n",
    "    text = text.lower()\n",
    "    text = re.sub(r'[^a-zA-Z0-9\\s]', '', text)  # Remove non-alphanumeric except spaces\n",
    "    text = re.sub(r'\\s+', ' ', text)  # Replace multiple spaces with single space\n",
    "    return text.strip()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5",
   "metadata": {},
   "source": [
    "## Filter freesound"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_filter = df_raw.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_filter['cap1'] = df_filter['cap1'].apply(lambda x: x.strip() if isinstance(x, str) else None)\n",
    "df_filter['cap1'] = df_filter['cap1'].apply(lambda x: x if isinstance(x,str) and len(x) > 5 else None)\n",
    "\n",
    "top_values = df_filter['cap1'].value_counts().head(25)\n",
    "\n",
    "plt.figure(figsize=(10, 6))\n",
    "top_values.plot(kind='barh')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_filter['cap2'] = df_filter['cap2'].apply(lambda x: x.strip() if isinstance(x, str) else None)\n",
    "df_filter['cap2'] = df_filter['cap2'].apply(lambda x: x if isinstance(x,str) and len(x) > 5 else None)\n",
    "\n",
    "captions_to_filter = [\n",
    "    \"Sound  by Stolting Media Group.\",\n",
    "    \"Recorded in the context of the good-sounds.\",\n",
    "    \"The recordings found in this foley pack are all original recordings made by myself and they are intended for free public use to help content creators with their commercial or non-commercial projects.\",\n",
    "    \"I had a goal for this pack.\",\n",
    "    \"This sound or sounds was created through the help of VST's, time shifting, parametric EQ, cutting, sampling, layering and many other methods of sound manipulation.\",\n",
    "    \"Recorded and processed at 24bit 48kHz using the Tascam DR-40 Linear PCM Recorder.\",\n",
    "    \"One of a series of recordings I made from a Hikari Monos CV.\",\n",
    "    \"Drum Loop by Stolting Media Group.\",\n",
    "    \"Made using Audacity.\",\n",
    "    \"Record in a Hotel with condenser Mic with no processing added (so its Raw).\",\n",
    "    \"A collection of samples/loops intended for personal and commercial music production.\",\n",
    "    \"CASA DA MÚSICA's VIRTUAL GAMELANThe Casa da Música's Javanese Gamelan aggregates more than 250 sounds recorded from its various parts: Bonang, Gambang, Gender, Kenong, Saron, Kethuk, Kempyang, Gongs and Drums.\",\n",
    "    \"Music/sound effect constructive kit.\"\n",
    "]\n",
    "captions_to_filter = [strip_text(caption) for caption in captions_to_filter]\n",
    "\n",
    "captions_to_replace = {\n",
    "    \"Single note sampled from an analog synthesizer by Modular Samples.\": \"Single note sampled from an analog synthesizer.\",\n",
    "    \"One-shot from Versilian Studios Chamber Orchestra 2: Community Edition sampling project.\": \"One-shot from Chamber Orchestra.\",\n",
    "    \"Drum Sound by Stolting Media Group.\": \"Drum Sound.\",\n",
    "    \"Drum Loop by Stolting Media Group.\": \"Drum Loop.\",\n",
    "}\n",
    "\n",
    "def filter_cap2(caption):\n",
    "    if not isinstance(caption, str):\n",
    "        return None\n",
    "    if strip_text(caption) in captions_to_filter:\n",
    "        return None\n",
    "    if caption in captions_to_replace:\n",
    "        return captions_to_replace[caption]\n",
    "    return caption\n",
    "\n",
    "df_filter['cap2_filtered'] = df_filter['cap2'].apply(filter_cap2)\n",
    "\n",
    "top_values = df_filter['cap2_filtered'].value_counts().head(25)\n",
    "\n",
    "plt.figure(figsize=(10, 6))\n",
    "top_values.plot(kind='barh')\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9",
   "metadata": {},
   "outputs": [],
   "source": [
    "top_values = df_filter['cap2'].value_counts().head(25)\n",
    "\n",
    "plt.figure(figsize=(10, 6))\n",
    "top_values.plot(kind='barh')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(df_filter))\n",
    "df_filter = df_filter[df_filter['duration_s'] <= 15.0]\n",
    "print(len(df_filter))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11",
   "metadata": {},
   "outputs": [],
   "source": [
    "def safe_process_tags(value):\n",
    "    \"\"\"Safely convert a value to a list of strings\"\"\"\n",
    "    if value is None:\n",
    "        return []\n",
    "    \n",
    "    # Handle NaN values\n",
    "    try:\n",
    "        if pd.isna(value):\n",
    "            return []\n",
    "    except (TypeError, ValueError):\n",
    "        pass\n",
    "    \n",
    "    # Handle lists\n",
    "    if isinstance(value, list):\n",
    "        return [str(item).strip() for item in value if str(item).strip() and str(item).strip().lower() != 'nan']\n",
    "    \n",
    "    # Handle strings\n",
    "    if isinstance(value, str) and value.strip() and value.strip().lower() != 'nan':\n",
    "        return [value.strip()]\n",
    "    \n",
    "    return []\n",
    "\n",
    "# Process each column separately then combine\n",
    "df_filter['tags_processed'] = df_filter['tags'].apply(safe_process_tags)\n",
    "\n",
    "# Clean up\n",
    "df_filter = df_filter.drop(['tags'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_filter['tags_processed'] = df_filter['tags_processed'].apply(lambda x: list(set(x)) if len(x) > 0 else None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fastest_dedupe_tags(tag_series):\n",
    "    # Convert to DataFrame with exploded tags\n",
    "    df_exploded = (tag_series.to_frame('tags')\n",
    "                   .explode('tags')\n",
    "                   .reset_index())\n",
    "    \n",
    "    # Quick filter and clean\n",
    "    mask = df_exploded['tags'].notna()\n",
    "    df_exploded = df_exploded[mask].copy()\n",
    "    \n",
    "    if df_exploded.empty:\n",
    "        return tag_series.apply(lambda x: [])\n",
    "    \n",
    "    df_exploded['tags'] = df_exploded['tags'].astype(str).str.strip()\n",
    "    df_exploded = df_exploded[df_exploded['tags'] != '']\n",
    "    \n",
    "    # Vectorized normalization\n",
    "    df_exploded['norm'] = df_exploded['tags'].apply(strip_text)\n",
    "    \n",
    "    # Remove empty normalized and deduplicate\n",
    "    df_exploded = df_exploded[df_exploded['norm'] != '']\n",
    "    df_exploded = df_exploded.drop_duplicates(['index', 'norm'], keep='first')\n",
    "    \n",
    "    # Regroup\n",
    "    return (df_exploded.groupby('index')['tags']\n",
    "            .apply(list)\n",
    "            .reindex(tag_series.index, fill_value=[]))\n",
    "\n",
    "# Apply\n",
    "df_filter['tags_processed'] = fastest_dedupe_tags(df_filter['tags_processed'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_to_filter = [\n",
    "    \"sound\",\n",
    "    \"h4n-zoom\",\n",
    "    \"sm-58\",\n",
    "    \"iitm\",\n",
    "    \"icassp2013-dataset\",\n",
    "    \"mridangam-stroke-dataset\",\n",
    "    \"good-sounds\",\n",
    "    \"neumann-U87\",\n",
    "    \"velocity\",\n",
    "    \"bpm\"\n",
    "]\n",
    "tags_to_filter = [strip_text(tag) for tag in tags_to_filter]\n",
    "\n",
    "def filter_tags(tags):\n",
    "    if not isinstance(tags, list):\n",
    "        return None\n",
    "    kept_tags = []\n",
    "    for tag in tags:\n",
    "        if strip_text(tag) not in tags_to_filter:\n",
    "            kept_tags.append(tag)\n",
    "\n",
    "    if len(kept_tags) == 0:\n",
    "        return None\n",
    "    \n",
    "    return kept_tags\n",
    "\n",
    "df_filter['tags_processed'] = df_filter['tags_processed'].apply(filter_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15",
   "metadata": {},
   "outputs": [],
   "source": [
    "top_values = df_filter['tags_processed'].explode().value_counts().head(25)\n",
    "\n",
    "plt.figure(figsize=(10, 6))\n",
    "top_values.plot(kind='barh')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_to_clean = ['cap1', 'cap2', 'cap2_filtered']\n",
    "df_filter[columns_to_clean] = df_filter[columns_to_clean].replace(r'\\.$', '', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(df_filter))\n",
    "df_filter.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18",
   "metadata": {},
   "source": [
    "### Save Filtered Metas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19",
   "metadata": {},
   "outputs": [],
   "source": [
    "upload_path = \"s3://suno-data/datasets/harvest/freesound/metas_v0.jsonl\"\n",
    "with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=True) as temp_file:\n",
    "    df_filter.to_json(temp_file.name, orient='records', lines=True)\n",
    "    temp_filename = temp_file.name\n",
    "    results = upload_s3_files(from_local_filepaths=[temp_filename], to_s3_filepaths=[upload_path])\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_clean",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}