{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "80c06cf2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 2, "id": "bbc054a8", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "markdown", "id": "d02ee475", "metadata": {}, "source": [ "### make en-us annotation queue to find 2m+2f speakers" ] }, { "cell_type": "code", "execution_count": 3, "id": "ec8e0bb6", "metadata": {}, "outputs": [], "source": [ "import time\n", "import os\n", "import tqdm\n", "import shutil\n", "import random\n", "import funcy\n", "import json\n", "import numpy as np\n", "import multiprocessing\n", "import pandas as pd\n", "from suno_utils.audio import Audio\n", "from suno_utils.audio.conversion import play_audio\n", "from suno_utils.utils.podcasts import (\n", " load_podcast_db, find_in_raw_feeds, load_rss_feed, multicore_apply, _load_rss_text, get_file_name,\n", " _clean_episode_url\n", ")\n", "\n", "PODCAST_DATA_DIR = \"/mnt/data-ssd-1/data/podcasts/\"\n", "\n", "podcast_df = load_podcast_db(os.path.join(PODCAST_DATA_DIR, \"meta/podcastindex_feeds.db\"), anchor_only=True)\n", "summaries_df = pd.read_csv(os.path.join(PODCAST_DATA_DIR, \"meta/summaries.csv\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "7b6ef1c9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "08f91e2c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "6bfe9875", "metadata": {}, "source": [ "### find via basic filtering" ] }, { "cell_type": "code", "execution_count": 19, "id": "4f86f87e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "10964 candidates found.\n" ] } ], "source": [ "candidate_ids = set(\n", " podcast_df[\n", " (podcast_df[\"language\"] == \"en-us\") &\n", " (podcast_df[\"episode_count\"].fillna(0) >= 20) &\n", " (podcast_df[\"episode_count\"].fillna(0) <= 50) &\n", " (podcast_df[\"host\"] == \"anchor.fm\") &\n", " (podcast_df[\"generator\"] == \"Anchor Podcasts\")\n", " ][\"uid\"]\n", ")\n", "# TODO: stratify male/female?\n", "# TODO: find ones with monologues?\n", "# TODO: certain topics?\n", "print(len(candidate_ids), \"candidates found.\")" ] }, { "cell_type": "code", "execution_count": null, "id": "354b8c8a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "77f7d231", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ba28bfcb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "6e8192ba", "metadata": {}, "source": [ "### find podcasts via summary" ] }, { "cell_type": "code", "execution_count": 5, "id": "78e3f873", "metadata": {}, "outputs": [], "source": [ "# raw_feed_dir = os.path.join(PODCAST_DATA_DIR, \"bulk_rss_feeds/raw_feeds\")\n", "# rss_filepaths = [\n", "# os.path.join(raw_feed_dir, filename)\n", "# for filename in os.listdir(raw_feed_dir)\n", "# ]" ] }, { "cell_type": "code", "execution_count": 6, "id": "7d3e28c2", "metadata": {}, "outputs": [], "source": [ "# def _foo(rss_filepath):\n", "# podcast_id = get_file_name(rss_filepath)\n", "# # short circuit if not in full text\n", "# fulltext = _load_rss_text(rss_filepath)\n", "# if (\n", "# re.search(r\"\\bspanish\\b\", fulltext, flags=re.IGNORECASE) and \n", "# re.search(r\"\\benglish\\b\", fulltext, flags=re.IGNORECASE)\n", "# ) or (\n", "# re.search(r\"\\bspanglish\\b\", fulltext, flags=re.IGNORECASE)\n", "# ):\n", "# pass\n", "# else:\n", "# return None\n", "# # check proper fields\n", "# try:\n", "# rss_info = load_rss_feed(rss_filepath)\n", "# except:\n", "# return None\n", "# text_fields = [\n", "# rss_info.get(\"title\", \"\"), \n", "# rss_info.get(\"subtitle\", \"\"),\n", "# rss_info.get(\"summary\", \"\"),\n", "# rss_info.get(\"description\", \"\"),\n", "# ]\n", "# fulltext = \"\".join([e for e in text_fields if e is not None])\n", "# if (\n", "# re.search(r\"\\bspanish\\b\", fulltext, flags=re.IGNORECASE) and \n", "# re.search(r\"\\benglish\\b\", fulltext, flags=re.IGNORECASE)\n", "# ) or (\n", "# re.search(r\"\\bspanglish\\b\", fulltext, flags=re.IGNORECASE)\n", "# ):\n", "# return podcast_id\n", "# return None\n", "\n", "# out = multicore_apply(\n", "# _foo,\n", "# rss_filepaths,\n", "# n_cores=20, chunksize=100_000, map_chunksize=100, quiet=False,\n", "# )\n", "# found_uids = [e for e in out if e is not None]\n", "# print(len(found_uids), \"potential candidates found.\")" ] }, { "cell_type": "code", "execution_count": null, "id": "571c584c", "metadata": {}, "outputs": [], "source": [ "# # check if anchor, language ok and long enough\n", "# valid_ids = set(podcast_df[\n", "# (podcast_df[\"language\"].fillna(\"\").str[:2].isin(set([\"en\", \"es\"]))) &\n", "# (podcast_df[\"episode_count\"].fillna(0) >= 3) &\n", "# (podcast_df[\"host\"] == \"anchor.fm\") &\n", "# (podcast_df[\"generator\"] == \"Anchor Podcasts\")\n", "# ][\"uid\"])\n", "# candidate_ids = set([uid for uid in found_uids if uid in valid_ids])\n", "# print(len(candidate_ids), \"spanglish candidates found.\")" ] }, { "cell_type": "code", "execution_count": null, "id": "86bb8bc4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d7cabce6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "52c0a19b", "metadata": {}, "source": [ "### assemble annotation output" ] }, { "cell_type": "code", "execution_count": 20, "id": "75270902", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "4875 good candidates found.\n" ] } ], "source": [ "# assemble info and exclude things like 'music' and anchor emails\n", "info_df = summaries_df[\n", " summaries_df[\"uid\"].isin(candidate_ids) &\n", " ~summaries_df[\"tags\"].fillna(\"\").str.contains(\"music\") & \n", " ~summaries_df[\"author_email\"].fillna(\"\").str.endswith(\"anchor.fm\")\n", "].reset_index(drop=True)\n", "# add info like n episodes, last episode URL etc\n", "tmp_map = podcast_df.drop_duplicates(subset=[\"uid\"], keep=False).set_index(\"uid\")\n", "info_df = info_df[info_df[\"uid\"].isin(set(tmp_map.index))].reset_index(drop=True)\n", "info_df[\"rss_url\"] = info_df[\"uid\"].map(tmp_map[\"url\"])\n", "info_df[\"link\"] = info_df[\"uid\"].map(tmp_map[\"link\"])\n", "info_df[\"language\"] = info_df[\"uid\"].map(tmp_map[\"language\"])\n", "info_df[\"n_episodes\"] = info_df[\"uid\"].map(tmp_map[\"episode_count\"])\n", "info_df[\"newest_episode_pubdate\"] = info_df[\"uid\"].map(tmp_map[\"newest_item_pubdate\"])\n", "info_df[\"oldest_episode_pubdate\"] = info_df[\"uid\"].map(tmp_map[\"oldest_item_pubdate\"])\n", "episode_links = []\n", "for uid in info_df[\"uid\"].values:\n", " rss_filepath = os.path.join(PODCAST_DATA_DIR, f\"bulk_rss_feeds/raw_feeds/{uid}.feed\")\n", " rss_info = load_rss_feed(rss_filepath)\n", " episode_links.append([e[\"audio_url\"] for e in rss_info[\"episodes\"][:3]])\n", "info_df[\"episode_url_0\"] = [e[0] if len(e) >= 1 else None for e in episode_links]\n", "info_df[\"episode_url_1\"] = [e[1] if len(e) >= 2 else None for e in episode_links]\n", "info_df[\"episode_url_2\"] = [e[2] if len(e) >= 3 else None for e in episode_links]\n", "print(info_df.shape[0], \"good candidates found.\")" ] }, { "cell_type": "code", "execution_count": 21, "id": "08d2db40", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Fantastic Gatherings\n", "This being the 'Greatest Podcast Ever Made!', we invite our listeners to join us on a wild podcastic adventure through our favorite fandoms, gaming software, comic series, scifi book and much more! (Literally 3 guys that found time and a way to get together and record us hanging out and you get the edited version of it.)\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "row = info_df.iloc[0]\n", "print(row[\"title\"])\n", "print(row[\"summary\"])\n", "play_audio(row[\"episode_url_0\"])" ] }, { "cell_type": "code", "execution_count": 23, "id": "93b3de89", "metadata": {}, "outputs": [], "source": [ "info_df.to_csv(\"tmp/us_en_anchor.csv\", index=False)\n", "info_df.sample(250, random_state=6006).to_csv(\"tmp/us_en_anchor_mini.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "fe3c4e75", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5a742937", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9bd89e56", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "e754eb1c", "metadata": {}, "source": [ "### gender classification" ] }, { "cell_type": "code", "execution_count": 45, "id": "81317088", "metadata": {}, "outputs": [], "source": [ "info_df = pd.read_csv(\"tmp/us_en_anchor.csv\")" ] }, { "cell_type": "code", "execution_count": 39, "id": "3256f4c9", "metadata": {}, "outputs": [], "source": [ "import gender_guesser.detector as gender\n", "d = gender.Detector()" ] }, { "cell_type": "code", "execution_count": 46, "id": "10dfcb28", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4875/4875 [00:00<00:00, 33236.07it/s]\n" ] } ], "source": [ "data = []\n", "for idx, row in tqdm.tqdm(info_df.iterrows(), total=info_df.shape[0]):\n", " name = row[\"author\"]\n", " if len(name.split()) != 2:\n", "# print(\"undef\", \"-\", name)\n", " continue\n", " gender = d.get_gender(name.split()[0])\n", "# print(gender, \"-\", name)\n", " if gender == \"female\":\n", " data.append(row)" ] }, { "cell_type": "code", "execution_count": 47, "id": "da0c116a", "metadata": {}, "outputs": [], "source": [ "df = pd.DataFrame(data)" ] }, { "cell_type": "code", "execution_count": 51, "id": "0b910bf2", "metadata": {}, "outputs": [], "source": [ "df.sample(50, random_state=6006).to_csv(\"tmp/us_en_anchor_female_mini.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "c5d969b5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "36e09b86", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9d621d4a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fdc497a9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "82d7950f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "4219f48f", "metadata": {}, "source": [ "## Annotate" ] }, { "cell_type": "code", "execution_count": 25, "id": "35f83db2", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "uid 7cfd2cf2-fe0a-5f6e-a07c-19d74932ad77\n", "title Why Can’t I? | Health Coaching, Confidence & D...\n", "summary Sarah Hagstrom is a plant-based health coach t...\n", "tags business;entrepreneurship\n", "author Sarah Hagstrom\n", "author_email sarah@plantbasedcoach.biz\n", "rss_url https://anchor.fm/s/2f4077b4/podcast/rss\n", "link https://plantbasedcoach.biz\n", "language en-us\n", "n_episodes 31\n", "newest_episode_pubdate 2020-08-02 19:00:00\n", "oldest_episode_pubdate 2019-07-07 15:00:00\n", "episode_url_0 https://d3ctxlq1ktw2nl.cloudfront.net/staging/...\n", "episode_url_1 https://d3ctxlq1ktw2nl.cloudfront.net/staging/...\n", "episode_url_2 https://d3ctxlq1ktw2nl.cloudfront.net/staging/...\n", "Name: 0, dtype: object" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "info_df = pd.read_csv(\"tmp/us_en_anchor_mini.csv\")\n", "info_df.iloc[0]" ] }, { "cell_type": "code", "execution_count": 26, "id": "8c7aba10", "metadata": {}, "outputs": [], "source": [ "import json\n", "from suno_utils.pipeline.labeler import SunoAPIClient\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " SUNO_API_TOKEN = json.load(f)[\"suno_api_token\"]\n", "\n", "suno_client = SunoAPIClient(token=SUNO_API_TOKEN) " ] }, { "cell_type": "code", "execution_count": 27, "id": "1565323b", "metadata": {}, "outputs": [], "source": [ "def get_task_metadata(uid, audio_url_0, audio_url_1, audio_url_2, language, title, description, url, n_episodes):\n", " return {\n", " \"blocks\": [\n", " {\n", " \"type\": \"audio\",\n", " \"label\": \"Audio Sample 1:\",\n", " \"data\": {\"value\": None if pd.isnull(audio_url_0) else audio_url_0},\n", " }, \n", " {\n", " \"type\": \"audio\",\n", " \"label\": \"Audio Sample 2:\",\n", " \"data\": {\"value\": None if pd.isnull(audio_url_1) else audio_url_1},\n", " }, \n", " {\n", " \"type\": \"audio\",\n", " \"label\": \"Audio Sample 3:\",\n", " \"data\": {\"value\": None if pd.isnull(audio_url_2) else audio_url_2},\n", " }, \n", " {\n", " \"type\": \"markdown\",\n", " \"data\": {\n", " \"value\": (\n", " f\"(id:{uid}, lang:{language})\\n\\n - {title}\\n\\n{description}\\n\\n{url}\\n\\n\"\n", " f\"Total Episodes: {n_episodes}\"\n", " )\n", " },\n", " }, \n", " {\n", " \"type\": \"radio\",\n", " \"label\": \"The podcast contains\",\n", " \"data\": {\"key\": \"contentType\"},\n", " \"props\": {\n", " \"options\": [\n", " {\"value\": v} \n", " for v in [\n", " \"English + Spanish (good mix)\",\n", " \"English + Spanish (one-sided mix)\",\n", " \"English + Spanish (but also en-us, other language or music)\",\n", " \"Other\",\n", " ]\n", " ],\n", " \"direction\": \"column\",\n", " },\n", " },\n", " ],\n", " \"podcast_id\": uid,\n", " }\n", "\n", "work_items = [\n", " {\n", " \"metadata\": get_task_metadata(\n", " row[\"uid\"], \n", " row[\"episode_url_0\"],\n", " row[\"episode_url_1\"],\n", " row[\"episode_url_2\"],\n", " row[\"language\"], \n", " row[\"title\"], \n", " row[\"summary\"], \n", " row[\"link\"], \n", " row[\"n_episodes\"],\n", " ),\n", " } \n", " for _, row in info_df.iterrows()\n", "]" ] }, { "cell_type": "code", "execution_count": 28, "id": "72f72422", "metadata": { "scrolled": true }, "outputs": [], "source": [ "task_name = \"US-EN Podcasts\"\n", "work_item_set = suno_client.create_work_item_set(name=task_name)\n", "remote_work_items = suno_client.create_work_items(work_items, work_item_set_id=work_item_set.id)\n", "with open(\"tmp/{}.json\".format(task_name.lower().replace(\" \", \"_\")), \"w\") as f:\n", " json.dump(remote_work_items, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "9838cae1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "11812c49", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "750c8bb8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fae2e406", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "e23e39c8", "metadata": {}, "source": [ "# TODO: here" ] }, { "cell_type": "markdown", "id": "d8587028", "metadata": {}, "source": [ "## Assemble output" ] }, { "cell_type": "code", "execution_count": null, "id": "9702d9ab", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "outreach_df = pd.read_csv(\"tmp/en_ph_list.csv\")\n", "print(outreach_df.shape[0], \"podcasts\")\n", "outreach_df.head(2)" ] }, { "cell_type": "code", "execution_count": 2, "id": "41164bc2", "metadata": {}, "outputs": [], "source": [ "import json\n", "from suno_utils.pipeline.labeler import SunoAPIClient\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " SUNO_API_TOKEN = json.load(f)[\"suno_api_token\"]\n", "\n", "suno_client = SunoAPIClient(token=SUNO_API_TOKEN) " ] }, { "cell_type": "code", "execution_count": 3, "id": "8f215a92", "metadata": {}, "outputs": [], "source": [ "from suno_utils.pipeline.labeler import WorkItemSet\n", "# labeled_results = WorkItemSet(\n", "# id=\"b42eec85-253f-4ed8-b4b9-4a9072eab232\", name=\"PHL Podcast Test\", client=suno_client\n", "# ).get_results()\n", "# labeled_results = WorkItemSet(\n", "# id=\"226ab584-ec50-4e45-9589-4757f9c935a1\", name=\"PHL Podcast gmail\", client=suno_client\n", "# ).get_results()\n", "labeled_results = WorkItemSet(\n", " id=\"80ce382e-c25f-4683-82ad-62bc90ba1aee\", name=\"PHL Podcast gmail 2\", client=suno_client\n", ").get_results()" ] }, { "cell_type": "code", "execution_count": null, "id": "17d91544", "metadata": {}, "outputs": [], "source": [ "usable_podcast_id = []\n", "for work_item in labeled_results[\"workItems\"]:\n", " podcast_id = work_item[\"metadata\"][\"podcast_id\"]\n", " for annotation in work_item[\"assignments\"]:\n", " if annotation[\"data\"].get(\"contentType\") == \"English (Filipino Accent)\":\n", " usable_podcast_id.append(podcast_id)\n", "print(len(usable_podcast_id), \"relevant podcasts found\")" ] }, { "cell_type": "code", "execution_count": null, "id": "002017dd", "metadata": {}, "outputs": [], "source": [ "outreach_df = outreach_df[outreach_df[\"id\"].isin(usable_podcast_id)]\n", "outreach_df = outreach_df.reset_index(drop=True)\n", "print(outreach_df.shape[0], \"podcasts found\")\n", "outreach_df.head(2)" ] }, { "cell_type": "code", "execution_count": 8, "id": "4f82ee88", "metadata": {}, "outputs": [], "source": [ "outreach_df.to_csv(\"outreach_en_phl.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "61d1e571", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6d66b26b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "e07bc7ec", "metadata": {}, "source": [ "## remove old tasks" ] }, { "cell_type": "code", "execution_count": 29, "id": "23ca0aa8", "metadata": {}, "outputs": [], "source": [ "import json\n", "from suno_utils.pipeline.labeler import SunoAPIClient\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " SUNO_API_TOKEN = json.load(f)[\"suno_api_token\"]\n", "\n", "suno_client = SunoAPIClient(token=SUNO_API_TOKEN) " ] }, { "cell_type": "code", "execution_count": 57, "id": "217ef71c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'deleteWorkItemSet': {'id': '5279645f-d7d5-451d-a414-8286883e5216'}}" ] }, "execution_count": 57, "metadata": {}, "output_type": "execute_result" } ], "source": [ "suno_client.delete_work_item_set(\"5279645f-d7d5-451d-a414-8286883e5216\")" ] }, { "cell_type": "code", "execution_count": null, "id": "474044d7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "aaa5ff3f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "eb044b51", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "56fc2e4c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8c86eb12", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }