{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "5ff0baea", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 2, "id": "77817f3e", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "code", "execution_count": 4, "id": "ec8e0bb6", "metadata": {}, "outputs": [], "source": [ "import time\n", "import os\n", "import tqdm\n", "import shutil\n", "import random\n", "import funcy\n", "import json\n", "import numpy as np\n", "import multiprocessing\n", "import pandas as pd\n", "from suno_utils.audio import Audio\n", "from suno_utils.audio.conversion import play_audio\n", "from suno_utils.utils.podcasts import load_podcast_db\n", "# load_podcast_db, find_in_raw_feeds, load_rss_feed, multicore_apply, _load_rss_text, get_file_name,\n", "# _clean_episode_url\n", "# )\n", "\n", "PODCAST_DATA_DIR = \"/mnt/data-ssd-1/data/podcasts/\"\n", "\n", "podcast_df = load_podcast_db(os.path.join(PODCAST_DATA_DIR, \"meta/podcastindex_feeds.db\"), anchor_only=True)" ] }, { "cell_type": "code", "execution_count": 7, "id": "e189a815", "metadata": {}, "outputs": [], "source": [ "df = podcast_df[podcast_df[\"language\"] == \"fr-ca\"]" ] }, { "cell_type": "code", "execution_count": 8, "id": "e8eba8a8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(1276, 28)" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.shape" ] }, { "cell_type": "code", "execution_count": 12, "id": "4d5aca0a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "16327" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"episode_count\"].clip(upper=50).sum()" ] }, { "cell_type": "code", "execution_count": null, "id": "e1425b7d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ce65e27c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 5, "id": "78e3f873", "metadata": {}, "outputs": [], "source": [ "raw_feed_dir = os.path.join(PODCAST_DATA_DIR, \"bulk_rss_feeds/raw_feeds\")\n", "rss_filepaths = [\n", " os.path.join(raw_feed_dir, filename)\n", " for filename in os.listdir(raw_feed_dir)\n", "]" ] }, { "cell_type": "code", "execution_count": 7, "id": "7d3e28c2", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 37/37 [16:02<00:00, 26.01s/it]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "1509 potential candidates found.\n" ] } ], "source": [ "def _foo(rss_filepath):\n", " podcast_id = get_file_name(rss_filepath)\n", " # short circuit if not in full text\n", " fulltext = _load_rss_text(rss_filepath)\n", " if (\n", " re.search(r\"\\bspanish\\b\", fulltext, flags=re.IGNORECASE) and \n", " re.search(r\"\\benglish\\b\", fulltext, flags=re.IGNORECASE)\n", " ) or (\n", " re.search(r\"\\bspanglish\\b\", fulltext, flags=re.IGNORECASE)\n", " ):\n", " pass\n", " else:\n", " return None\n", " # check proper fields\n", " try:\n", " rss_info = load_rss_feed(rss_filepath)\n", " except:\n", " return None\n", " text_fields = [\n", " rss_info.get(\"title\", \"\"), \n", " rss_info.get(\"subtitle\", \"\"),\n", " rss_info.get(\"summary\", \"\"),\n", " rss_info.get(\"description\", \"\"),\n", " ]\n", " fulltext = \"\".join([e for e in text_fields if e is not None])\n", " if (\n", " re.search(r\"\\bspanish\\b\", fulltext, flags=re.IGNORECASE) and \n", " re.search(r\"\\benglish\\b\", fulltext, flags=re.IGNORECASE)\n", " ) or (\n", " re.search(r\"\\bspanglish\\b\", fulltext, flags=re.IGNORECASE)\n", " ):\n", " return podcast_id\n", " return None\n", "\n", "out = multicore_apply(\n", " _foo,\n", " rss_filepaths,\n", " n_cores=20, chunksize=100_000, map_chunksize=100, quiet=False,\n", ")\n", "found_uids = [e for e in out if e is not None]\n", "print(len(found_uids), \"potential candidates found.\")" ] }, { "cell_type": "code", "execution_count": 8, "id": "a0eca572", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "409 spanglish candidates found.\n" ] } ], "source": [ "# check if anchor, language ok and long enough\n", "valid_ids = set(podcast_df[\n", " (podcast_df[\"language\"].fillna(\"\").str[:2].isin(set([\"en\", \"es\"]))) &\n", " (podcast_df[\"episode_count\"].fillna(0) >= 3) &\n", " (podcast_df[\"host\"] == \"anchor.fm\") &\n", " (podcast_df[\"generator\"] == \"Anchor Podcasts\")\n", "][\"uid\"])\n", "candidate_ids = set([uid for uid in found_uids if uid in valid_ids])\n", "print(len(candidate_ids), \"spanglish candidates found.\")" ] }, { "cell_type": "code", "execution_count": 42, "id": "75270902", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "123 good spanglish candidates found.\n" ] } ], "source": [ "# assemble info and exclude things like 'music' and anchor emails\n", "info_df = summaries_df[\n", " summaries_df[\"uid\"].isin(candidate_ids) &\n", " ~summaries_df[\"tags\"].fillna(\"\").str.contains(\"music\") & \n", " ~summaries_df[\"author_email\"].fillna(\"\").str.endswith(\"anchor.fm\")\n", "].reset_index(drop=True)\n", "# add info like n episodes, last episode URL etc\n", "tmp_map = podcast_df.drop_duplicates(subset=[\"uid\"], keep=False).set_index(\"uid\")\n", "info_df = info_df[info_df[\"uid\"].isin(set(tmp_map.index))].reset_index(drop=True)\n", "info_df[\"rss_url\"] = info_df[\"uid\"].map(tmp_map[\"url\"])\n", "info_df[\"link\"] = info_df[\"uid\"].map(tmp_map[\"link\"])\n", "info_df[\"language\"] = info_df[\"uid\"].map(tmp_map[\"language\"])\n", "info_df[\"n_episodes\"] = info_df[\"uid\"].map(tmp_map[\"episode_count\"])\n", "info_df[\"newest_episode_pubdate\"] = info_df[\"uid\"].map(tmp_map[\"newest_item_pubdate\"])\n", "info_df[\"oldest_episode_pubdate\"] = info_df[\"uid\"].map(tmp_map[\"oldest_item_pubdate\"])\n", "episode_links = []\n", "for uid in info_df[\"uid\"].values:\n", " rss_filepath = os.path.join(PODCAST_DATA_DIR, f\"bulk_rss_feeds/raw_feeds/{uid}.feed\")\n", " rss_info = load_rss_feed(rss_filepath)\n", " episode_links.append([e[\"audio_url\"] for e in rss_info[\"episodes\"][:3]])\n", "info_df[\"episode_url_0\"] = [e[0] if len(e) >= 1 else None for e in episode_links]\n", "info_df[\"episode_url_1\"] = [e[1] if len(e) >= 2 else None for e in episode_links]\n", "info_df[\"episode_url_2\"] = [e[2] if len(e) >= 3 else None for e in episode_links]\n", "print(info_df.shape[0], \"good spanglish candidates found.\")" ] }, { "cell_type": "code", "execution_count": 10, "id": "8c2aa490", "metadata": {}, "outputs": [], "source": [ "# spanglish:\n", "# 0, 10, 11, 12, 13" ] }, { "cell_type": "code", "execution_count": 47, "id": "08d2db40", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Pensando Out Loud\n", "Bienvenidos a Pensando Out Loud, un podcast para hablar sobre esas unpopular opinions que muchos no se atreven a compartir en voz alta. Porque creemos que aún cuando puede ser difícil, there is POWER IN SHARING OUR STORIES, en ser vulnerables y en hablar sin tabúes de lo bueno, lo malo, y lo no tan perfecto de ser humano. Acompáñennos en este Spanglish journey a aprender, crecer y pensar out loud, JUNTOS!\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "row = info_df.iloc[17]\n", "print(row[\"title\"])\n", "print(row[\"summary\"])\n", "play_audio(row[\"episode_url_0\"])" ] }, { "cell_type": "code", "execution_count": 74, "id": "93b3de89", "metadata": {}, "outputs": [], "source": [ "info_df.to_csv(\"tmp/batch_1.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "45a42f6a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7641ddd9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "82d7950f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "4219f48f", "metadata": {}, "source": [ "## Annotate" ] }, { "cell_type": "code", "execution_count": 75, "id": "35f83db2", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "uid 19130554-3955-5aea-a964-41103f863f4a\n", "title Transcending La Familia\n", "summary We are a virtual vocational academy that focus...\n", "tags business;non-profit\n", "author Learning Idiom\n", "author_email LearningIdiomar@gmail.com\n", "rss_url https://anchor.fm/s/62762d40/podcast/rss\n", "link https://linktr.ee/Learningidiom\n", "language es\n", "n_episodes 14\n", "newest_episode_pubdate 2021-10-26 18:32:32\n", "oldest_episode_pubdate 2021-07-02 09:48:55\n", "episode_url_0 https://d3ctxlq1ktw2nl.cloudfront.net/staging/...\n", "episode_url_1 https://d3ctxlq1ktw2nl.cloudfront.net/staging/...\n", "episode_url_2 https://d3ctxlq1ktw2nl.cloudfront.net/staging/...\n", "Name: 0, dtype: object" ] }, "execution_count": 75, "metadata": {}, "output_type": "execute_result" } ], "source": [ "info_df = pd.read_csv(\"tmp/batch_1.csv\")\n", "info_df.iloc[0]" ] }, { "cell_type": "code", "execution_count": 76, "id": "8c7aba10", "metadata": {}, "outputs": [], "source": [ "import json\n", "from suno_utils.pipeline.labeler import SunoAPIClient\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " SUNO_API_TOKEN = json.load(f)[\"suno_api_token\"]\n", "\n", "suno_client = SunoAPIClient(token=SUNO_API_TOKEN) " ] }, { "cell_type": "code", "execution_count": 98, "id": "1565323b", "metadata": {}, "outputs": [], "source": [ "def get_task_metadata(uid, audio_url_0, audio_url_1, audio_url_2, language, title, description, url, n_episodes):\n", " return {\n", " \"blocks\": [\n", " {\n", " \"type\": \"audio\",\n", " \"label\": \"Audio Sample 1:\",\n", " \"data\": {\"value\": None if pd.isnull(audio_url_0) else audio_url_0},\n", " }, \n", " {\n", " \"type\": \"audio\",\n", " \"label\": \"Audio Sample 2:\",\n", " \"data\": {\"value\": None if pd.isnull(audio_url_1) else audio_url_1},\n", " }, \n", " {\n", " \"type\": \"audio\",\n", " \"label\": \"Audio Sample 3:\",\n", " \"data\": {\"value\": None if pd.isnull(audio_url_2) else audio_url_2},\n", " }, \n", " {\n", " \"type\": \"markdown\",\n", " \"data\": {\n", " \"value\": (\n", " f\"(id:{uid}, lang:{language})\\n\\n - {title}\\n\\n{description}\\n\\n{url}\\n\\n\"\n", " f\"Total Episodes: {n_episodes}\"\n", " )\n", " },\n", " }, \n", " {\n", " \"type\": \"radio\",\n", " \"label\": \"The podcast contains\",\n", " \"data\": {\"key\": \"contentType\"},\n", " \"props\": {\n", " \"options\": [\n", " {\"value\": v} \n", " for v in [\n", " \"English + Spanish (good mix)\",\n", " \"English + Spanish (one-sided mix)\",\n", " \"English + Spanish (but also en-us, other language or music)\",\n", " \"Other\",\n", " ]\n", " ],\n", " \"direction\": \"column\",\n", " },\n", " },\n", " ],\n", " \"podcast_id\": uid,\n", " }\n", "\n", "work_items = [\n", " {\n", " \"metadata\": get_task_metadata(\n", " row[\"uid\"], \n", " row[\"episode_url_0\"],\n", " row[\"episode_url_1\"],\n", " row[\"episode_url_2\"],\n", " row[\"language\"], \n", " row[\"title\"], \n", " row[\"summary\"], \n", " row[\"link\"], \n", " row[\"n_episodes\"],\n", " ),\n", " } \n", " for _, row in info_df.iterrows()\n", "]" ] }, { "cell_type": "code", "execution_count": 99, "id": "72f72422", "metadata": { "scrolled": true }, "outputs": [], "source": [ "# task_name = \"Spanglish Podcasts (Test)\"\n", "task_name = \"Spanglish Podcasts 2\"\n", "work_item_set = suno_client.create_work_item_set(name=task_name)\n", "remote_work_items = suno_client.create_work_items(work_items, work_item_set_id=work_item_set.id)\n", "with open(\"tmp/{}.json\".format(task_name.lower().replace(\" \", \"_\")), \"w\") as f:\n", " json.dump(remote_work_items, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "9838cae1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "11812c49", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "750c8bb8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fae2e406", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "d8587028", "metadata": {}, "source": [ "## Assemble output" ] }, { "cell_type": "code", "execution_count": null, "id": "9702d9ab", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "outreach_df = pd.read_csv(\"tmp/en_ph_list.csv\")\n", "print(outreach_df.shape[0], \"podcasts\")\n", "outreach_df.head(2)" ] }, { "cell_type": "code", "execution_count": 2, "id": "41164bc2", "metadata": {}, "outputs": [], "source": [ "import json\n", "from suno_utils.pipeline.labeler import SunoAPIClient\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " SUNO_API_TOKEN = json.load(f)[\"suno_api_token\"]\n", "\n", "suno_client = SunoAPIClient(token=SUNO_API_TOKEN) " ] }, { "cell_type": "code", "execution_count": 3, "id": "8f215a92", "metadata": {}, "outputs": [], "source": [ "from suno_utils.pipeline.labeler import WorkItemSet\n", "# labeled_results = WorkItemSet(\n", "# id=\"b42eec85-253f-4ed8-b4b9-4a9072eab232\", name=\"PHL Podcast Test\", client=suno_client\n", "# ).get_results()\n", "# labeled_results = WorkItemSet(\n", "# id=\"226ab584-ec50-4e45-9589-4757f9c935a1\", name=\"PHL Podcast gmail\", client=suno_client\n", "# ).get_results()\n", "labeled_results = WorkItemSet(\n", " id=\"80ce382e-c25f-4683-82ad-62bc90ba1aee\", name=\"PHL Podcast gmail 2\", client=suno_client\n", ").get_results()" ] }, { "cell_type": "code", "execution_count": null, "id": "17d91544", "metadata": {}, "outputs": [], "source": [ "usable_podcast_id = []\n", "for work_item in labeled_results[\"workItems\"]:\n", " podcast_id = work_item[\"metadata\"][\"podcast_id\"]\n", " for annotation in work_item[\"assignments\"]:\n", " if annotation[\"data\"].get(\"contentType\") == \"English (Filipino Accent)\":\n", " usable_podcast_id.append(podcast_id)\n", "print(len(usable_podcast_id), \"relevant podcasts found\")" ] }, { "cell_type": "code", "execution_count": null, "id": "002017dd", "metadata": {}, "outputs": [], "source": [ "outreach_df = outreach_df[outreach_df[\"id\"].isin(usable_podcast_id)]\n", "outreach_df = outreach_df.reset_index(drop=True)\n", "print(outreach_df.shape[0], \"podcasts found\")\n", "outreach_df.head(2)" ] }, { "cell_type": "code", "execution_count": 8, "id": "4f82ee88", "metadata": {}, "outputs": [], "source": [ "outreach_df.to_csv(\"outreach_en_phl.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "61d1e571", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6d66b26b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "2824386e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "39cbbb96", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3e1bb53f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5e86044c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f39c48f9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f9860672", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "44a424d7", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 17, "id": "cca113ed", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'uid': '43298fe6-d1ac-5ae6-879b-e0000d2e0dc0',\n", " 'language': 'en-us',\n", " 'title': \"ikonoklast's Show\",\n", " 'subtitle': None,\n", " 'summary': 'Thriving Thursday',\n", " 'description': 'Thriving Thursday',\n", " 'url': 'https://castbox.fm/ch/2137884',\n", " 'url_main': 'http://rss.castbox.fm/everest/9da262d1e0024f16aed43a60f7231249.xml',\n", " 'author': 'ikonoklast',\n", " 'author_email': 'marguz814@gmail.com',\n", " 'rights': '',\n", " 'published': None,\n", " 'updated': None,\n", " 'tags': 'business;comedy;society & culture',\n", " 'episodes': [{'uid': '01ad0019-3837-5126-984c-e4f62c5380b6',\n", " 'audio_url': 'https://s3.castbox.fm/f0/7a/25/9f87c044d6ac257b8d00ce5878.mp3',\n", " 'title': 'Intro',\n", " 'subtitle': None,\n", " 'summary': 'Listen to my newest episode and discover more great content from my show!',\n", " 'url': 'album-9da262d1e0024f16aed43a60f7231249-fc4560d220bd4ccab21ed28cfecb4611',\n", " 'author': '',\n", " 'duration_s': 83,\n", " 'published': '5/23/2019'}]}" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "fp = os.path.join(PODCAST_DATA_DIR, \"bulk_rss_feeds/raw_feeds/43298fe6-d1ac-5ae6-879b-e0000d2e0dc0.feed\")\n", "feed_info = load_rss_feed(fp)\n", "feed_info" ] }, { "cell_type": "code", "execution_count": null, "id": "01076114", "metadata": {}, "outputs": [], "source": [ "# query_ptn = r\"\\b[Ss]panglish\\b\"\n", "# raw_feed_dir = os.path.join(PODCAST_DATA_DIR, \"bulk_rss_feeds/raw_feeds\")\n", "# found_uids = find_in_raw_feeds(query_ptn, raw_feed_dir)\n", "# print(len(found_uids), \"found.\")" ] }, { "cell_type": "code", "execution_count": null, "id": "3ff152db", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1b013374", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "701d6cab", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d4dbedbd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "327b6044", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5de9cf05", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9783a8bb", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }