{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "d62a02e5", "metadata": {}, "outputs": [], "source": [ "from bs4 import BeautifulSoup\n", "import feedparser\n", "import re\n", "import tqdm\n", "import os\n", "import pandas as pd\n", "import sqlite3\n", "import time\n", "import random\n", "import requests\n", "import multiprocessing\n", "import numpy as np\n", "import json\n", "import urllib\n", "\n", "from suno_utils.utils.text import normalize_whitespace\n", "from suno_utils.utils.notebook import Audio" ] }, { "cell_type": "code", "execution_count": 2, "id": "a1020c00", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "4037236 podcasts\n" ] } ], "source": [ "con = sqlite3.connect(\"/mnt/data-ssd-1/data/podcasts/podcastindex_feeds.db\")\n", "cur = con.cursor()\n", "main_df = pd.read_sql(\"SELECT * FROM podcasts;\", con)\n", "con.close()\n", "main_df[\"language\"] = main_df[\"language\"].str.lower().apply(normalize_whitespace)\n", "main_df[\"language\"] = main_df[\"language\"].str.replace(\" \", \"-\").str.replace(\"_\", \"-\").replace(\"\", np.nan)\n", "print(main_df.shape[0], \"podcasts\")" ] }, { "cell_type": "code", "execution_count": null, "id": "a062725d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b4a9a1cf", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "f2776857", "metadata": {}, "source": [ "## Get cc episodes from raw rss feeds" ] }, { "cell_type": "code", "execution_count": 2, "id": "0e538d34", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 3802061/3802061 [39:26<00:00, 1606.55it/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ "17259 found\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "helper_df = main_df.set_index(\"id\")\n", "\n", "# TODO: multicore this\n", "license_types = [\n", " \"creativecommons.org/publicdomain/zero/1.0\",\n", " \"creativecommons.org/publicdomain/mark/1.0\",\n", " \"creativecommons.org/licenses/publicdomain\",\n", " \"creativecommons.org/licenses/cc0/3.0\",\n", " \"creativecommons.org/licenses/by/4.0\",\n", " \"creativecommons.org/licenses/by/3.0\",\n", " \"creativecommons.org/licenses/by/2.5\",\n", " \"creativecommons.org/licenses/by/2.0\",\n", " \"creativecommons.org/licenses/by/1.0\",\n", "]\n", "license_ptn = re.compile(\n", " r\"|\".join([re.escape(s.lower()) for s in sorted(license_types, key=len, reverse=True)]), \n", " flags=re.IGNORECASE\n", ")\n", "\n", "RSS_DATA_DIR = \"/mnt/data-ssd-1/data/podcasts/raw_rss_feeds/\"\n", "\n", "cc_fns = []\n", "for fn in tqdm.tqdm(os.listdir(RSS_DATA_DIR)):\n", " with open(RSS_DATA_DIR + fn, \"rb\") as f:\n", " xml_str = f.read()\n", " string_rep = str(xml_str).lower()\n", " # check link license\n", " if not license_ptn.search(string_rep):\n", " continue\n", " cc_fns.append(fn)\n", "print(len(cc_fns), \"found\")" ] }, { "cell_type": "code", "execution_count": 99, "id": "3ff82d08", "metadata": {}, "outputs": [], "source": [ "def _parse_license_info(podcast_license_1, podcast_license_2, episode_license_1, episode_license_2):\n", " podcast_license = None\n", " episode_license = None\n", " if license_ptn.search(podcast_license_1):\n", " podcast_license = podcast_license_1\n", " if license_ptn.search(podcast_license_2):\n", " podcast_license = podcast_license_2\n", " if license_ptn.search(episode_license_1):\n", " episode_license = episode_license_1\n", " if license_ptn.search(episode_license_2):\n", " episode_license = episode_license_2\n", " return podcast_license, episode_license\n", "\n", "def _get_cc_episodes(fn):\n", " with open(RSS_DATA_DIR + fn, \"rb\") as f:\n", " xml_str = f.read()\n", " \n", " feed = feedparser.parse(xml_str)\n", " \n", " podcast_id = int(fn.split(\".\")[0])\n", " row = helper_df.loc[podcast_id]\n", " \n", " # check global license\n", " podcast_license_1 = feed[\"feed\"].get(\"rights\", \"\")\n", " tmp = \" ; \".join([e.get(\"href\", \"\") for e in feed[\"feed\"].get(\"links\", []) if e[\"rel\"] == \"license\"])\n", " podcast_license_2 = tmp if len(tmp) > 0 else \"\"\n", "\n", " # check license per episode\n", " tmp_data = []\n", " for episode_info in feed[\"entries\"]:\n", " episode_license_1 = episode_info.get(\"blip_license\", \"\")\n", " tmp = \" ; \".join([e.get(\"href\", \"\") for e in episode_info.get(\"links\", []) if e[\"rel\"] == \"license\"])\n", " episode_license_2 = tmp if len(tmp) > 0 else \"\"\n", " podcast_license, episode_license = _parse_license_info(\n", " podcast_license_1, podcast_license_2, episode_license_1, episode_license_2\n", " )\n", " if podcast_license is not None or episode_license is not None:\n", " # video (vodcasts) will get skipped here\n", " audio_links = [\n", " e[\"href\"] for e in episode_info.get(\"links\", []) \n", " if \"audio\" in e.get(\"type\", \"\") and \"href\" in e\n", " ]\n", " if len(audio_links) == 0:\n", " continue\n", " audio_link = audio_links[0]\n", " url_components = [c for c in re.split(r\"(https?\\:\\/\\/)\", audio_link) if len(c) > 0]\n", " episode_url = \"\".join(url_components[-2:])\n", " tmp_data.append({\n", " \"podcast_id\": podcast_id, \n", " \"podcast_license\": podcast_license,\n", " \"episode_license\": episode_license,\n", " \"episode_url\": episode_url,\n", " \"episode_info\": json.dumps(episode_info, default=str),\n", " })\n", " return tmp_data" ] }, { "cell_type": "code", "execution_count": 97, "id": "472a45e4", "metadata": {}, "outputs": [], "source": [ "# data = []\n", "# for fn in tqdm.tqdm(cc_fns[:10]): \n", "# data.extend(_get_cc_episodes(fn))" ] }, { "cell_type": "code", "execution_count": 100, "id": "26e4533c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2514 CC podcasts\n", "36937 CC episodes\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
podcast_idpodcast_licenseepisode_licenseepisode_urlepisode_info
04635113http://creativecommons.org/licenses/by/2.0/Nonehttps://faringdonradio.jellycast.com/files/aud...{\"title\": \"Faringdon Local - Episode 13 - 28th...
14635783http://creativecommons.org/licenses/by/2.0/Nonehttps://faringdonradio.jellycast.com/files/aud...{\"title\": \"gurney 1\", \"title_detail\": {\"type\":...
\n", "
" ], "text/plain": [ " podcast_id podcast_license episode_license \\\n", "0 4635113 http://creativecommons.org/licenses/by/2.0/ None \n", "1 4635783 http://creativecommons.org/licenses/by/2.0/ None \n", "\n", " episode_url \\\n", "0 https://faringdonradio.jellycast.com/files/aud... \n", "1 https://faringdonradio.jellycast.com/files/aud... \n", "\n", " episode_info \n", "0 {\"title\": \"Faringdon Local - Episode 13 - 28th... \n", "1 {\"title\": \"gurney 1\", \"title_detail\": {\"type\":... " ] }, "execution_count": 100, "metadata": {}, "output_type": "execute_result" } ], "source": [ "p = multiprocessing.Pool(20)\n", "out = p.map(_get_cc_episodes, cc_fns, chunksize=100)\n", "data = []\n", "for e in out:\n", " data.extend(e)\n", "p.close()\n", "p.join()\n", "\n", "license_df = pd.DataFrame(data)\n", "print(license_df[\"podcast_id\"].nunique(), \"CC podcasts\")\n", "print(license_df.shape[0], \"CC episodes\")\n", "# 2514 CC podcasts\n", "# 36937 CC episodes\n", "license_df.head(2)" ] }, { "cell_type": "code", "execution_count": 101, "id": "ae4153f6", "metadata": {}, "outputs": [], "source": [ "license_df.to_csv(\"/mnt/data-ssd-1/data/podcasts/cc_data/episode_license_raw.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "16e5e9f7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "fc699e8e", "metadata": {}, "source": [ "### Take only safe license" ] }, { "cell_type": "code", "execution_count": 102, "id": "b16ed75a", "metadata": {}, "outputs": [], "source": [ "license_df = pd.read_csv(\"/mnt/data-ssd-1/data/podcasts/cc_data/episode_license_raw.csv\")" ] }, { "cell_type": "code", "execution_count": 103, "id": "d22b4724", "metadata": {}, "outputs": [], "source": [ "# check most common types of license and label 'safe' ones\n", "\n", "verified_podcast_licenses = set([\n", " \"http://creativecommons.org/licenses/by/2.0/\",\n", " \"http://creativecommons.org/licenses/by/3.0/\",\n", " \"© https://creativecommons.org/licenses/by/4.0/\",\n", " \"http://creativecommons.org/licenses/by/2.5/\",\n", " \"This work is licensed under the Creative Commons Attribution 4.0 International License. To view a copy of this license, visit http://creativecommons.org/licenses/by/4.0/ or send a letter to Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.\",\n", " \"https://creativecommons.org/licenses/by/4.0/\",\n", " \"Creative Commons Attribution LIcense: copy and distribute this work, but don't charge for it and respect any other overriding rights. See the link for details.\",\n", " \"Attribution 2.0 Generic (CC BY 2.0) - http://creativecommons.org/licenses/by/2.0/\",\n", " \"https://creativecommons.org/licenses/by/4.0/legalcode\",\n", " \"CC BY 3.0 DE https://creativecommons.org/licenses/by/3.0/de/\",\n", " \"\\\"Blown Away\\\" Kevin MacLeod (incompetech.com) Licensed under Creative Commons: By Attribution 3.0 http://creativecommons.org/licenses/by/3.0/\",\n", " \"Copyright 2006 Acan Media, Inc. Licensed to the public under http://creativecommons.org/licenses/by/2.0/\",\n", " \"http://creativecommons.org/licenses/by/3.0/es/\",\n", " \"https://creativecommons.org/licenses/by/4.0\",\n", " \"(c) greenjobs GmbH - veröffentlicht unter der freien Lizenz Creative Commons Namensnennung 3.0 Deutschland (http://creativecommons.org/licenses/by/3.0/de/)\",\n", " \"cc-by (https://creativecommons.org/licenses/by/3.0/de/)\",\n", " \"Hockey masks, High Schools and Popcorn - www.hhp-podcast.com | http://creativecommons.org/licenses/by/2.0/\",\n", " \"RetroFuture Clean Kevin MacLeod (incompetech.com) Licensed under Creative Commons: By Attribution 3.0 License http://creativecommons.org/licenses/by/3.0/\",\n", " \"http://creativecommons.org/licenses/by/3.0/us/\",\n", " \"http://creativecommons.org/licenses/by/4.0/\",\n", " \"Attribution 3.0 Unported http://creativecommons.org/licenses/by/3.0/\",\n", " \"https://creativecommons.org/licenses/by/3.0/au/deed.en\",\n", " \"\\\"Creative
Das ist ja wohl eine Unverschämtheit von Elisa Roth ist lizenziert unter einer Creative Commons Namensnennung 4.0 International Lizenz.\",\n", " \"(c) 1&1 IONOS SE 2021. Veröffentlicht unter CC by 4.0\",\n", " \"http://creativecommons.org/licenses/by/2.0/The Portcast 2014\",\n", " \"All material; licensed under Creative Commons - http://creativecommons.org/licenses/by/2.0/\",\n", " \"\\\"Go Cart\\\" Kevin MacLeod (incompetech.com) Licensed under Creative Commons: By Attribution 3.0 http://creativecommons.org/licenses/by/3.0/\",\n", " \"Copyright © Podcast Papo Livre. O conteúdo do podcast está disponível sob a licença Creative Commons Atribuição (CC-BY). A trilha sonora de cada episódio pode estar sob uma licença diferente. As imagens estão licenciadas sob diferentes licenças, veja a listagem para mais informações.\",\n", " \"licence Creative Commons CC BY (Attribution 4.0 International) https://creativecommons.org/licenses/by/4.0/deed.fr\",\n", " \"Creative Commons Attribution 3.0 Unported License http://creativecommons.org/licenses/by/3.0/ 2019\",\n", " \"© http://creativecommons.org/licenses/by/2.0/\",\n", " \"http://creativecommons.org/licenses/by/2.0/wearesofresh\",\n", " \"This work is licensed under a Creative Commons Attribution 4.0 International License. https://creativecommons.org/licenses/by/4.0/\",\n", " \"http://creativecommons.org/licenses/by/2.0/PeterAugerPhilBeresfordMarkFinch\",\n", " \"CC Attribution 4.0 Intl. Public License https://creativecommons.org/licenses/by/4.0/legalcode\",\n", " \"http://www.creativecommons.org/licenses/publicdomain\",\n", " \"https://creativecommons.org/licenses/by/3.0/de/\",\n", " \"licensed under Creative Commons: By Attribution 3.0. http://creativecommons.org/licenses/by/3.0/\",\n", " \"\\\"Creative
KI-Board Podcast von Andreas Klug ist lizenziert unter einer Creative Commons Namensnennung 4.0 International Lizenz.
Über diese Lizenz hinausgehende Erlaubnisse können Sie unter www.andreasklug.com erhalten.\",\n", " \"http://creativecommons.org/licenses/by/2.0/Quiff-Cast 2014\",\n", " \"Kicking it podcast © 2021 by Oluwaseun Toyobo is licensed under Attribution 4.0 International. To view a copy of this license, visit http://creativecommons.org/licenses/by/4.0/\",\n", " \"http://creativecommons.org/licenses/by/2.0/TLD 2012\",\n", " \"http://creativecommons.org/licenses/by/2.0/Oct 8/08\",\n", " \"http://creativecommons.org/licenses/by/2.0/chipsforthepoorechoesfromtheskyLondon07042008\",\n", " \"Richard Newman and Seth Mason http://creativecommons.org/licenses/by/2.0/\",\n", " \"https://creativecommons.org/licenses/by/3.0/au/\",\n", " \"CC BY 3.0 (http://creativecommons.org/licenses/by/3.0/at/)\"\n", "])\n", "\n", "# for idx, n in license_df[\"podcast_license\"].value_counts().iteritems():\n", "# if n >= 10 and \"music\" not in idx.lower():\n", "# print(\"\\\"\" + idx.replace(\"\\\"\", '\\\\\"') + \"\\\",\")\n", "\n", "verified_episode_licenses = set([\n", " \"http://creativecommons.org/publicdomain/mark/1.0/\",\n", " \"http://creativecommons.org/licenses/by/2.0/\",\n", " \"http://creativecommons.org/licenses/by/3.0/\",\n", " \"http://creativecommons.org/publicdomain/zero/1.0/\",\n", " \"http://creativecommons.org/licenses/by/3.0/deed.de\",\n", " \"https://creativecommons.org/licenses/by/4.0/\",\n", " \"http://creativecommons.org/licenses/publicdomain/\",\n", " \"http://creativecommons.org/licenses/by/4.0/\",\n", "])\n", "\n", "# for idx, n in license_df[\"episode_license\"].value_counts().iteritems():\n", "# if n >= 10 and \"music\" not in idx.lower():\n", "# print(\"\\\"\" + idx.replace(\"\\\"\", '\\\\\"') + \"\\\",\")" ] }, { "cell_type": "code", "execution_count": 105, "id": "ccf48b00", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2455 CC podcasts\n", "36216 CC episodes\n" ] } ], "source": [ "license_df = license_df[\n", " license_df[\"podcast_license\"].isin(verified_podcast_licenses) |\n", " license_df[\"episode_license\"].isin(verified_episode_licenses)\n", "]\n", "print(license_df[\"podcast_id\"].nunique(), \"CC podcasts\")\n", "print(license_df.shape[0], \"CC episodes\")\n", "# 2455 CC podcasts\n", "# 36216 CC episodes" ] }, { "cell_type": "code", "execution_count": null, "id": "0200f984", "metadata": {}, "outputs": [], "source": [ "import uuid\n", "license_df[\"uuid\"] = [str(uuid.uuid4()) for _ in range(license_df.shape[0])]" ] }, { "cell_type": "code", "execution_count": 16, "id": "8d21e68b", "metadata": {}, "outputs": [], "source": [ "license_df.to_csv(\"/mnt/data-ssd-1/data/podcasts/cc_data/episode_license.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "47add210", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5726f84d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "873cda8d", "metadata": {}, "source": [ "## (optional) listen to some episodes" ] }, { "cell_type": "code", "execution_count": 108, "id": "e309e5fb", "metadata": {}, "outputs": [], "source": [ "from IPython.display import HTML, display\n", "\n", "license_df = pd.read_csv(\"/mnt/data-ssd-1/data/podcasts/cc_data/episode_license.csv\")" ] }, { "cell_type": "code", "execution_count": 152, "id": "41563cdf", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "https://star-one.jellycast.com/files/audio/simon-gray-the-golden-temple.mp3\n" ] }, { "data": { "text/html": [ "the golden temple" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "row = license_df.sample(1).iloc[0]\n", "print(row[\"episode_url\"])\n", "json.loads(row[\"episode_info\"]).get(\"title\")\n", "display(HTML(json.loads(row[\"episode_info\"]).get(\"summary\")))" ] }, { "cell_type": "code", "execution_count": 197, "id": "8b7fae70", "metadata": {}, "outputs": [], "source": [ "# data = []\n", "# for idx, row in license_df.iterrows():\n", "# if \"published_parsed\" in row[\"episode_info\"]:\n", "# data.append(idx)\n", "# random.seed(6006)\n", "# random.shuffle(data)\n", "# len(data)" ] }, { "cell_type": "code", "execution_count": null, "id": "83ec5a30", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1c262875", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "2099d850", "metadata": {}, "source": [ "## Filter to en corpus and add metadata" ] }, { "cell_type": "code", "execution_count": 3, "id": "23a6ba26", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.podcasts import get_description, get_title, load_rss_feed, load_rss_text\n", "from suno_utils.web.podcasts import get_url_filesize_bytes" ] }, { "cell_type": "code", "execution_count": 24, "id": "9f8084b5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "28843 episodes remaining\n" ] } ], "source": [ "episodes_df = pd.read_csv(\"/mnt/data-ssd-1/data/podcasts/cc_data/episode_license.csv\")\n", "\n", "# filter to english and known host\n", "KNOWN_CC_HOSTS = set([\"jellycast.com\", \"feedburner.com\"])\n", "df = main_df.copy()\n", "df = df[df[\"language\"].str[:2] == \"en\"]\n", "df = df[df[\"host\"].isin(KNOWN_CC_HOSTS)]\n", "allowed_podcast_ids = set(df[\"id\"])\n", "episodes_df = episodes_df[episodes_df[\"podcast_id\"].isin(allowed_podcast_ids)]\n", "episodes_df = episodes_df.drop([\"podcast_license\", \"episode_license\"], axis=1)\n", "episodes_df = episodes_df.rename(columns={\"episode_url\": \"audio_url\", \"episode_info\": \"meta_json\"})\n", "episodes_df = episodes_df.reset_index(drop=True)\n", "print(episodes_df.shape[0], \"episodes remaining\")" ] }, { "cell_type": "code", "execution_count": 37, "id": "4a7d44e5", "metadata": {}, "outputs": [], "source": [ "# check episode availability and size\n", "p = multiprocessing.Pool(20)\n", "filesizes = p.map(get_url_filesize_bytes, episode_license_df[\"audio_url\"].values, chunksize=10)\n", "p.close()\n", "p.join()" ] }, { "cell_type": "code", "execution_count": null, "id": "0226922b", "metadata": {}, "outputs": [], "source": [ "# # TODO: get duration for remote file (and filetype?)\n", "# url = \"https://cornucopia.jellycast.com/files/audio/Radio%20Fore-Witches.mp3\"\n", "\n", "# headers = {\"Range\": \"bytes=0-1000\"}\n", "# r = requests.get(url, headers=headers)\n", "# with open(\"bla.mp3\", \"wb\") as f:\n", "# f.write(r.content)\n", "# header_info = tinytag.TinyTag.get(\"bla.mp3\")\n", "# total_bytes = int(r.headers[\"Content-Range\"].split(\"/\")[-1])\n", "# duration_s = round(total_bytes * 8 / header_info.as_dict()[\"bitrate\"] / 1_000, 1)" ] }, { "cell_type": "code", "execution_count": 47, "id": "05dde96a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1062 podcasts remaining\n", "7337 episodes remaining\n" ] } ], "source": [ "episodes_df[\"filesize_bytes\"] = filesizes\n", "episodes_df = episodes_df[episodes_df[\"filesize_bytes\"] >= 5_000_000]\n", "episodes_df[\"filesize_bytes\"] = episodes_df[\"filesize_bytes\"].astype(int)\n", "episodes_df = episodes_df.reset_index(drop=True)\n", "print(episodes_df[\"podcast_id\"].nunique(), \"podcasts remaining\")\n", "print(episodes_df.shape[0], \"episodes remaining\")" ] }, { "cell_type": "code", "execution_count": 169, "id": "add1efe8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1047 podcasts remaining\n", "7132 episodes remaining\n" ] } ], "source": [ "# add episode info\n", "title_list = []\n", "summary_list = []\n", "tags_list = []\n", "date_list = []\n", "for _, row in episodes_df.iterrows():\n", " episode_info = json.loads(row[\"meta_json\"])\n", " title_list.append(episode_info.get(\"title\", \"\"))\n", " summary_list.append(episode_info.get(\"summary\", \"\"))\n", " tags = list(set([t[\"term\"].strip().lower() for t in episode_info.get(\"tags\", [])]))\n", " if len(tags) > 20:\n", " tags = []\n", " tags_list.append(\";\".join(tags))\n", " published_date = None\n", " date_ints = episode_info.get(\"published_parsed\", [])\n", " if isinstance(date_ints, list) and len(date_ints) >= 3:\n", " published_date = \"{}/{}/{}\".format(date_ints[1], date_ints[2], date_ints[0])\n", " date_list.append(published_date) \n", "episodes_df[\"title\"] = title_list\n", "episodes_df[\"summary\"] = summary_list\n", "episodes_df[\"tags\"] = tags_list\n", "episodes_df[\"published_date\"] = date_list\n", "episodes_df = episodes_df.drop([\"meta_json\"], axis=1)\n", "episodes_df = episodes_df[~episodes_df[\"tags\"].str.contains(r\"\\bmusic\\b\", regex=True)]\n", "episodes_df = episodes_df.dropna(subset=[\"published_date\"])\n", "episodes_df[\"published_date\"] = pd.to_datetime(episodes_df[\"published_date\"])\n", "episodes_df = episodes_df.sort_values(by=[\"podcast_id\", \"published_date\"], ascending=[True, False])\n", "episodes_df = episodes_df.reset_index(drop=True)\n", "print(episodes_df[\"podcast_id\"].nunique(), \"podcasts remaining\")\n", "print(episodes_df.shape[0], \"episodes remaining\")" ] }, { "cell_type": "code", "execution_count": 181, "id": "68b1aec3", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
podcast_idaudio_urluuidfilesize_bytestitlesummarytagspublished_date
01739https://eirecana.jellycast.com/files/audio/Epi...b9da5193-ab1f-4c03-94f1-38959badcb07139857145Episode XVI: Simply Good MusicThe latest episode of the Eirecana podcast fea...alt-country;irish;blues;folk;americana2017-08-07
13872https://wileyblackwellhealth.jellycast.com/fil...2784a4d7-b9da-49f6-9581-baeff97551b725226581Neurogastroenterology and Motility - June 2015Discussion of the paper: 'Altered viscerotopic...put-some-keywords-here2015-06-08
\n", "
" ], "text/plain": [ " podcast_id audio_url \\\n", "0 1739 https://eirecana.jellycast.com/files/audio/Epi... \n", "1 3872 https://wileyblackwellhealth.jellycast.com/fil... \n", "\n", " uuid filesize_bytes \\\n", "0 b9da5193-ab1f-4c03-94f1-38959badcb07 139857145 \n", "1 2784a4d7-b9da-49f6-9581-baeff97551b7 25226581 \n", "\n", " title \\\n", "0 Episode XVI: Simply Good Music \n", "1 Neurogastroenterology and Motility - June 2015 \n", "\n", " summary \\\n", "0 The latest episode of the Eirecana podcast fea... \n", "1 Discussion of the paper: 'Altered viscerotopic... \n", "\n", " tags published_date \n", "0 alt-country;irish;blues;folk;americana 2017-08-07 \n", "1 put-some-keywords-here 2015-06-08 " ] }, "execution_count": 181, "metadata": {}, "output_type": "execute_result" } ], "source": [ "episodes_df.head(2)" ] }, { "cell_type": "code", "execution_count": null, "id": "af226718", "metadata": {}, "outputs": [], "source": [ "# TODO: get podcast info that might help (eg 'health' in title)\n", "# remove more epsiodes with eg 'music' in title" ] }, { "cell_type": "code", "execution_count": 69, "id": "122f122d", "metadata": {}, "outputs": [], "source": [ "# # add podcast info like title description url and tags\n", "# RSS_DATA_DIR = \"/mnt/data-ssd-1/data/podcasts/raw_rss_feeds/\"\n", "\n", "# feed_uris = []\n", "# podcast_title_list = []\n", "# podcast_description_list = []\n", "# author_name_list = []\n", "# author_email_list = []\n", "# tags_list = []\n", "# for _, row in tqdm.tqdm(cc_df.iterrows(), total=cc_df.shape[0]):\n", "# feed_uri = \"{}.feed\".format(row[\"podcast_id\"])\n", "# rss_text = load_rss_text(RSS_DATA_DIR + feed_uri)\n", "# podcast_title = get_rss_tag(\"title\", rss_text)\n", "# podcast_description = get_rss_tag(\"description\", rss_text)\n", "# rss_feed = load_rss_feed(RSS_DATA_DIR + feed_uri)\n", "# author_detail = rss_feed[\"feed\"].get(\"publisher_detail\", {})\n", "# author_name = author_detail.get(\"name\")\n", "# author_email = author_detail.get(\"email\")\n", "# tags = [e[\"term\"] for e in json.loads(row[\"episode_info\"]).get(\"tags\", [])]\n", "# if len(tags) > 20:\n", "# tag_str = \"\"\n", "# else:\n", "# tag_str = \";\".join(sorted(tags))\n", "# feed_uris.append(feed_uri)\n", "# podcast_title_list.append(podcast_title)\n", "# podcast_description_list.append(podcast_description)\n", "# author_name_list.append(author_name)\n", "# author_email_list.append(author_email)\n", "# tags_list.append(tag_str)\n", "\n", "# cc_df[\"feed_uri\"] = feed_uris\n", "# cc_df[\"podcast_title\"] = podcast_title_list\n", "# cc_df[\"podcast_description\"] = podcast_description_list\n", "# cc_df[\"author_name\"] = author_name_list\n", "# cc_df[\"author_email\"] = author_email_list\n", "# cc_df[\"tags\"] = tags_list\n", "# cc_df = cc_df[~cc_df[\"podcast_tags\"].str.contains(r\"\\bmusic\\b\", regex=True)]\n", "# print(cc_df.shape[0], \"episodes\")" ] }, { "cell_type": "code", "execution_count": 182, "id": "3c1cfb1c", "metadata": {}, "outputs": [], "source": [ "episodes_df.to_csv(\"/mnt/data-ssd-1/data/podcasts/cc_data/cc_episodes.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": 186, "id": "bf9ea240", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(7132, 8)" ] }, "execution_count": 186, "metadata": {}, "output_type": "execute_result" } ], "source": [ "episodes_df.shape" ] }, { "cell_type": "code", "execution_count": null, "id": "af299402", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3b79cfa9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3963dc75", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "21802764", "metadata": {}, "source": [ "### Investigate tags" ] }, { "cell_type": "code", "execution_count": 135, "id": "05018bc1", "metadata": {}, "outputs": [], "source": [ "# look at tags\n", "all_tags = \";\".join([s for s in episodes_df[\"tags\"].values if len(s) > 0]).split(\";\")\n", "tags_vc = pd.Series(all_tags).value_counts()" ] }, { "cell_type": "code", "execution_count": 141, "id": "591799c4", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "mental health 46\n", "health 13\n", "healthcare 3\n", "mental health north east 3\n", "public health 2\n", "health visitor 1\n", "honest.health 1\n", "atos healthcare 1\n", "dtype: int64" ] }, "execution_count": 141, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tags_vc.loc[[s for s in tags_vc.index if \"health\" in s]]" ] }, { "cell_type": "code", "execution_count": 140, "id": "d630766f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Episode 51: Mental Hellness\n", "http://www.thelofishow.com/podcast/2015/2015-03-29-Episode51-MentalHellness.mp3\n", "\n", "The David Shayler Show [Hector Christie] 10-11-2006\n", "https://marshall.jellycast.com/files/audio/Hector%20Interview%20Part%20One.m4a\n", "\n", "Mentally Sound live (31st May 2019)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2031052019%20full.mp3\n", "\n", "Mentally Sound live MHAW special (may 16th)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20sound%20special%20mhaw%2016th%20may%2016052019%20full.mp3\n", "\n", "Mentally Sound live (10th May 2019)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2010052019%20%281%29%20full.mp3\n", "\n", "Mentally Sound live (3rd may 2019 replacement for 11th april)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2003052019%20%28april%2011th%20replacement%29.mp3\n", "\n", "Mentally Sound live (26th April 2019)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2026042019%20full.mp3\n", "\n", "Mentally Sound live (22nd March 2019)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2022032019%20full.mp3\n", "\n", "Mentally Sound live (8th march 2019)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound08032019%20full.mp3\n", "\n", "Mentally Sound live (22nd feb 2019)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound22022019%20full.mp3\n", "\n", "Mentally Sound live (8th Feb 2019)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2008022019%20full.mp3\n", "\n", "Mentally Sound live (25th Jan 2019)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2025012019%20full.mp3\n", "\n", "Mentally Sound live (11th Jan 2019)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2011012019%20full.mp3\n", "\n", "Mentally Sound 2018 end of year review (28th December)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2028122018%20edited%20full.mp3\n", "\n", "mentally Sound Xmas special live (21st December)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20sound%2021st%203%20hour%20xmas%20special%2021122018%20full.mp3\n", "\n", "mentally Sound live (9th November)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2009112018%20full.mp3\n", "\n", "mentally Sound live 28th September\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2028092018%20full.mp3\n", "\n", "Mentally Sound live 14th september\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2014092018%20full.mp3\n", "\n", "Mentally Sound live 31 August\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2031082018%20full.mp3\n", "\n", "Mentally Sound live 10th August\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2010082018%20full.mp3\n", "\n", "Mentally Sound live 27th July\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2027072018%20full%20two.mp3\n", "\n", "Mentally Sound live 13th July\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2013072018%20full.mp3\n", "\n", "Mentally Sound live 6th july (moved from june 29th)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound06072018%20full.mp3\n", "\n", "Mentally Sound live 8th June\n", "https://mentallysound.jellycast.com/files/audio/8th%20june%20full.mp3\n", "\n", "Mentally Sound live 11th May\n", "https://mentallysound.jellycast.com/files/audio/mentally%20sound11052018%20full.mp3\n", "\n", "Mentally Sound live 27th April\n", "https://mentallysound.jellycast.com/files/audio/Mentally%20Sound%20april%2027th%202018%20full.mp3\n", "\n", "Mentally Sound live 13th April\n", "https://mentallysound.jellycast.com/files/audio/full%20mentally%20Sound13042018.mp3\n", "\n", "Mentally Sound Live (march 9th)\n", "https://mentallysound.jellycast.com/files/audio/march%202018%20MS.mp3\n", "\n", "Mentally Sound Live (9th Feb)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%209th%20Feb%20full.mp3\n", "\n", "Mentally Sound Live (12th jan)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20sound%20jan%202018.mp3\n", "\n", "Mentally Sound Xmas Special live (8th december 2017)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20sound%20xmas%20special%202017.mp3\n", "\n", "Mentally Sound live (10th November)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound%2010112017%20full.mp3\n", "\n", "Mentally Sound live (13th October)\n", "https://mentallysound.jellycast.com/files/audio/Mentally%20Sound%20live%20october%2013th.mp3\n", "\n", "Mentally Sound live (9th September)\n", "https://mentallysound.jellycast.com/files/audio/Mentally%20Sound%20live%20sept%209%2017.mp3\n", "\n", "Best of mentally Sound (cover episode)\n", "https://mentallysound.jellycast.com/files/audio/may%20cover%20best%20of%20mentally%20Sound%21.mp3\n", "\n", "Spice Fm Pilot Episode!\n", "https://mentallysound.jellycast.com/files/audio/mentally%20Sound14042017%20%28edited%29.mp3\n", "\n", "Mentally Sound show 15 (11/5/16)\n", "https://mentallysound.jellycast.com/files/audio/MS%205th%202016%20full.mp3\n", "\n", "Mentally Sound show 12 (12/2/16)\n", "https://mentallysound.jellycast.com/files/audio/show%2012%20full%20show.mp3\n", "\n", "Mentally Sound Show 11 (8/1/16)\n", "https://mentallysound.jellycast.com/files/audio/show%2011%20MS%20full.mp3\n", "\n", "Mentally Sound Show 10 (11/12/15)\n", "https://mentallysound.jellycast.com/files/audio/full%20show%20mentally%20sound.mp3\n", "\n", "Mentally Sound Show 9 (13/11/15)\n", "https://mentallysound.jellycast.com/files/audio/show%209%20MS.mp3\n", "\n", "Mentally Sound Show 8 (9/10/15)\n", "https://mentallysound.jellycast.com/files/audio/mentally%20sound%20show%208.mp3\n", "\n", "Mentally Sound Radio Show #1\n", "https://mentallysound.jellycast.com/files/audio/MHNE_MentallySoundRadioShow_show1.mp3\n", "\n", "Mentally Sound Show 2 10/04/15\n", "https://mentallysound.jellycast.com/files/audio/full%20show%20mentally%20sound%20show%202.mp3\n", "\n", "Mentally Sound show 3 8/5/2015\n", "https://mentallysound.jellycast.com/files/audio/mentally%20sound%20show%203.mp3\n", "\n", "Mentally Sound show 4 12/6/2015\n", "https://mentallysound.jellycast.com/files/audio/show%204%20mentallysound%20%20podcast.mp3\n", "\n", "Mentally Sound Show 5 10/7/15\n", "https://mentallysound.jellycast.com/files/audio/mentally%20sound%20show%205%20full.mp3\n", "\n", "Mentally Sound Show 6 (14/8/15)\n", "https://mentallysound.jellycast.com/files/audio/show%206%20mentally%20sound.mp3\n", "\n", "Ann Lewis: How Nature Photography Can Change Your Life\n", "https://creativeconversations.jellycast.com/files/audio/180409%20ann%20ym%20MP3%20AUD%2002.mp3\n", "\n", "Tony Dillon on Seaside and Wellbeing\n", "https://creativeconversations.jellycast.com/files/audio/tony%20audio%20mp3%20%20podcast%20170908%2B1145.mp3\n", "\n", "Vocal Health Tips in the Winter Weather\n", "https://elocution.jellycast.com/files/audio/Vocal%20health%20tips.m4a\n", "\n", "Brian Davies on Safety Lessons from the Airline Industry\n", "https://sfassociates.jellycast.com/files/audio/Fraser_Brian.mp3\n", "\n", "Dr Mike Davies, National Director of System Redesign of the VA (USA)\n", "https://sfassociates.jellycast.com/files/audio/Fraser_Mike.mp3\n", "\n", "In conversation with... Dr Kate Silvester; expert on the application of Lean Thinking in Healthcare\n", "https://sfassociates.jellycast.com/files/audio/Fraser_kate.mp3\n", "\n", "Die Hard or Live Long\n", "https://owenvsthegenius.files.wordpress.com/2013/02/ovsg-die-hard-or-live-long.mp3\n", "\n" ] } ], "source": [ "search_tags = set([\"mental health\", \"health\", \"healthcare\"])\n", "\n", "for _, row in episodes_df.iterrows():\n", " if len(set([e.strip() for e in row[\"tags\"].split(\";\")]) & search_tags) > 0:\n", " print(row[\"title\"])\n", " print(row[\"audio_url\"])\n", " print()" ] }, { "cell_type": "code", "execution_count": null, "id": "5bdaf71c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "486ae98d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "2d3b19cd", "metadata": {}, "source": [ "## Download episodes" ] }, { "cell_type": "code", "execution_count": null, "id": "420dc7a2", "metadata": {}, "outputs": [], "source": [ "from collections import Counter\n", "from suno_utils.web.harvest import get_file_ext\n", "from suno_utils.utils.conversion import has_valid_header" ] }, { "cell_type": "code", "execution_count": 201, "id": "129d1f02", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "7132\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
podcast_idaudio_urluuidfilesize_bytestitlesummarytagspublished_date
01739https://eirecana.jellycast.com/files/audio/Epi...b9da5193-ab1f-4c03-94f1-38959badcb07139857145Episode XVI: Simply Good MusicThe latest episode of the Eirecana podcast fea...alt-country;irish;blues;folk;americana2017-08-07
13872https://wileyblackwellhealth.jellycast.com/fil...2784a4d7-b9da-49f6-9581-baeff97551b725226581Neurogastroenterology and Motility - June 2015Discussion of the paper: 'Altered viscerotopic...put-some-keywords-here2015-06-08
\n", "
" ], "text/plain": [ " podcast_id audio_url \\\n", "0 1739 https://eirecana.jellycast.com/files/audio/Epi... \n", "1 3872 https://wileyblackwellhealth.jellycast.com/fil... \n", "\n", " uuid filesize_bytes \\\n", "0 b9da5193-ab1f-4c03-94f1-38959badcb07 139857145 \n", "1 2784a4d7-b9da-49f6-9581-baeff97551b7 25226581 \n", "\n", " title \\\n", "0 Episode XVI: Simply Good Music \n", "1 Neurogastroenterology and Motility - June 2015 \n", "\n", " summary \\\n", "0 The latest episode of the Eirecana podcast fea... \n", "1 Discussion of the paper: 'Altered viscerotopic... \n", "\n", " tags published_date \n", "0 alt-country;irish;blues;folk;americana 2017-08-07 \n", "1 put-some-keywords-here 2015-06-08 " ] }, "execution_count": 201, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cc_episodes_df = pd.read_csv(\"/mnt/data-ssd-1/data/podcasts/cc_data/cc_episodes.csv\")\n", "cc_episodes_df[\"published_date\"] = pd.to_datetime(cc_episodes_df[\"published_date\"])\n", "# cc_episodes_df = cc_episodes_df[cc_episodes_df[\"published_date\"].dt.year >= 2010]\n", "# cc_episodes_df = cc_episodes_df.reset_index(drop=True)\n", "print(cc_episodes_df.shape[0])\n", "cc_episodes_df.head(2)" ] }, { "cell_type": "code", "execution_count": 207, "id": "4d271d9f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2618 episodes ready for download\n" ] } ], "source": [ "AUDIO_DATA_DIR = \"/mnt/data-ssd-1/data/podcasts/cc_data/episode_audio/\"\n", "\n", "n = 5 # how many of each podcast, None - unlimited\n", "\n", "episode_counter = Counter()\n", "work_items = []\n", "for idx, row in cc_episodes_df.iterrows():\n", " podcast_id = row[\"podcast_id\"]\n", " if n is not None and episode_counter[podcast_id] >= n:\n", " continue\n", " work_items.append((row[\"uuid\"], row[\"audio_url\"]))\n", " episode_counter[podcast_id] += 1\n", "print(len(work_items), \"episodes ready for download\")" ] }, { "cell_type": "code", "execution_count": 243, "id": "802465b4", "metadata": {}, "outputs": [], "source": [ "def download_file(work_item, courtesy_wait_s=1):\n", " uuid, audio_url = work_item\n", " file_ext = get_file_ext(audio_url)\n", " try:\n", " r = requests.get(audio_url, timeout=2)\n", " if not r.ok:\n", " raise ValueError(\"grabbing failed\")\n", " filename = f\"{uuid}.{file_ext}\"\n", " with open(AUDIO_DATA_DIR + filename, \"wb\") as f:\n", " f.write(r.content)\n", " # verify by getting header metadata\n", " if not has_valid_header(AUDIO_DATA_DIR + filename):\n", " raise ValueError(\"downloaded file corrupt\")\n", " except:\n", " return (uuid, None)\n", " time.sleep(courtesy_wait_s + random.random() * courtesy_wait_s * 0.2)\n", " return (uuid, filename)" ] }, { "cell_type": "code", "execution_count": 242, "id": "7ac598d6", "metadata": {}, "outputs": [], "source": [ "# out = []\n", "# for work_item in tqdm.tqdm(work_items):\n", "# e = download_file(work_item)\n", "# out.append(e)\n", "# break" ] }, { "cell_type": "code", "execution_count": 248, "id": "3b9eb81a", "metadata": {}, "outputs": [], "source": [ "# ~500 per hour with 5 cores\n", "p = multiprocessing.Pool(5)\n", "out = p.map(download_file, work_items, chunksize=5)\n", "p.close()\n", "p.join()" ] }, { "cell_type": "code", "execution_count": 283, "id": "747cec5d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2564 episodes downloaded\n" ] } ], "source": [ "uuid_to_filename = {uuid: filename for uuid, filename in out}\n", "cc_episodes_df[\"filename\"] = cc_episodes_df[\"uuid\"].map(uuid_to_filename)\n", "out_df = cc_episodes_df[~cc_episodes_df[\"filename\"].isnull()]\n", "out_df = out_df[out_df[\"filename\"].str.split(\".\").str[-1].isin(set([\"mp3\", \"m4a\"]))]\n", "out_df = out_df.reset_index(drop=True)\n", "print(out_df.shape[0], \"episodes downloaded\")" ] }, { "cell_type": "code", "execution_count": 12, "id": "ccd3af23", "metadata": {}, "outputs": [], "source": [ "# !ls /mnt/data-ssd-1/data/podcasts/cc_data/episode_audio | wc -l" ] }, { "cell_type": "code", "execution_count": 284, "id": "8967912a", "metadata": {}, "outputs": [], "source": [ "out_df.to_csv(\"/mnt/data-ssd-1/data/podcasts/cc_data/episodes_with_audio.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "66ce7289", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 292, "id": "968332e1", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
podcast_idaudio_urluuidfilesize_bytestitlesummarytagspublished_datefilename
01739https://eirecana.jellycast.com/files/audio/Epi...b9da5193-ab1f-4c03-94f1-38959badcb07139857145Episode XVI: Simply Good MusicThe latest episode of the Eirecana podcast fea...alt-country;irish;blues;folk;americana2017-08-07b9da5193-ab1f-4c03-94f1-38959badcb07.mp3
\n", "
" ], "text/plain": [ " podcast_id audio_url \\\n", "0 1739 https://eirecana.jellycast.com/files/audio/Epi... \n", "\n", " uuid filesize_bytes \\\n", "0 b9da5193-ab1f-4c03-94f1-38959badcb07 139857145 \n", "\n", " title \\\n", "0 Episode XVI: Simply Good Music \n", "\n", " summary \\\n", "0 The latest episode of the Eirecana podcast fea... \n", "\n", " tags published_date \\\n", "0 alt-country;irish;blues;folk;americana 2017-08-07 \n", "\n", " filename \n", "0 b9da5193-ab1f-4c03-94f1-38959badcb07.mp3 " ] }, "execution_count": 292, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df.head(1)" ] }, { "cell_type": "code", "execution_count": 296, "id": "9cc5d225", "metadata": {}, "outputs": [], "source": [ "out_df[\"language\"] = out_df[\"podcast_id\"].map(main_df.set_index(\"id\")[\"language\"])" ] }, { "cell_type": "code", "execution_count": 297, "id": "a75cc4e8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "en-pi 2338\n", "en 131\n", "en-us 95\n", "Name: language, dtype: int64" ] }, "execution_count": 297, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df[\"language\"].value_counts()" ] }, { "cell_type": "code", "execution_count": null, "id": "c37eafd4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "62e49ea5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3085fdca", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "dbb9b168", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": null, "id": "be5e3bae", "metadata": {}, "outputs": [], "source": [ "# TODO filter music (and foreign language?) with model (maybe asr and find speech stretches vs vad)\n", "\n", "# TODO subsets\n", "# episode tags\n", "# low sample frequency\n", "\n", "# TODO: filter advertisements?\n", "\n", "# good CC music example\n", "# https://faringdonradio.jellycast.com/podcast/feed/35\n", "# https://faringdonradio.jellycast.com/files/audio/Faringdon%20Local%20Episode%2013%20-%2028th%20August%202011.mp3\n", "\n", "# all music\n", "# http://traffic.libsyn.com/beardo1/show690.mp3\n", "# http://traffic.libsyn.com/beardo1/show377.mp3" ] }, { "cell_type": "code", "execution_count": null, "id": "094933f6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c61c573b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6ee98b1b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "2b18535a", "metadata": {}, "source": [ "## Count hours" ] }, { "cell_type": "code", "execution_count": null, "id": "fd68f834", "metadata": {}, "outputs": [], "source": [ "# def _parse_ts_as_s(timestamp):\n", "# # TODO: deal with fractional\n", "# if \".\" in timestamp:\n", "# return None\n", "# time_s = 0\n", "# if \",\" in timestamp:\n", "# time_s += float(\"0.\" + timestamp.split(\",\")[-1])\n", "# timestamp = timestamp.split(\",\")[0]\n", "# parts = timestamp.split(\":\")\n", "# if len(parts) == 3:\n", "# h, m, s = timestamp.split(\":\")\n", "# time_s += int(s) + int(m) * 60 + int(h) * 60 * 60\n", "# elif len(parts) == 2:\n", "# m, s = timestamp.split(\":\")\n", "# time_s += int(s) + int(m) * 60\n", "# else:\n", "# return None\n", "# return time_s\n", "\n", "# tmp = []\n", "# for _, row in valid_license_df.iterrows():\n", "# d = json.loads(row[\"episode_info\"])\n", "# duration_s = d.get(\"itunes_duration\")\n", "# if duration_s is None:\n", "# continue\n", "# duration_s = _parse_ts_as_s(duration_s)\n", "# if duration_s is None:\n", "# continue\n", "# tmp.append(duration_s)\n", "# round(sum(tmp) / len(tmp) * valid_license_df.shape[0] / 60 / 60, 1)\n", "# # 993.3" ] }, { "cell_type": "code", "execution_count": null, "id": "5e4c77f8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "95549de7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b2d09803", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "aa47c481", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }