{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "ca39482c", "metadata": {}, "outputs": [], "source": [ "import tqdm\n", "import feedparser\n", "import time\n", "import random\n", "import datetime\n", "import pandas as pd\n", "import sqlite3\n", "import numpy as np\n", "\n", "from suno_utils.utils.text import normalize_whitespace" ] }, { "cell_type": "code", "execution_count": null, "id": "0b027267", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c2aa1c41", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "296fac34", "metadata": {}, "source": [ "## Get anchor subset" ] }, { "cell_type": "code", "execution_count": 143, "id": "243487a6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1858426 total anchor podcasts\n" ] } ], "source": [ "con = sqlite3.connect(\"/mnt/data-ssd-1/data/podcasts/podcastindex_feeds.db\")\n", "cur = con.cursor()\n", "df = pd.read_sql(\"SELECT * FROM podcasts;\", con)\n", "con.close()\n", "\n", "df = df[\n", " (df[\"host\"] == \"anchor.fm\") &\n", " (df[\"generator\"] == \"Anchor Podcasts\")\n", "].reset_index(drop=True).copy()\n", "df[\"language\"] = df[\"language\"].str.lower().apply(normalize_whitespace)\n", "df[\"language\"] = df[\"language\"].str.replace(\" \", \"-\").str.replace(\"_\", \"-\").replace(\"\", np.nan)\n", "print(df.shape[0], \"total anchor podcasts\")" ] }, { "cell_type": "code", "execution_count": 144, "id": "7ed7c5d2", "metadata": {}, "outputs": [], "source": [ "df.to_csv(\"/mnt/data-ssd-1/data/podcasts/anchor_subset.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "6b67f32d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6e0bea24", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7538d3fc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0dac9ade", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "a80f7d3e", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 2, "id": "762e4a33", "metadata": {}, "outputs": [], "source": [ "df = pd.read_csv(\"/mnt/data-ssd-1/data/podcasts/anchor_subset.csv\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "4ca6f3d6", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idurltitlelastUpdatelinklastHttpStatusdeadcontentTypeitunesIdoriginalUrl...oldestItemPubdateepisodeCountpopularityScoreprioritycreatedOnupdateFrequencychashhostnewestEnclosureUrlpodcastGuid
02https://anchor.fm/s/19ccb320/podcast/rssRahdo Talks Through1648033575https://patreon.com/rahdo2000application/rss+xml; charset=utf-81.000016e+09https://anchor.fm/s/19ccb320/podcast/rss...14330412002171215967524871f9fce8424a9918be9182c2d67fc51c16anchor.fmhttps://anchor.fm/s/19ccb320/podcast/play/4946...655b4e1a-4deb-56a2-9256-cad10f099410
16https://anchor.fm/s/212c47d4/podcast/rssOriginALL1624463994https://originall.citypodcast.ro2000application/rss+xml; charset=utf-81.000036e+09https://anchor.fm/s/212c47d4/podcast/rss...1432937220480115967524877256f22c369b872a5b1f73c8636d21053anchor.fmhttps://anchor.fm/s/212c47d4/podcast/play/3585...fc42afa1-1aa7-52dd-aa49-851ae703a66a
28https://anchor.fm/s/28f184c/podcast/rssRNG Show1607423070https://anchor.fm/rngshow2000application/rss+xml; charset=utf-81.000075e+09https://anchor.fm/s/28f184c/podcast/rss...1517097600700-1159675248798cf077673fb4e9be26311c1726125b79anchor.fmhttps://anchor.fm/s/28f184c/podcast/play/15180...6ac4018d-ce57-5468-bb8b-d804699b9371
322https://anchor.fm/s/2e03f13c/podcast/rssLet's Watch Podcast1648150905anchor.fm/lets-watch-podcast2000application/rss+xml; charset=utf-81.000116e+09https://anchor.fm/s/2e03f13c/podcast/rss...14328576001170115967524872e08f0d0918ba35aabc117d08cb3283cdanchor.fmhttps://anchor.fm/s/2e03f13c/podcast/play/4955...2897322f-acd0-553b-829b-961e9f52b5cc
464https://anchor.fm/s/2118ce48/podcast/rssTechVolution1648055093https://techvolution.citypodcast.ro2000application/rss+xml; charset=utf-81.000277e+09https://anchor.fm/s/2118ce48/podcast/rss...1433032020590115967524875fe11470174f94e5d3b02f02ad3600fd0anchor.fmhttps://anchor.fm/s/2118ce48/podcast/play/4197...847ad4e1-c743-5594-93e7-3b663e1eaa8b
\n", "

5 rows × 28 columns

\n", "
" ], "text/plain": [ " id url title \\\n", "0 2 https://anchor.fm/s/19ccb320/podcast/rss Rahdo Talks Through \n", "1 6 https://anchor.fm/s/212c47d4/podcast/rss OriginALL \n", "2 8 https://anchor.fm/s/28f184c/podcast/rss RNG Show \n", "3 22 https://anchor.fm/s/2e03f13c/podcast/rss Let's Watch Podcast \n", "4 64 https://anchor.fm/s/2118ce48/podcast/rss TechVolution \n", "\n", " lastUpdate link lastHttpStatus dead \\\n", "0 1648033575 https://patreon.com/rahdo 200 0 \n", "1 1624463994 https://originall.citypodcast.ro 200 0 \n", "2 1607423070 https://anchor.fm/rngshow 200 0 \n", "3 1648150905 anchor.fm/lets-watch-podcast 200 0 \n", "4 1648055093 https://techvolution.citypodcast.ro 200 0 \n", "\n", " contentType itunesId \\\n", "0 application/rss+xml; charset=utf-8 1.000016e+09 \n", "1 application/rss+xml; charset=utf-8 1.000036e+09 \n", "2 application/rss+xml; charset=utf-8 1.000075e+09 \n", "3 application/rss+xml; charset=utf-8 1.000116e+09 \n", "4 application/rss+xml; charset=utf-8 1.000277e+09 \n", "\n", " originalUrl ... oldestItemPubdate \\\n", "0 https://anchor.fm/s/19ccb320/podcast/rss ... 1433041200 \n", "1 https://anchor.fm/s/212c47d4/podcast/rss ... 1432937220 \n", "2 https://anchor.fm/s/28f184c/podcast/rss ... 1517097600 \n", "3 https://anchor.fm/s/2e03f13c/podcast/rss ... 1432857600 \n", "4 https://anchor.fm/s/2118ce48/podcast/rss ... 1433032020 \n", "\n", " episodeCount popularityScore priority createdOn updateFrequency \\\n", "0 217 1 2 1596752487 1 \n", "1 48 0 1 1596752487 7 \n", "2 70 0 -1 1596752487 9 \n", "3 117 0 1 1596752487 2 \n", "4 59 0 1 1596752487 5 \n", "\n", " chash host \\\n", "0 f9fce8424a9918be9182c2d67fc51c16 anchor.fm \n", "1 256f22c369b872a5b1f73c8636d21053 anchor.fm \n", "2 8cf077673fb4e9be26311c1726125b79 anchor.fm \n", "3 e08f0d0918ba35aabc117d08cb3283cd anchor.fm \n", "4 fe11470174f94e5d3b02f02ad3600fd0 anchor.fm \n", "\n", " newestEnclosureUrl \\\n", "0 https://anchor.fm/s/19ccb320/podcast/play/4946... \n", "1 https://anchor.fm/s/212c47d4/podcast/play/3585... \n", "2 https://anchor.fm/s/28f184c/podcast/play/15180... \n", "3 https://anchor.fm/s/2e03f13c/podcast/play/4955... \n", "4 https://anchor.fm/s/2118ce48/podcast/play/4197... \n", "\n", " podcastGuid \n", "0 655b4e1a-4deb-56a2-9256-cad10f099410 \n", "1 fc42afa1-1aa7-52dd-aa49-851ae703a66a \n", "2 6ac4018d-ce57-5468-bb8b-d804699b9371 \n", "3 2897322f-acd0-553b-829b-961e9f52b5cc \n", "4 847ad4e1-c743-5594-93e7-3b663e1eaa8b \n", "\n", "[5 rows x 28 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head()" ] }, { "cell_type": "code", "execution_count": null, "id": "372210eb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b76e33ea", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "35a2cce9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d196eaee", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "31ef4102", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }