{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d2dd78f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tqdm\n",
    "import feedparser\n",
    "import time\n",
    "import random\n",
    "import datetime\n",
    "import pandas as pd\n",
    "import sqlite3\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d33de3ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/mnt/data-ssd-1/data/podcasts/anchor_subset.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "59c900af",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2297 total aussie podcasts\n",
      "84441 total episodes\n"
     ]
    }
   ],
   "source": [
    "aussie_df = df[\n",
    "    (df[\"language\"] == \"en-au\") |\n",
    "    (\n",
    "        (df[\"language\"] == \"en\") & \n",
    "        df[\"link\"].str.contains(r\"\\.[aA][uU]\\b\")\n",
    "    )\n",
    "]\n",
    "aussie_df = aussie_df[aussie_df[\"episodeCount\"] >= 5].reset_index(drop=True)\n",
    "print(aussie_df.shape[0], \"total aussie podcasts\")\n",
    "print(aussie_df[\"episodeCount\"].sum(), \"total episodes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f98e5b0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "feed_map = {}\n",
    "failed_ids = []\n",
    "for _, row in tqdm.tqdm(aussie_df.iterrows(), total=aussie_df.shape[0]):\n",
    "    url = row[\"url\"]\n",
    "    uid = row[\"id\"]\n",
    "    feed = feedparser.parse(url)\n",
    "    if len(feed[\"feed\"]) == 0:\n",
    "        failed_ids.append(uid)\n",
    "        continue\n",
    "    feed_map[uid] = feed\n",
    "    time.sleep(random.random() * 0.5)\n",
    "print(len(failed_ids), \"failed\")\n",
    "\n",
    "data_map = {}\n",
    "for uid, feed in feed_map.items():\n",
    "    # get avg episode duration\n",
    "    duration_l = []\n",
    "    for e in feed[\"entries\"]:\n",
    "        for l in e[\"links\"]:\n",
    "            if \"audio\" in l[\"type\"]:\n",
    "                duration_l.append(int(l[\"length\"]) / 14414)\n",
    "    data_map[uid] = {\n",
    "        \"avg_duration_s\": int(round(np.mean(duration_l))),\n",
    "        \"publisher_name\": feed[\"feed\"][\"publisher_detail\"][\"name\"],\n",
    "        \"publisher_email\": feed[\"feed\"][\"publisher_detail\"][\"email\"],\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "d9d33535",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49/2255 dropped\n"
     ]
    }
   ],
   "source": [
    "# subselect data\n",
    "aussie_df = aussie_df.rename(columns={\n",
    "    \"newestItemPubdate\": \"newest_item_date\",\n",
    "    \"oldestItemPubdate\": \"oldest_item_date\",\n",
    "    \"episodeCount\": \"episode_count\",\n",
    "})\n",
    "aussie_df = aussie_df[\n",
    "    [\"id\", \"url\", \"link\", \"title\", \"newest_item_date\", \"oldest_item_date\", \"episode_count\", \"explicit\"]\n",
    "]\n",
    "# format dates\n",
    "aussie_df[\"newest_item_date\"] = [\n",
    "    datetime.datetime.fromtimestamp(ts).strftime('%m/%d/%Y') for ts in aussie_df[\"newest_item_date\"].values\n",
    "]\n",
    "aussie_df[\"oldest_item_date\"] = [\n",
    "    datetime.datetime.fromtimestamp(ts).strftime('%m/%d/%Y') for ts in aussie_df[\"oldest_item_date\"].values\n",
    "]\n",
    "# add author name & email, average episode length\n",
    "aussie_df[\"publisher_name\"] = [data_map.get(uid, {}).get(\"publisher_name\") for uid in aussie_df[\"id\"].values]\n",
    "aussie_df[\"publisher_email\"] = [data_map.get(uid, {}).get(\"publisher_email\") for uid in aussie_df[\"id\"].values]\n",
    "aussie_df[\"avg_duration_s\"] = [data_map.get(uid, {}).get(\"avg_duration_s\") for uid in aussie_df[\"id\"].values]\n",
    "aussie_df[\"publisher_email\"] = aussie_df[\"publisher_email\"].str.strip().replace(\"\", np.nan)\n",
    "n_before = aussie_df.shape[0]\n",
    "aussie_df = aussie_df.dropna().reset_index(drop=True)\n",
    "aussie_df[\"avg_duration_s\"] = aussie_df[\"avg_duration_s\"].astype(int)\n",
    "aussie_df[\"total_duration_estimate_h\"] = (aussie_df[\"episode_count\"] * aussie_df[\"avg_duration_s\"] / 60 / 60).round(1)\n",
    "print(\"{}/{} dropped\".format(n_before - aussie_df.shape[0], n_before))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "88e89b22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2206 AU podcasts with 66902 hours total content.\n",
      "624 podcasts with a private email\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>link</th>\n",
       "      <th>title</th>\n",
       "      <th>newest_item_date</th>\n",
       "      <th>oldest_item_date</th>\n",
       "      <th>episode_count</th>\n",
       "      <th>explicit</th>\n",
       "      <th>publisher_name</th>\n",
       "      <th>publisher_email</th>\n",
       "      <th>avg_duration_s</th>\n",
       "      <th>total_duration_estimate_s</th>\n",
       "      <th>total_duration_estimate_h</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4550</td>\n",
       "      <td>https://anchor.fm/s/a5ae2f4/podcast/rss</td>\n",
       "      <td>https://anchor.fm/comments-and-musings</td>\n",
       "      <td>Comments and Musings</td>\n",
       "      <td>10/12/2020</td>\n",
       "      <td>07/22/2015</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>Francis Lynch</td>\n",
       "      <td>francislynch.me@gmail.com</td>\n",
       "      <td>1591</td>\n",
       "      <td>25456</td>\n",
       "      <td>7.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4770</td>\n",
       "      <td>https://anchor.fm/s/343ab914/podcast/rss</td>\n",
       "      <td>https://anchor.fm/genericliveshow</td>\n",
       "      <td>The Generic Live Show</td>\n",
       "      <td>03/28/2021</td>\n",
       "      <td>07/17/2017</td>\n",
       "      <td>107</td>\n",
       "      <td>0</td>\n",
       "      <td>Dale Campbell</td>\n",
       "      <td>podcasts60+343ab914@anchor.fm</td>\n",
       "      <td>3495</td>\n",
       "      <td>373965</td>\n",
       "      <td>103.9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id                                       url  \\\n",
       "0  4550   https://anchor.fm/s/a5ae2f4/podcast/rss   \n",
       "1  4770  https://anchor.fm/s/343ab914/podcast/rss   \n",
       "\n",
       "                                     link                  title  \\\n",
       "0  https://anchor.fm/comments-and-musings   Comments and Musings   \n",
       "1       https://anchor.fm/genericliveshow  The Generic Live Show   \n",
       "\n",
       "  newest_item_date oldest_item_date  episode_count  explicit publisher_name  \\\n",
       "0       10/12/2020       07/22/2015             16         0  Francis Lynch   \n",
       "1       03/28/2021       07/17/2017            107         0  Dale Campbell   \n",
       "\n",
       "                 publisher_email  avg_duration_s  total_duration_estimate_s  \\\n",
       "0      francislynch.me@gmail.com            1591                      25456   \n",
       "1  podcasts60+343ab914@anchor.fm            3495                     373965   \n",
       "\n",
       "   total_duration_estimate_h  \n",
       "0                        7.1  \n",
       "1                      103.9  "
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_podcasts = aussie_df.shape[0]\n",
    "n_total_hours = int(round(aussie_df[\"total_duration_estimate_h\"].sum()))\n",
    "print(\"{} AU podcasts with {} hours total content.\".format(n_podcasts, n_total_hours))\n",
    "n_email_nonanchor = (aussie_df[\"publisher_email\"].str.split(\"@\").str[-1] != \"anchor.fm\").sum()\n",
    "print(\"{} podcasts with a private email\".format(n_email_nonanchor))\n",
    "aussie_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "c040c18a",
   "metadata": {},
   "outputs": [],
   "source": [
    "aussie_df.to_csv(\"au_podcasts.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "713e599e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf7b414b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5635253d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79e4b37a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f647ff6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}