{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "55b6e321",
   "metadata": {},
   "source": [
    "## get all anchors with twitter links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d99628c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import requests\n",
    "import re\n",
    "import multiprocessing\n",
    "import tqdm\n",
    "import funcy\n",
    "import random\n",
    "import time\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "from suno_utils.audio import Audio\n",
    "from suno_utils.utils.podcasts import load_podcast_db\n",
    "\n",
    "def get_social_links(station_id):\n",
    "    try:\n",
    "        url = f\"https://anchor.fm/api/proxy/v3/userSocialUrl/station/webStationId:{station_id}\"\n",
    "        res = requests.get(url, timeout=1)\n",
    "        out = res.json()\n",
    "    except:\n",
    "        out = None\n",
    "    time.sleep(0.05 * (1 + random.random()))\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ed674344",
   "metadata": {},
   "outputs": [],
   "source": [
    "# podcast_df = load_podcast_db(\n",
    "#     \"/data/georg/podcasts/podcastindex_feeds.db\", english_only=True, anchor_only=True\n",
    "# )\n",
    "podcast_df = pd.read_csv(\"/home/georg/notebooks/podcast_urls_anchor.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5f9c7914",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9 invalid station IDs found.\n",
      "1793760 station IDs found.\n"
     ]
    }
   ],
   "source": [
    "re_ptn = r\"\\/s\\/(.+?)\\/podcast\"\n",
    "\n",
    "failed_ids = []\n",
    "work_items = list(zip(podcast_df[\"uid\"].tolist(), podcast_df[\"url\"].tolist()))\n",
    "for n, (uid, url) in enumerate(work_items):\n",
    "    m = re.search(re_ptn, url.lower())\n",
    "    if m:\n",
    "        station_id = m.group(1)\n",
    "        work_items[n] = (uid, station_id)\n",
    "    else:\n",
    "        failed_ids.append(uid)\n",
    "print(len(failed_ids), \"invalid station IDs found.\")\n",
    "print(len(work_items), \"station IDs found.\")\n",
    "# 3 invalid station IDs found.\n",
    "# 922118 station IDs found.\n",
    "# 9 invalid station IDs found.\n",
    "# 1793760 station IDs found."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "172c27a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'userSocialUrls': [{'type': 'facebook',\n",
       "   'url': 'https://facebook.com/rahdorunsthrough',\n",
       "   'username': 'rahdorunsthrough'},\n",
       "  {'type': 'instagram',\n",
       "   'url': 'https://instagram.com/rahdorunsthrough',\n",
       "   'username': 'rahdorunsthrough'},\n",
       "  {'type': 'twitter', 'url': 'https://twitter.com/Rahdo', 'username': 'Rahdo'},\n",
       "  {'type': 'youtube',\n",
       "   'url': 'https://youtube.com/rahdo',\n",
       "   'username': 'rahdo'}]}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "social_links = get_social_links(work_items[0][1])\n",
    "social_links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fd763d2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "out_filepath = \"/home/georg/notebooks/tmp/twitter_links.jsonl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6cdabacf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "520971 already done\n",
      "1272789 work items left\n"
     ]
    }
   ],
   "source": [
    "out = []\n",
    "if os.path.exists(out_filepath):\n",
    "    with open(out_filepath) as f:\n",
    "        for e in f.read().strip().split(\"\\n\"):\n",
    "            out.append(json.loads(e))\n",
    "print(len(out), \"already done\")\n",
    "seen_uids = set([e[\"uid\"] for e in out])\n",
    "rel_work_items = [e for e in work_items if e[0] not in seen_uids]\n",
    "print(len(rel_work_items), \"work items left\")\n",
    "# 410000 already done\n",
    "# 1383760 work items left"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0409689b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_9095/1645826830.py:2: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  for work_items_chunk in tqdm.tqdm_notebook(list(funcy.chunks(10_000, rel_work_items))):\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b984aee8b6cd4ac2ac5b0052ea53d39a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/128 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = multiprocessing.Pool(5)\n",
    "for work_items_chunk in tqdm.tqdm_notebook(list(funcy.chunks(10_000, rel_work_items))):\n",
    "    o = p.map(get_social_links, [e[-1] for e in work_items_chunk], chunksize=50)\n",
    "    for social_links, (uid, url) in zip(o, work_items_chunk):\n",
    "        out.append({\n",
    "            \"uid\": uid,\n",
    "            \"url\": url,\n",
    "            \"social_links\": social_links,\n",
    "        })\n",
    "    with open(out_filepath, \"w\") as f:\n",
    "        for e in out:\n",
    "            f.write(json.dumps(e) + \"\\n\")    \n",
    "p.close()\n",
    "p.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e64967c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1793760\r\n"
     ]
    }
   ],
   "source": [
    "# !cat /home/georg/notebooks/tmp/twitter_links.jsonl | wc -l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f63b51be",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "639511da",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b41cf28",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "809ee96a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49a080b3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11e9085d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fa623c7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}