{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "da8c9577", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 2, "id": "7ad009ea", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"2\"" ] }, { "cell_type": "code", "execution_count": 51, "id": "f3b73488", "metadata": {}, "outputs": [], "source": [ "import time\n", "import os\n", "import tqdm\n", "import shutil\n", "import random\n", "import funcy\n", "import numpy as np\n", "import multiprocessing\n", "import pandas as pd\n", "from suno_utils.audio import Audio\n", "from suno_utils.utils.podcasts import load_podcast_db, get_title, get_description, load_rss_text" ] }, { "cell_type": "code", "execution_count": null, "id": "de886102", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "407b28bc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "c6021652", "metadata": {}, "source": [ "### get eastern european" ] }, { "cell_type": "code", "execution_count": 19, "id": "c95b8dd3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "25086 podcasts\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idtitledescriptioncountryevidence_termslatest_episode_urltagsauthor_nameauthor_emailn_episodesrss_urlurllast_updatedcountry_iso
04216979Canada Crossroads..CanadaCanadahttps://d3ctxlq1ktw2nl.cloudfront.net/staging/...news;sports newsAbby Lucianopodcasts60+431c17e8@anchor.fm3https://anchor.fm/s/431c17e8/podcast/rsshttps://anchor.fm/abby-luciano1630350441CAN
14484930Postcolonials in PetticoatsWelcome to Postcolonials in Petticoats, a podc...BrazilBrazilianhttps://d3ctxlq1ktw2nl.cloudfront.net/staging/...after shows;tv & filmPostcolonials in Petticoatspodcasts60+537be0c8@anchor.fm8https://anchor.fm/s/537be0c8/podcast/rsshttps://anchor.fm/postinpetti1636985131BRA
\n", "
" ], "text/plain": [ " id title \\\n", "0 4216979 Canada Crossroads \n", "1 4484930 Postcolonials in Petticoats \n", "\n", " description country evidence_terms \\\n", "0 .. Canada Canada \n", "1 Welcome to Postcolonials in Petticoats, a podc... Brazil Brazilian \n", "\n", " latest_episode_url tags \\\n", "0 https://d3ctxlq1ktw2nl.cloudfront.net/staging/... news;sports news \n", "1 https://d3ctxlq1ktw2nl.cloudfront.net/staging/... after shows;tv & film \n", "\n", " author_name author_email n_episodes \\\n", "0 Abby Luciano podcasts60+431c17e8@anchor.fm 3 \n", "1 Postcolonials in Petticoats podcasts60+537be0c8@anchor.fm 8 \n", "\n", " rss_url url \\\n", "0 https://anchor.fm/s/431c17e8/podcast/rss https://anchor.fm/abby-luciano \n", "1 https://anchor.fm/s/537be0c8/podcast/rss https://anchor.fm/postinpetti \n", "\n", " last_updated country_iso \n", "0 1630350441 CAN \n", "1 1636985131 BRA " ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "country_podcasts_df = pd.read_csv(\"/mnt/data-ssd-1/data/podcasts/meta/en_country_podcasts.csv\")\n", "print(country_podcasts_df.shape[0], \"podcasts\")\n", "country_podcasts_df.head(2)" ] }, { "cell_type": "code", "execution_count": 96, "id": "e19a5a50", "metadata": {}, "outputs": [], "source": [ "# easter_europe_country_names = [\n", "# \"Belarus\",\n", "# \"Czech Republic\",\n", "# \"Hungary\",\n", "# \"Moldova\",\n", "# \"Poland\",\n", "# \"Ukraine\",\n", "# ]\n", "\n", "# country_podcast_ids = {}\n", "# for country_name in tqdm.tqdm(easter_europe_country_names):\n", "# sub_df = country_podcasts_df[\n", "# (country_podcasts_df[\"country\"] == country_name) & \n", "# (country_podcasts_df[\"n_episodes\"] >= 3) &\n", "# (~country_podcasts_df[\"tags\"].str.contains(\"music\").fillna(False))\n", "# ]\n", "# podcast_ids = sub_df[\"id\"].tolist()\n", "# random.seed(6006)\n", "# random.shuffle(podcast_ids)\n", "# country_podcast_ids[country_name] = podcast_ids[:5]\n", "\n", "COUNTRY_PODCAST_IDS = {\n", " 'Belarus': [936831],\n", " 'Czech Republic': [396476, 892112, 3005898, 3008309],\n", " 'Hungary': [4537267, 1003291, 2280460, 3624680],\n", " 'Moldova': [800201, 3502233],\n", " 'Poland': [3050953, 1760119],\n", " 'Ukraine': [100045, 735616, 3107943, 3361147],\n", "}" ] }, { "cell_type": "code", "execution_count": 48, "id": "d77023b4", "metadata": {}, "outputs": [], "source": [ "# # manually check title and description and eliminate if needed\n", "# all_ids = set(funcy.flatten(COUNTRY_PODCAST_IDS.values()))\n", "# sub_df = country_podcasts_df[country_podcasts_df[\"id\"].isin(all_ids)]\n", "# for k, v in COUNTRY_PODCAST_IDS.items():\n", "# for vv in v:\n", "# row = sub_df[sub_df[\"id\"] == vv].iloc[0]\n", "# print(k, \"-\", vv)\n", "# print(row[\"title\"])\n", "# print(row[\"description\"])\n", "# print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": null, "id": "169571a9", "metadata": {}, "outputs": [], "source": [ "# # download episodes\n", "# \";\".join([str(e) for e in funcy.flatten(COUNTRY_PODCAST_IDS.values())])\n", "\n", "# python /home/georg/code/glockenspiel/suno_utils/suno_utils/scripts/fetch_podcasts.py \\\n", "# --podcast_db_filepath=\"/mnt/data-ssd-1/data/podcasts/meta/podcastindex_feeds.db\" \\\n", "# --podcast_ids=\"936831\" \\\n", "# --output_dir=\"/mnt/data-ssd-1/data/podcasts/audio\" \\\n", "# --refresh_feed \\\n", "# --last_n_episodes=1" ] }, { "cell_type": "code", "execution_count": 53, "id": "d8dee56b", "metadata": {}, "outputs": [], "source": [ "PODCAST_AUDIO_DIR = \"/mnt/data-ssd-1/data/podcasts/audio/\"" ] }, { "cell_type": "code", "execution_count": null, "id": "8fdbed4f", "metadata": {}, "outputs": [], "source": [ "country_episode_metas = {}\n", "for country, podcast_ids in COUNTRY_PODCAST_IDS.items():\n", " episode_metas = []\n", " for podcast_id in podcast_ids:\n", " print('bla')\n", " podcast_id = str(podcast_id)\n", " podcast_dir = os.path.join(PODCAST_AUDIO_DIR, podcast_id)\n", " for fn in os.listdir(podcast_dir):\n", " try:\n", " filepath = os.path.join(podcast_dir, fn)\n", " duration_s = Audio.get_details(filepath)[\"duration_s\"]\n", " print(duration_s)\n", " if duration_s >= 3*60:\n", " episode_id = re.sub(r\"\\.[^\\.]+$\", \"\", fn)\n", " episode_metas.append({\n", " \"podcast_id\": podcast_id, \n", " \"episode_id\": episode_id, \n", " \"filepath\": filepath, \n", " \"duration_s\": np.min([duration_s, 20*60]),\n", " })\n", " except:\n", " pass\n", " country_episode_metas[country] = episode_metas" ] }, { "cell_type": "code", "execution_count": 100, "id": "6c6aee65", "metadata": {}, "outputs": [], "source": [ "country_episode_metas = {}\n", "for country, podcast_ids in COUNTRY_PODCAST_IDS.items():\n", " episode_metas = []\n", " for podcast_id in podcast_ids:\n", " podcast_id = str(podcast_id)\n", " podcast_dir = os.path.join(PODCAST_AUDIO_DIR, podcast_id)\n", " for fn in os.listdir(podcast_dir):\n", " try:\n", " filepath = os.path.join(podcast_dir, fn)\n", " duration_s = Audio.get_details(filepath)[\"duration_s\"]\n", " if duration_s >= 3*60:\n", " episode_id = re.sub(r\"\\.[^\\.]+$\", \"\", fn)\n", " episode_metas.append({\n", " \"podcast_id\": podcast_id, \n", " \"episode_id\": episode_id, \n", " \"filepath\": filepath, \n", " \"duration_s\": np.min([duration_s, 20*60]),\n", " })\n", " except:\n", " pass\n", " country_episode_metas[country] = episode_metas" ] }, { "cell_type": "code", "execution_count": null, "id": "4ac3eb63", "metadata": {}, "outputs": [], "source": [ "country_episode_metas[\"Belarus\"][0]" ] }, { "cell_type": "code", "execution_count": 78, "id": "631775b6", "metadata": {}, "outputs": [], "source": [ "shutil.rmtree(\"tmp/easter_europe_segments\", ignore_errors=True)\n", "os.makedirs(\"tmp/easter_europe_segments\", exist_ok=True)" ] }, { "cell_type": "code", "execution_count": 79, "id": "d443f6a5", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████| 6/6 [09:15<00:00, 92.53s/it]\n" ] } ], "source": [ "import random\n", "import tqdm\n", "from suno_utils.utils.slicer import get_segments\n", "import uuid\n", "\n", "SEGMENT_DIR = \"/home/georg/notebooks/datasets/tmp/easter_europe_segments/\"\n", "\n", "country_segment_metas = {}\n", "for country, episode_metas in tqdm.tqdm(country_episode_metas.items()):\n", " segment_metas = []\n", " for episode_meta in episode_metas:\n", " audio = Audio.from_file(episode_meta[\"filepath\"]).get_slice(to_s=episode_meta[\"duration_s\"])\n", " episode_segments = get_segments(\n", " audio,\n", " min_gap_s=0.05,\n", " vad_level=3,\n", " min_dur_s=10,\n", " max_dur_s=20,\n", " )\n", " for segment_audio, (start_s, end_s) in episode_segments:\n", " segment_id = str(uuid.uuid4())\n", " segment_filepath = os.path.join(SEGMENT_DIR, segment_id) + \".mp3\"\n", " segment_audio.to_mp3(segment_filepath)\n", " segment_metas.append({\n", " \"podcast_id\": episode_meta[\"podcast_id\"], \n", " \"episode_id\": episode_meta[\"episode_id\"], \n", " \"uuid\": segment_id,\n", " \"source_audio_offset_s\": start_s,\n", " \"filepath\": segment_filepath, \n", " \"duration_s\": end_s - start_s,\n", " })\n", " random.seed(6006)\n", " random.shuffle(segment_metas)\n", " country_segment_metas[country] = segment_metas" ] }, { "cell_type": "code", "execution_count": 83, "id": "78a2f73f", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[{'podcast_id': '735616',\n", " 'episode_id': '829cab75-ab99-45da-96fe-dd0407602e6f',\n", " 'uuid': '01bf26f7-3c0e-4dd2-bda1-c19c0052d4d1',\n", " 'source_audio_offset_s': 880.5,\n", " 'filepath': '/home/georg/notebooks/datasets/tmp/easter_europe_segments/01bf26f7-3c0e-4dd2-bda1-c19c0052d4d1.mp3',\n", " 'duration_s': 19.710000000000036},\n", " {'podcast_id': '3107943',\n", " 'episode_id': 'e50eca0f-1dde-4627-8f82-d0fb56246edd',\n", " 'uuid': 'b1fc2f08-b5b1-4dc5-9b87-ca18cbe8d6aa',\n", " 'source_audio_offset_s': 719.13,\n", " 'filepath': '/home/georg/notebooks/datasets/tmp/easter_europe_segments/b1fc2f08-b5b1-4dc5-9b87-ca18cbe8d6aa.mp3',\n", " 'duration_s': 19.480000000000018}]" ] }, "execution_count": 83, "metadata": {}, "output_type": "execute_result" } ], "source": [ "segment_metas[:2]" ] }, { "cell_type": "code", "execution_count": 86, "id": "6fc11369", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Belarus - 1553355 - 1eaa098b-189c-41ad-9a53-4ec40bc6d976\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Belarus - 1553355 - a2896fcb-8044-4572-8148-8b15672ce057\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Belarus - 1553355 - b7a1f920-3090-4fc9-ac7a-d47c3c64f292\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Belarus - 1553355 - f37d24bb-1666-4025-ae85-50da2df409f4\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Belarus - 1553355 - b77d1e48-207b-4d90-b4a8-485500b9d1e6\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Czech Republic - 892112 - 26466e38-48d6-413a-8cca-6caf9fe2c703\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Czech Republic - 3005898 - a637100f-a1db-4f96-98fb-7012de16f861\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Czech Republic - 396476 - 68b944f0-9cfb-496e-8414-9d11b3d0e386\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Czech Republic - 3005898 - b423957c-614e-4831-9a34-08894315d3db\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Czech Republic - 396476 - a5b00f19-06da-46ce-b7d5-e8ba4451f845\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Hungary - 1003291 - b69b89c6-4546-46ec-a163-3d35b17325ce\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Hungary - 4537267 - 664f9349-dfd1-4edc-ab26-6f3e8584ae29\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Hungary - 3624680 - 6d5787c7-0214-4946-8d80-2c869be4ba42\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Hungary - 2280460 - 8e34adb6-959b-47f5-9f01-419da6826771\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Hungary - 3624680 - 683df16f-acd4-49ae-b1df-9310c2186c33\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Moldova - 3502233 - 9f651230-ff87-4893-81be-8f4a00d3438c\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Moldova - 800201 - be980b3b-f3c4-4d74-b5b1-995861310a75\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Moldova - 3502233 - e0c10d7e-ba79-4224-b05b-528f3e9ca0cd\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Moldova - 3502233 - 808395df-1cb8-4618-a052-f5e629be1966\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Moldova - 800201 - b4bf966d-35ee-4693-a47b-28f5231a9081\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Poland - 1760119 - a51959eb-03b7-49bc-bfbb-3b7a542070f7\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Poland - 3050953 - 7cbbd3a6-b953-4825-aac2-b354d0aa83eb\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Poland - 1760119 - b614057d-2924-49b5-b556-0bd0f8f59f44\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Poland - 1760119 - 985c1e68-53a6-4d8a-9c10-77b1cccf0d94\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Poland - 3050953 - 149b12ce-556a-4c21-a66b-429b87df72a5\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Ukraine - 735616 - 01bf26f7-3c0e-4dd2-bda1-c19c0052d4d1\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Ukraine - 3107943 - b1fc2f08-b5b1-4dc5-9b87-ca18cbe8d6aa\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Ukraine - 3361147 - 86d81fd9-5dcd-483f-a892-850f773d4c01\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Ukraine - 735616 - 0ca1328c-b43a-455c-b662-03ad098c0e88\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "Ukraine - 735616 - c2ab2ea3-0a50-4779-8eaf-8797db235fd6\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n" ] } ], "source": [ "for country, segment_metas in country_segment_metas.items():\n", " for e in segment_metas[:5]:\n", " print(country, \"-\", e[\"podcast_id\"], \"-\", e[\"uuid\"])\n", " Audio.from_file(e[\"filepath\"]).play()\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 77, "id": "0547432d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'3361147'}" ] }, "execution_count": 77, "metadata": {}, "output_type": "execute_result" } ], "source": [ "set([e[\"podcast_id\"] for e in country_segment_metas[\"Czech Republic\"]])" ] }, { "cell_type": "code", "execution_count": 120, "id": "16fc5a42", "metadata": {}, "outputs": [], "source": [ "s = \"\"\"\n", "Belarus - 936831 - 65cf6baa-d457-43f6-93a2-4982ee2cc5b8\n", "Belarus - 936831 - 9c6c4ffc-1ae3-481e-926f-6c18183a052a\n", "Czech Republic - 892112 - 26466e38-48d6-413a-8cca-6caf9fe2c703\n", "Czech Republic - 3005898 - b423957c-614e-4831-9a34-08894315d3db\n", "Hungary - 4537267 - 664f9349-dfd1-4edc-ab26-6f3e8584ae29\n", "Hungary - 2280460 - 8e34adb6-959b-47f5-9f01-419da6826771\n", "Moldova - 3502233 - 9f651230-ff87-4893-81be-8f4a00d3438c\n", "Moldova - 3502233 - 808395df-1cb8-4618-a052-f5e629be1966\n", "Poland - 3050953 - 7cbbd3a6-b953-4825-aac2-b354d0aa83eb\n", "Poland - 3050953 - 149b12ce-556a-4c21-a66b-429b87df72a5\n", "Ukraine - 3107943 - b1fc2f08-b5b1-4dc5-9b87-ca18cbe8d6aa\n", "Ukraine - 735616 - 0ca1328c-b43a-455c-b662-03ad098c0e88\n", "\"\"\"\n", "data = []\n", "for l in s.strip().split(\"\\n\"):\n", " c, pid, sid = l.split(\" - \")\n", " data.append({\n", " \"country\": c,\n", " \"podcast_id\": pid,\n", " \"uid\": sid,\n", " \"filepath\": SEGMENT_DIR + sid + \".mp3\",\n", " })" ] }, { "cell_type": "code", "execution_count": 125, "id": "2eda3a2f", "metadata": {}, "outputs": [], "source": [ "import json\n", "with open(\"tmp/eastern_europe_meta.json\", \"w\") as f:\n", " json.dump(data, f)" ] }, { "cell_type": "code", "execution_count": 126, "id": "25aba252", "metadata": {}, "outputs": [], "source": [ "!mkdir tmp/balto" ] }, { "cell_type": "code", "execution_count": 130, "id": "cc6f02bf", "metadata": {}, "outputs": [], "source": [ "from collections import Counter\n", "c_counter = Counter()\n", "for e in data:\n", " name = e[\"country\"].lower().replace(\" \", \"_\") + f\"_{c_counter[e['country']]}\"\n", " c_counter[e['country']] += 1\n", " to_fp = f\"tmp/balto/{name}.mp3\"\n", " shutil.copy(e[\"filepath\"], to_fp)" ] }, { "cell_type": "code", "execution_count": 131, "id": "84335ddd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "belarus_0.mp3\t czech_republic_1.mp3 moldova_0.mp3 poland_1.mp3\r\n", "belarus_1.mp3\t hungary_0.mp3\t moldova_1.mp3 ukraine_0.mp3\r\n", "czech_republic_0.mp3 hungary_1.mp3\t poland_0.mp3 ukraine_1.mp3\r\n" ] } ], "source": [ "!ls tmp/balto" ] }, { "cell_type": "code", "execution_count": null, "id": "b7466da2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "15996aba", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d6955b00", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7d7d8c6e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "3396aa54", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": null, "id": "a64c9770", "metadata": {}, "outputs": [], "source": [ "country_podcasts_df = pd.read_csv(\"country_podcasts.csv\")\n", "\n", "countries_df = pd.read_json(\"meta_data/country_aliases.json\")\n", "name2iso = (\n", " countries_df.drop_duplicates(subset=[\"country_name\"], keep=\"first\")\n", " .set_index(\"country_name\")[\"country_iso\"].to_dict()\n", ")\n", "\n", "plot_df = country_podcasts_df[\"country\"].value_counts().to_frame(\"n_podcasts\")\n", "plot_df = plot_df.rename_axis(\"country\").reset_index()\n", "plot_df[\"country_iso\"] = plot_df[\"country\"].map(name2iso)\n", "plot_df = plot_df.dropna().reset_index(drop=True)\n", "en_tot_podcasts = 2_155_251\n", "avg_hours = 22\n", "plot_df[\"frac\"] = plot_df[\"n_podcasts\"] / plot_df[\"n_podcasts\"].sum()\n", "plot_df[\"n_hours_tot\"] = plot_df[\"frac\"] * en_tot_podcasts * avg_hours\n", "plot_df.head(2)" ] }, { "cell_type": "code", "execution_count": null, "id": "c3f550be", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d7afa2bd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 59, "id": "5aeae4ab", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "4943337" ] }, "execution_count": 59, "metadata": {}, "output_type": "execute_result" } ], "source": [ "inv_url_map[\"https://anchor.fm/sahil8097\"]" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }