{ "cells": [ { "cell_type": "code", "execution_count": 168, "id": "1fbd4ed1", "metadata": {}, "outputs": [], "source": [ "import requests\n", "import json\n", "import re\n", "\n", "def _parse_youtube_search_results(html_str):\n", " start_idx = html_str.index(\"ytInitialData\") + len(\"ytInitialData\") + 3\n", " end_idx = html_str.index(\"};\", start_idx) + 1\n", " json_str = html_str[start_idx:end_idx]\n", " page_info = json.loads(json_str)\n", " primary_content = page_info[\"contents\"][\"twoColumnSearchResultsRenderer\"].get(\"primaryContents\")\n", " if primary_content is not None:\n", " content_blocks = primary_content[\"sectionListRenderer\"][\"contents\"]\n", " else:\n", " coninuation_content = page_info[\"onResponseReceivedCommands\"][0][\"appendContinuationItemsAction\"]\n", " content_blocks = coninuation_content[\"continuationItems\"]\n", " results = []\n", " for contents in content_blocks[:-1]:\n", " for video in contents[\"itemSectionRenderer\"][\"contents\"]:\n", " res = {}\n", " if \"videoRenderer\" in video.keys():\n", " video_data = video.get(\"videoRenderer\", {})\n", " res[\"id\"] = video_data.get(\"videoId\", None)\n", " res[\"title\"] = video_data.get(\"title\", {}).get(\"runs\", [[{}]])[0].get(\"text\", None)\n", " res[\"long_desc\"] = video_data.get(\"descriptionSnippet\", {}).get(\"runs\", [{}])[0].get(\"text\", None)\n", " res[\"channel\"] = video_data.get(\"longBylineText\", {}).get(\"runs\", [[{}]])[0].get(\"text\", None)\n", " res[\"duration\"] = video_data.get(\"lengthText\", {}).get(\"simpleText\", None)\n", " res[\"views\"] = video_data.get(\"viewCountText\", {}).get(\"simpleText\", None)\n", " res[\"publish_time\"] = video_data.get(\"publishedTimeText\", {}).get(\"simpleText\", None)\n", " res[\"url_suffix\"] = video_data.get(\n", " \"navigationEndpoint\", {}\n", " ).get(\"commandMetadata\", {}).get(\"webCommandMetadata\", {}).get(\"url\", None)\n", " results.append(res)\n", " continuation_endpoint = content_blocks[-1][\"continuationItemRenderer\"][\"continuationEndpoint\"]\n", " continuation_token = continuation_endpoint[\"continuationCommand\"][\"token\"]\n", " return continuation_token, results" ] }, { "cell_type": "code", "execution_count": 170, "id": "a6c68440", "metadata": {}, "outputs": [], "source": [ "out = requests.get(\"https://www.youtube.com/results?search_query=business&sp=EgIoAQ%253D%253D\")" ] }, { "cell_type": "code", "execution_count": 137, "id": "03bba856", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "20" ] }, "execution_count": 137, "metadata": {}, "output_type": "execute_result" } ], "source": [ "continuation_token, results = _parse_youtube_search_results(out.text)\n", "len(results)" ] }, { "cell_type": "code", "execution_count": 180, "id": "a47c0dd8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'id': 'zuBEwXo69nA',\n", " 'title': 'Fire destroys family-owned business in north Houston',\n", " 'long_desc': None,\n", " 'channel': 'FOX 26 Houston',\n", " 'duration': '2:22',\n", " 'views': '284 views',\n", " 'publish_time': '1 hour ago',\n", " 'url_suffix': '/watch?v=zuBEwXo69nA'}" ] }, "execution_count": 180, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[0]" ] }, { "cell_type": "code", "execution_count": 174, "id": "c99d8c7d", "metadata": {}, "outputs": [], "source": [ "out2 = requests.get(f\"https://www.youtube.com/results?continuation={continuation_token}&ctoken={continuation_token}\")" ] }, { "cell_type": "code", "execution_count": 175, "id": "45e73fc3", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "20" ] }, "execution_count": 175, "metadata": {}, "output_type": "execute_result" } ], "source": [ "continuation_token2, results2 = _parse_youtube_search_results(out2.text)\n", "len(results2)" ] }, { "cell_type": "code", "execution_count": null, "id": "1784b36c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a2a176b5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "d9ffebb9", "metadata": {}, "source": [ "## use google account" ] }, { "cell_type": "code", "execution_count": 1, "id": "b16079fa", "metadata": {}, "outputs": [], "source": [ "from apiclient.discovery import build\n", "from apiclient.errors import HttpError\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " DEVELOPER_KEY = json.load(f)[\"personal_google_api_key\"]\n", "\n", "YOUTUBE_API_SERVICE_NAME = \"youtube\"\n", "YOUTUBE_API_VERSION = \"v3\"\n", "\n", "youtube = build(\n", " YOUTUBE_API_SERVICE_NAME, \n", " YOUTUBE_API_VERSION,\n", " developerKey=DEVELOPER_KEY\n", ")" ] }, { "cell_type": "code", "execution_count": 79, "id": "92d3000f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "50\n", "100\n", "149\n" ] } ], "source": [ "# https://developers.google.com/youtube/v3/docs/search/list\n", "GIGA_DOMAINS = [\n", " \"Arts\", \"Business\", \"Education\", \"Autos and Vehicles\", \"Comedy\", \"Crime\", \"Entertainment\", \n", " \"Film and Animation\", \"Gaming\", \"Health and Fitness\", \"History\", \"Howto and Style\", \n", " \"Kids and Family, Leisure\", \"Music\", \"News and Politics\", \"Nonprofits and Activism\", \n", " \"People and Blogs\", \"Pets and Animals\", \"Religion and Spirituality\", \"Science and Technology\", \n", " \"Society and Culture\", \"Sports\", \"Travel and Events\",\n", "]\n", "\n", "def get_youtube_api_results(query_term, max_n_pages=None):\n", " all_results = []\n", " seen_ids = set()\n", " continuation_token = None\n", " n_pages = 1\n", " while True:\n", " search_response = youtube.search().list(\n", " q=query_term,\n", " type=\"video\",\n", " pageToken=continuation_token,\n", " order=\"relevance\",\n", " part=\"id,snippet\", # recordingDetails, statistics\n", " maxResults=50,\n", " videoCaption=\"closedCaption\",\n", "# videoCategoryId=1,\n", "# location=\"40.7128, 74.0060\",\n", "# locationRadius=\"50mi\",\n", " regionCode=\"AT\",\n", " relevanceLanguage=\"en\",\n", " ).execute()\n", " for m in search_response[\"items\"]:\n", " video_id = m[\"id\"][\"videoId\"]\n", " if video_id in seen_ids:\n", " continue\n", " seen_ids.add(video_id)\n", " all_results.append(m)\n", " continuation_token = search_response.get(\"nextPageToken\")\n", " if continuation_token is None:\n", " break\n", " if max_n_pages is not None and n_pages >= max_n_pages:\n", " break\n", " n_pages += 1\n", " return all_results\n", "\n", "# can use api to get captions from here\n", "# https://www.googleapis.com/youtube/v3/captions?part=id,snippet&videoId=&key=\n", "# need oauth to download from here\n", "# https://www.googleapis.com/youtube/v3/captions/\n", "\n", "results = get_youtube_api_results(\"business\", max_n_pages=3)" ] }, { "cell_type": "code", "execution_count": 78, "id": "65f4cfad", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "8" ] }, "execution_count": 78, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(all_results)" ] }, { "cell_type": "code", "execution_count": null, "id": "4b88cae5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c00f94e2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "69d6eb2d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "62f91c7a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "609ae1ab", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b1b27b36", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "6d574491", "metadata": {}, "source": [ "## get close captions" ] }, { "cell_type": "code", "execution_count": 66, "id": "5dbe3f6e", "metadata": {}, "outputs": [], "source": [ "import youtube_dl" ] }, { "cell_type": "code", "execution_count": 68, "id": "7e0ea9d4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[youtube] zuBEwXo69nA: Downloading webpage\n", "[info] Writing video subtitles to: /home/georg/notebooks/datasets/youtube/test.en.vtt\n" ] } ], "source": [ "youtube_url = \"http://www.youtube.com/watch?v=zuBEwXo69nA\"\n", "\n", "ydl_opts = {\n", " \"skip_download\": True,\n", " \"writesubtitles\": True,\n", " \"outtmpl\": \"/home/georg/notebooks/datasets/youtube/test\",\n", " \"allsubtitles\": True,\n", " \"subtitlesformat\": \"best\",\n", "# \"listsubtitles\": True,\n", "# \"subtitleslangs\": [\"en\", \"en-US\"],\n", "}\n", "with youtube_dl.YoutubeDL(ydl_opts) as ydl:\n", " out = ydl.download([youtube_url])" ] }, { "cell_type": "code", "execution_count": 67, "id": "8a214f0f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[youtube] zuBEwXo69nA: Downloading webpage\n", "Available automatic captions for zuBEwXo69nA:\n", "Language formats\n", "af vtt, ttml, srv3, srv2, srv1\n", "ak vtt, ttml, srv3, srv2, srv1\n", "sq vtt, ttml, srv3, srv2, srv1\n", "am vtt, ttml, srv3, srv2, srv1\n", "ar vtt, ttml, srv3, srv2, srv1\n", "hy vtt, ttml, srv3, srv2, srv1\n", "as vtt, ttml, srv3, srv2, srv1\n", "ay vtt, ttml, srv3, srv2, srv1\n", "az vtt, ttml, srv3, srv2, srv1\n", "bn vtt, ttml, srv3, srv2, srv1\n", "eu vtt, ttml, srv3, srv2, srv1\n", "be vtt, ttml, srv3, srv2, srv1\n", "bho vtt, ttml, srv3, srv2, srv1\n", "bs vtt, ttml, srv3, srv2, srv1\n", "bg vtt, ttml, srv3, srv2, srv1\n", "my vtt, ttml, srv3, srv2, srv1\n", "ca vtt, ttml, srv3, srv2, srv1\n", "ceb vtt, ttml, srv3, srv2, srv1\n", "zh-Hans vtt, ttml, srv3, srv2, srv1\n", "zh-Hant vtt, ttml, srv3, srv2, srv1\n", "co vtt, ttml, srv3, srv2, srv1\n", "hr vtt, ttml, srv3, srv2, srv1\n", "cs vtt, ttml, srv3, srv2, srv1\n", "da vtt, ttml, srv3, srv2, srv1\n", "dv vtt, ttml, srv3, srv2, srv1\n", "nl vtt, ttml, srv3, srv2, srv1\n", "en vtt, ttml, srv3, srv2, srv1\n", "eo vtt, ttml, srv3, srv2, srv1\n", "et vtt, ttml, srv3, srv2, srv1\n", "ee vtt, ttml, srv3, srv2, srv1\n", "fil vtt, ttml, srv3, srv2, srv1\n", "fi vtt, ttml, srv3, srv2, srv1\n", "fr vtt, ttml, srv3, srv2, srv1\n", "gl vtt, ttml, srv3, srv2, srv1\n", "lg vtt, ttml, srv3, srv2, srv1\n", "ka vtt, ttml, srv3, srv2, srv1\n", "de vtt, ttml, srv3, srv2, srv1\n", "el vtt, ttml, srv3, srv2, srv1\n", "gn vtt, ttml, srv3, srv2, srv1\n", "gu vtt, ttml, srv3, srv2, srv1\n", "ht vtt, ttml, srv3, srv2, srv1\n", "ha vtt, ttml, srv3, srv2, srv1\n", "haw vtt, ttml, srv3, srv2, srv1\n", "iw vtt, ttml, srv3, srv2, srv1\n", "hi vtt, ttml, srv3, srv2, srv1\n", "hmn vtt, ttml, srv3, srv2, srv1\n", "hu vtt, ttml, srv3, srv2, srv1\n", "is vtt, ttml, srv3, srv2, srv1\n", "ig vtt, ttml, srv3, srv2, srv1\n", "id vtt, ttml, srv3, srv2, srv1\n", "ga vtt, ttml, srv3, srv2, srv1\n", "it vtt, ttml, srv3, srv2, srv1\n", "ja vtt, ttml, srv3, srv2, srv1\n", "jv vtt, ttml, srv3, srv2, srv1\n", "kn vtt, ttml, srv3, srv2, srv1\n", "kk vtt, ttml, srv3, srv2, srv1\n", "km vtt, ttml, srv3, srv2, srv1\n", "rw vtt, ttml, srv3, srv2, srv1\n", "ko vtt, ttml, srv3, srv2, srv1\n", "kri vtt, ttml, srv3, srv2, srv1\n", "ku vtt, ttml, srv3, srv2, srv1\n", "ky vtt, ttml, srv3, srv2, srv1\n", "lo vtt, ttml, srv3, srv2, srv1\n", "la vtt, ttml, srv3, srv2, srv1\n", "lv vtt, ttml, srv3, srv2, srv1\n", "ln vtt, ttml, srv3, srv2, srv1\n", "lt vtt, ttml, srv3, srv2, srv1\n", "lb vtt, ttml, srv3, srv2, srv1\n", "mk vtt, ttml, srv3, srv2, srv1\n", "mg vtt, ttml, srv3, srv2, srv1\n", "ms vtt, ttml, srv3, srv2, srv1\n", "ml vtt, ttml, srv3, srv2, srv1\n", "mt vtt, ttml, srv3, srv2, srv1\n", "mi vtt, ttml, srv3, srv2, srv1\n", "mr vtt, ttml, srv3, srv2, srv1\n", "mn vtt, ttml, srv3, srv2, srv1\n", "ne vtt, ttml, srv3, srv2, srv1\n", "nso vtt, ttml, srv3, srv2, srv1\n", "no vtt, ttml, srv3, srv2, srv1\n", "ny vtt, ttml, srv3, srv2, srv1\n", "or vtt, ttml, srv3, srv2, srv1\n", "om vtt, ttml, srv3, srv2, srv1\n", "ps vtt, ttml, srv3, srv2, srv1\n", "fa vtt, ttml, srv3, srv2, srv1\n", "pl vtt, ttml, srv3, srv2, srv1\n", "pt vtt, ttml, srv3, srv2, srv1\n", "pa vtt, ttml, srv3, srv2, srv1\n", "qu vtt, ttml, srv3, srv2, srv1\n", "ro vtt, ttml, srv3, srv2, srv1\n", "ru vtt, ttml, srv3, srv2, srv1\n", "sm vtt, ttml, srv3, srv2, srv1\n", "sa vtt, ttml, srv3, srv2, srv1\n", "gd vtt, ttml, srv3, srv2, srv1\n", "sr vtt, ttml, srv3, srv2, srv1\n", "sn vtt, ttml, srv3, srv2, srv1\n", "sd vtt, ttml, srv3, srv2, srv1\n", "si vtt, ttml, srv3, srv2, srv1\n", "sk vtt, ttml, srv3, srv2, srv1\n", "sl vtt, ttml, srv3, srv2, srv1\n", "so vtt, ttml, srv3, srv2, srv1\n", "st vtt, ttml, srv3, srv2, srv1\n", "es vtt, ttml, srv3, srv2, srv1\n", "su vtt, ttml, srv3, srv2, srv1\n", "sw vtt, ttml, srv3, srv2, srv1\n", "sv vtt, ttml, srv3, srv2, srv1\n", "tg vtt, ttml, srv3, srv2, srv1\n", "ta vtt, ttml, srv3, srv2, srv1\n", "tt vtt, ttml, srv3, srv2, srv1\n", "te vtt, ttml, srv3, srv2, srv1\n", "th vtt, ttml, srv3, srv2, srv1\n", "ti vtt, ttml, srv3, srv2, srv1\n", "ts vtt, ttml, srv3, srv2, srv1\n", "tr vtt, ttml, srv3, srv2, srv1\n", "tk vtt, ttml, srv3, srv2, srv1\n", "uk vtt, ttml, srv3, srv2, srv1\n", "und vtt, ttml, srv3, srv2, srv1\n", "ur vtt, ttml, srv3, srv2, srv1\n", "ug vtt, ttml, srv3, srv2, srv1\n", "uz vtt, ttml, srv3, srv2, srv1\n", "vi vtt, ttml, srv3, srv2, srv1\n", "cy vtt, ttml, srv3, srv2, srv1\n", "fy vtt, ttml, srv3, srv2, srv1\n", "xh vtt, ttml, srv3, srv2, srv1\n", "yi vtt, ttml, srv3, srv2, srv1\n", "yo vtt, ttml, srv3, srv2, srv1\n", "zu vtt, ttml, srv3, srv2, srv1\n", "Available subtitles for zuBEwXo69nA:\n", "Language formats\n", "en vtt, ttml, srv3, srv2, srv1\n" ] } ], "source": [ "# youtube_url = \"http://www.youtube.com/watch?v=oktMBFQNeQM\" # no subtitles\n", "# youtube_url = \"http://www.youtube.com/watch?v=mBxNwuxFYxo\" # automated and en-US\n", "# youtube_url = \"http://www.youtube.com/watch?v=zuBEwXo69nA\" # automated and en\n", "# youtube_url = \"http://www.youtube.com/watch?v=Fpu5a0Bl8eY\" # en/de\n", "# youtube_url = \"http://www.youtube.com/watch?v=Fpu5a0Bl8eY\"\n", "\n", "ydl_opts = {\n", " \"listsubtitles\": True,\n", "}\n", "with youtube_dl.YoutubeDL(ydl_opts) as ydl:\n", " out = ydl.download([youtube_url])" ] }, { "cell_type": "code", "execution_count": null, "id": "93f06848", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5556e067", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "259fc4ea", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "8ce92069", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 1, "id": "b995a71f", "metadata": {}, "outputs": [], "source": [ "import youtube_dl\n", "import cloudscraper\n", "from youtube_search import YoutubeSearch\n", "\n", "from suno_utils.audio import Audio" ] }, { "cell_type": "code", "execution_count": null, "id": "6df0f920", "metadata": {}, "outputs": [], "source": [ "youtube_url = \"http://www.youtube.com/watch?v=72UO0v5ESUo\"\n", "\n", "ydl_opts = {\n", " \"format\": \"worstaudio/worst\",\n", " \"outtmpl\": \"test.webm\",\n", "}\n", "with youtube_dl.YoutubeDL(ydl_opts) as ydl:\n", " out = ydl.download([youtube_url])\n", "assert(out == 0)" ] }, { "cell_type": "code", "execution_count": null, "id": "310412ee", "metadata": {}, "outputs": [], "source": [ "videos = YoutubeSearch('Hurt Johnny Cash', max_results=10).to_dict()\n", "video_url = \"https://www.youtube.com/\" + videos[0][\"url_suffix\"]" ] }, { "cell_type": "code", "execution_count": null, "id": "70fc4e69", "metadata": {}, "outputs": [], "source": [ "YoutubeSearch()" ] }, { "cell_type": "code", "execution_count": 35, "id": "10029c96", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fe5a9912", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 38, "id": "9fc90653", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[{'id': 'mBxNwuxFYxo',\n", " 'title': 'Kellyanne Conway: This is a disaster for Biden',\n", " 'long_desc': None,\n", " 'channel': 'Fox Business',\n", " 'duration': '9:02',\n", " 'views': '282,626 views',\n", " 'publish_time': '21 hours ago',\n", " 'url_suffix': '/watch?v=mBxNwuxFYxo'},\n", " {'id': 'XgsjaDe5Qd8',\n", " 'title': 'Global Quinoa Prices Crashed. Peruvian Farmers Still Harvest 100,000 Tons A Year | Big Business',\n", " 'long_desc': None,\n", " 'channel': 'Business Insider',\n", " 'duration': '10:07',\n", " 'views': '129,942 views',\n", " 'publish_time': '2 days ago',\n", " 'url_suffix': '/watch?v=XgsjaDe5Qd8'},\n", " {'id': 'GPhMtjFB3ZU',\n", " 'title': 'Small business wages, job growth are moderating, says Paychex CEO Marty Mucci',\n", " 'long_desc': None,\n", " 'channel': 'CNBC Television',\n", " 'duration': '3:04',\n", " 'views': '2,057 views',\n", " 'publish_time': '8 hours ago',\n", " 'url_suffix': '/watch?v=GPhMtjFB3ZU'},\n", " {'id': 'Fk8316DMF_g',\n", " 'title': 'Stuart Varney challenges GOP congresswoman over deporting migrants',\n", " 'long_desc': None,\n", " 'channel': 'Fox Business',\n", " 'duration': '3:17',\n", " 'views': '12,043 views',\n", " 'publish_time': '3 hours ago',\n", " 'url_suffix': '/watch?v=Fk8316DMF_g'},\n", " {'id': 'V2Pkz8KYL4s',\n", " 'title': 'The Bloodline is united in taking care of business: The SmackDown LowDown, Oct. 1, 2022',\n", " 'long_desc': None,\n", " 'channel': 'WWE',\n", " 'duration': '4:55',\n", " 'views': '517,448 views',\n", " 'publish_time': '3 days ago',\n", " 'url_suffix': '/watch?v=V2Pkz8KYL4s'},\n", " {'id': 'mtt8HxEFzqE',\n", " 'title': 'Expert predicts when you can expect the market to rebound',\n", " 'long_desc': None,\n", " 'channel': 'Fox Business',\n", " 'duration': '2:00',\n", " 'views': '14,582 views',\n", " 'publish_time': '1 day ago',\n", " 'url_suffix': '/watch?v=mtt8HxEFzqE'},\n", " {'id': 'nCg3ufihKyU',\n", " 'title': 'Tiësto - The Business (Official Music Video)',\n", " 'long_desc': None,\n", " 'channel': 'Tiësto',\n", " 'duration': '3:12',\n", " 'views': '215,310,174 views',\n", " 'publish_time': '2 years ago',\n", " 'url_suffix': '/watch?v=nCg3ufihKyU'},\n", " {'id': 'Ox50bB2UmTE',\n", " 'title': \"Larry Kudlow: This has become the left's biggest weapon\",\n", " 'long_desc': None,\n", " 'channel': 'Fox Business',\n", " 'duration': '4:38',\n", " 'views': '79,403 views',\n", " 'publish_time': '22 hours ago',\n", " 'url_suffix': '/watch?v=Ox50bB2UmTE'},\n", " {'id': 'qBZ1RlkpTZM',\n", " 'title': \"California can't keep the lights on as it is: Stephen Schork\",\n", " 'long_desc': None,\n", " 'channel': 'Fox Business',\n", " 'duration': '6:07',\n", " 'views': '87,836 views',\n", " 'publish_time': '1 day ago',\n", " 'url_suffix': '/watch?v=qBZ1RlkpTZM'},\n", " {'id': 'EGDWimzjLIU',\n", " 'title': \"Kudlow: OPEC cutting oil production would be a 'very stupid' thing to do\",\n", " 'long_desc': None,\n", " 'channel': 'Fox Business',\n", " 'duration': '4:11',\n", " 'views': '15,686 views',\n", " 'publish_time': '4 hours ago',\n", " 'url_suffix': '/watch?v=EGDWimzjLIU'},\n", " {'id': 'RjYfGX1fMi8',\n", " 'title': \"We think Amazon's business is becoming more efficient, says Citi's Ron Josey\",\n", " 'long_desc': None,\n", " 'channel': 'CNBC Television',\n", " 'duration': '3:02',\n", " 'views': '796 views',\n", " 'publish_time': '4 hours ago',\n", " 'url_suffix': '/watch?v=RjYfGX1fMi8'},\n", " {'id': 'PvNfHJcJeKg',\n", " 'title': 'Gen. Keane: China will not let this situation get ‘out of hand’',\n", " 'long_desc': None,\n", " 'channel': 'Fox Business',\n", " 'duration': '7:39',\n", " 'views': '88,820 views',\n", " 'publish_time': '5 hours ago',\n", " 'url_suffix': '/watch?v=PvNfHJcJeKg'},\n", " {'id': 'vNAZSwTvRdA',\n", " 'title': '4 family members, including infant, kidnapped from CA business, deputies say',\n", " 'long_desc': None,\n", " 'channel': 'ABC7',\n", " 'duration': '2:40',\n", " 'views': '130,166 views',\n", " 'publish_time': '13 hours ago',\n", " 'url_suffix': '/watch?v=vNAZSwTvRdA'},\n", " {'id': 'zlcs-FOgxNw',\n", " 'title': 'Kennedy hits the streets to ask: Is college worth it?',\n", " 'long_desc': None,\n", " 'channel': 'Fox Business',\n", " 'duration': '7:47',\n", " 'views': '41,756 views',\n", " 'publish_time': '15 hours ago',\n", " 'url_suffix': '/watch?v=zlcs-FOgxNw'},\n", " {'id': 'OODfq5G9slo',\n", " 'title': \"US 'could slide' itself into a war that nobody wants, McFarland says\",\n", " 'long_desc': None,\n", " 'channel': 'Fox Business',\n", " 'duration': '3:24',\n", " 'views': '42,751 views',\n", " 'publish_time': '1 day ago',\n", " 'url_suffix': '/watch?v=OODfq5G9slo'},\n", " {'id': 'oXqhNq3AmLw',\n", " 'title': \"Biden has been 'bad' for our country in 'so many ways': GOP lawmaker\",\n", " 'long_desc': None,\n", " 'channel': 'Fox Business',\n", " 'duration': '6:22',\n", " 'views': '56,037 views',\n", " 'publish_time': '8 hours ago',\n", " 'url_suffix': '/watch?v=oXqhNq3AmLw'},\n", " {'id': 'vMYk3IQrYi4',\n", " 'title': 'We’re seeing buyers look at alternative financing: Kirsten Jordan',\n", " 'long_desc': None,\n", " 'channel': 'Fox Business',\n", " 'duration': '4:56',\n", " 'views': '7,558 views',\n", " 'publish_time': '10 hours ago',\n", " 'url_suffix': '/watch?v=vMYk3IQrYi4'},\n", " {'id': 'l2fehkZbipU',\n", " 'title': \"VP Harris' 'equity' comment regarding hurricane aid is a 'controversy': Varney\",\n", " 'long_desc': None,\n", " 'channel': 'Fox Business',\n", " 'duration': '1:11',\n", " 'views': '15,508 views',\n", " 'publish_time': '1 day ago',\n", " 'url_suffix': '/watch?v=l2fehkZbipU'},\n", " {'id': 'uRmedeWCRGI',\n", " 'title': \"Forbes: Americans want to see real policies to reduce 'bloated' spending\",\n", " 'long_desc': None,\n", " 'channel': 'Fox Business',\n", " 'duration': '3:05',\n", " 'views': '13,684 views',\n", " 'publish_time': '1 day ago',\n", " 'url_suffix': '/watch?v=uRmedeWCRGI'},\n", " {'id': 'geYRBajNCfY',\n", " 'title': 'Target 7 gets results: Albuquerque business owner receives violation notice for homeless on sidewalk',\n", " 'long_desc': None,\n", " 'channel': 'KOAT',\n", " 'duration': '3:09',\n", " 'views': '5,309 views',\n", " 'publish_time': '17 hours ago',\n", " 'url_suffix': '/watch?v=geYRBajNCfY'}]" ] }, "execution_count": 38, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results" ] }, { "cell_type": "code", "execution_count": 36, "id": "752ce5e2", "metadata": {}, "outputs": [], "source": [ "_, results = _parse_youtube_search_results(out.text)" ] }, { "cell_type": "code", "execution_count": 49, "id": "5a08b024", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'Ep0DEghidXNpbmVzcxqQA0VnSW9BVWdVZ2dFTGJVSjRUbmQxZUVaWmVHLUNBUXRZWjNOcVlVUmxOVkZrT0lJQkMwZFFhRTEwYWtaQ00xcFZnZ0VMUm1zNE16RTJSRTFHWDJlQ0FRdFdNbEJyZWpoTFdVdzBjNElCQzIxMGREaEllRVZHZW5GRmdnRUxia05uTTNWbWFXaExlVldDQVF0UGVEVXdZa0l5VlcxVVJZSUJDM0ZDV2pGU2JHdHdWRnBOZ2dFTFJVZEVWMmx0ZW1wTVNWV0NBUXRTYWxsbVIxZ3haazFwT0lJQkMxQjJUbVpJU21OS1pVdG5nZ0VMZGs1QldsTjNWSFpTWkVHQ0FRdDZiR056TFVaUFozaE9kNElCQzA5UFJHWnhOVWM1YzJ4dmdnRUxiMWh4YUU1eE0wRnRUSGVDQVF0MlRWbHJNMGxSY2xscE5JSUJDMnd5Wm1Wb2ExcGlhWEJWZ2dFTGRWSnRaV1JsVjBOU1IwbUNBUXRuWlZsU1FtRnFUa05tV2JJQkJnb0VDQlFRQWclM0QlM0QYgeDoGCILc2VhcmNoLWZlZWQ%3D'" ] }, "execution_count": 49, "metadata": {}, "output_type": "execute_result" } ], "source": [ "continuation_token" ] }, { "cell_type": "code", "execution_count": null, "id": "2db61afd", "metadata": {}, "outputs": [], "source": [ "https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false" ] }, { "cell_type": "code", "execution_count": null, "id": "ab6bbbcf", "metadata": {}, "outputs": [], "source": [ "# client context\n", "# # {\n", "# # \"hl\": \"en\",\n", "# # \"gl\": \"US\",\n", "# # }" ] }, { "cell_type": "code", "execution_count": null, "id": "7917f1c0", "metadata": {}, "outputs": [], "source": [ "/watch?v=mBxNwuxFYxo" ] }, { "cell_type": "code", "execution_count": 88, "id": "62344f8e", "metadata": {}, "outputs": [], "source": [ "import os" ] }, { "cell_type": "code", "execution_count": 89, "id": "f742dd4e", "metadata": {}, "outputs": [], "source": [ "with redirect_stdout(open(os.devnull, 'w')):\n", " print(\"blabla\")" ] }, { "cell_type": "code", "execution_count": 83, "id": "4d55c8b9", "metadata": {}, "outputs": [], "source": [ "from contextlib import redirect_stdout" ] }, { "cell_type": "code", "execution_count": 90, "id": "556da7ea", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "rm: cannot remove 'test.en.vtt': No such file or directory\r\n" ] } ], "source": [ "!rm test.en.vtt" ] }, { "cell_type": "code", "execution_count": 91, "id": "51449515", "metadata": {}, "outputs": [], "source": [ "youtube_url = \"http://www.youtube.com/watch?v=zuBEwXo69nA\"\n", "\n", "ydl_opts = {\n", " \"skip_download\": True,\n", " \"writesubtitles\": True,\n", " \"outtmpl\": \"/home/georg/notebooks/datasets/youtube/test\",\n", " \"allsubtitles\": True,\n", " \"subtitlesformat\": \"best\",\n", "# \"listsubtitles\": True,\n", "# \"subtitleslangs\": [\"en\", \"en-US\"],\n", "}\n", "with redirect_stdout(open(os.devnull, 'w')):\n", " with youtube_dl.YoutubeDL(ydl_opts) as ydl:\n", " out = ydl.download([youtube_url])" ] }, { "cell_type": "code", "execution_count": 92, "id": "32b204a5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "test.en.vtt Untitled.ipynb\r\n" ] } ], "source": [ "!ls" ] }, { "cell_type": "code", "execution_count": null, "id": "633e210c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "925a6c65", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "de48c3e0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0e2e45d8", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }