{ "cells": [ { "cell_type": "code", "execution_count": 2, "id": "fde1ffe0", "metadata": {}, "outputs": [], "source": [ "import json" ] }, { "cell_type": "code", "execution_count": 3, "id": "4e5afd19", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0\n" ] }, { "data": { "text/plain": [ "(None, 'failures')" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "failure_metas = []\n", "with open(\"/data/suno/data/harvest/youtube_lg/subtitles.jsonl\") as f:\n", " for line in f:\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " m = json.loads(line)\n", " if not m[\"success\"]:\n", " failure_metas.append(m)\n", "print(len(failure_metas)), \"failures\"" ] }, { "cell_type": "code", "execution_count": null, "id": "55c2139d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "891c9b6d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "eb14eb12", "metadata": {}, "source": [ "## Try geo diversity in google" ] }, { "cell_type": "code", "execution_count": 9, "id": "d208414a", "metadata": {}, "outputs": [], "source": [ "import requests\n", "import json\n", "import re\n", "import time\n", "import random\n", "\n", "def _parse_youtube_search_results(html_str):\n", " start_idx = html_str.index(\"ytInitialData\") + len(\"ytInitialData\") + 3\n", " end_idx = html_str.index(\"};\", start_idx) + 1\n", " json_str = html_str[start_idx:end_idx]\n", " page_info = json.loads(json_str)\n", " primary_content = page_info[\"contents\"][\"twoColumnSearchResultsRenderer\"].get(\"primaryContents\")\n", " if primary_content is not None:\n", " content_blocks = primary_content[\"sectionListRenderer\"][\"contents\"]\n", " else:\n", " coninuation_content = page_info[\"onResponseReceivedCommands\"][0][\"appendContinuationItemsAction\"]\n", " content_blocks = coninuation_content[\"continuationItems\"]\n", " results = []\n", " for contents in content_blocks[:-1]:\n", " for video in contents[\"itemSectionRenderer\"][\"contents\"]:\n", " res = {}\n", " if \"videoRenderer\" in video.keys():\n", " video_data = video.get(\"videoRenderer\", {})\n", " res[\"id\"] = video_data.get(\"videoId\", None)\n", " res[\"title\"] = video_data.get(\"title\", {}).get(\"runs\", [[{}]])[0].get(\"text\", None)\n", " res[\"long_desc\"] = video_data.get(\"descriptionSnippet\", {}).get(\"runs\", [{}])[0].get(\"text\", None)\n", " res[\"channel\"] = video_data.get(\"longBylineText\", {}).get(\"runs\", [[{}]])[0].get(\"text\", None)\n", " res[\"duration\"] = video_data.get(\"lengthText\", {}).get(\"simpleText\", None)\n", " res[\"views\"] = video_data.get(\"viewCountText\", {}).get(\"simpleText\", None)\n", " res[\"publish_time\"] = video_data.get(\"publishedTimeText\", {}).get(\"simpleText\", None)\n", " res[\"url_suffix\"] = video_data.get(\n", " \"navigationEndpoint\", {}\n", " ).get(\"commandMetadata\", {}).get(\"webCommandMetadata\", {}).get(\"url\", None)\n", " results.append(res)\n", " if \"continuationItemRenderer\" not in content_blocks[-1]:\n", " continuation_token = None\n", " else:\n", " continuation_endpoint = content_blocks[-1][\"continuationItemRenderer\"][\"continuationEndpoint\"]\n", " continuation_token = continuation_endpoint[\"continuationCommand\"][\"token\"]\n", " return continuation_token, results\n", "\n", "def get_youtube_search_results(query_term, add_str=\"\", max_n_pages=None):\n", " # NOTE: this returns max ~500 results\n", " base_url = \"https://www.youtube.com/results?\"\n", " all_results = []\n", " seen_ids = set()\n", " n_pages = 1\n", " while True:\n", " if n_pages == 1:\n", " harvest_url = f\"{base_url}search_query={query_term}&sp=EgIoAQ%253D%253D\" + add_str\n", " else:\n", " harvest_url = f\"{base_url}continuation={continuation_token}&ctoken={continuation_token}\"\n", " out = requests.get(harvest_url)\n", " assert(out.ok)\n", " continuation_token, results = _parse_youtube_search_results(out.text)\n", " for m in results:\n", " video_id = m[\"id\"]\n", " if video_id in seen_ids:\n", " continue\n", " seen_ids.add(video_id)\n", " all_results.append(m)\n", " if continuation_token is None:\n", " break\n", " if max_n_pages is not None and n_pages >= max_n_pages:\n", " break\n", " n_pages += 1\n", " _courtesy_sleep(avg_sleep_dur_s=0.3)\n", " return all_results" ] }, { "cell_type": "code", "execution_count": 5, "id": "296568f4", "metadata": {}, "outputs": [], "source": [ "query_term = \"toast\"" ] }, { "cell_type": "code", "execution_count": 14, "id": "7510d5a5", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[\"A Beginner's Guide To Buying Great Coffee\",\n", " 'Coffee Brewing Methods: French Press vs Pour Over vs AeroPress and more!',\n", " 'Coffee and Crime Time: Eliza Fletcher',\n", " 'Koffee - Pull Up (Official Video)',\n", " 'Koffee - Pressure (Remix) [Official Video] ft. Buju Banton',\n", " 'Coffee and Crime Time: What Happened To Dylan Rounds?',\n", " 'Koffee - Toast (Official Video)',\n", " '\"Coffee\" | Jack Stauber Animation',\n", " 'The Most Wonderfully Absurd Coffee Brewer Ever Made',\n", " 'Modern Marvels: How Coffee is Made (S12, E51) | Full Episode | History',\n", " 'What Coffee Looks Like Around The World | Food Insider',\n", " 'Inside the Global Fight to Save Coffee',\n", " 'All Espresso Drinks Explained: Cappuccino vs Latte vs Flat White and more!',\n", " '7\\xa0Facts About Coffee You Probably Didn’t Know',\n", " '$1 Coffee Vs. $914 Coffee • Japan',\n", " \"What you didn't know about coffee: Asher Yaron at TEDxUbud\",\n", " 'How He Started a Coffee Business (With $1,800)',\n", " 'Drip Coffee Makers — super simple, super cheap',\n", " 'The Bizarre And Surprising Coffee Of The Nespresso Vertuo',\n", " 'The Ugly Truth About Coffee’s Effects On Your Body']" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out = get_youtube_search_results(\"coffee\", max_n_pages=1)\n", "[e[\"title\"] for e in out]" ] }, { "cell_type": "code", "execution_count": 2, "id": "7b5841aa", "metadata": {}, "outputs": [], "source": [ "import requests\n", "from bs4 import BeautifulSoup\n", "import re" ] }, { "cell_type": "code", "execution_count": 3, "id": "ea4d533a", "metadata": {}, "outputs": [], "source": [ "out = requests.get(\"https://en.wikipedia.org/wiki/List_of_dialects_of_English\")\n", "soup = BeautifulSoup(out.text)" ] }, { "cell_type": "code", "execution_count": 10, "id": "5af66fa0", "metadata": {}, "outputs": [], "source": [ "lines = soup.findAll(\"li\")[78:255]\n", "reduced_html = \"\".join([str(l) for l in lines])\n", "reduced_soup = BeautifulSoup(reduced_html)" ] }, { "cell_type": "code", "execution_count": null, "id": "5d0066a9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 30, "id": "22f8d774", "metadata": {}, "outputs": [], "source": [ "ENGLISH_DIALECTS = [\n", " 'Abercraf', 'Aboriginal English in Canada', 'African American English', 'African-American Vernacular English', \n", " 'Afro-Seminole Creole', 'Angloromani', 'Arablish', 'Atlantic Canadian English', 'Bahamian Creole', \n", " 'Bahamian English', 'Bajan English', 'Baltimore English', 'Bangladeshi English', 'Barrovian', \n", " 'Barrow-in-Furness', 'Bay Islands English', 'Belizean English', 'Bermudian English', 'Birmingham', \n", " 'Black Country', 'Bolton', 'Boontling', 'Boston', 'Bristolian', 'British English', 'British Indian Empire', \n", " 'Brummie', 'Brunei English', 'Bungi', 'Burmese English', 'Butler English', 'Cajun Vernacular English', \n", " 'California English', 'Cameroonian English', 'Cape Flats English', 'Cardiff', 'Caribbean English', \n", " 'Cayman Islands English', 'Channel Island English', 'Cheshire', 'Chicago', 'Chicano English', \n", " 'Chinese Pidgin English', 'Chinglish', 'Cleveland', 'Cockney', 'Cornwall', 'County Durham', 'County Wexford', \n", " 'Coventry', 'Cumbria', 'Cumbrian', 'Danish English', 'Detroit', 'Dorset', 'Dublin', 'Dutch English', \n", " 'East Anglian', 'East Midlands', 'Eastern New England English', 'Ebonics', 'English in Japan', 'Engrish', \n", " 'Estuary', 'Euro English', 'Falkland Islands English', 'Fingal', 'Fingallian', 'Finnish English', \n", " 'Forth and Bargy', 'Gambian English', 'General American', 'Geordie', 'German English', \n", " 'Ghanaian English', 'Gibraltarian English', 'Glasgow', 'Gower', 'Great Lakes region', \n", " 'Great Northern Coalfield', 'Greater Toronto English', 'Gullah language', 'Guyanese English', \n", " 'Hampshire', 'Hawaiian Pidgin', 'Hiberno-English', 'Highland English', 'Hinglish', 'Hoi Toider English', \n", " 'Home Counties', 'Hong Kong English', 'Indian English', 'Inland Northern English', 'Jamaican English', \n", " 'Jamaican Patois', 'Janner', 'Kanglish', 'Kenyan English', 'Konglish', 'Korean English', 'Lancashire', \n", " 'Lancastrian', 'Leinster', 'Liberian English', 'Lincolnshire', 'London', 'Lower Peninsula of Michigan', \n", " 'Lunenburg English', 'Mackem', 'Maine English', 'Malawian English', 'Malaysian English', 'Maltese English', \n", " 'Manchester', 'Mancunian', 'Manglish', 'Manx English', 'Merico language', 'Merseyside', 'Metis', \n", " 'Metropolitan New York English', 'Miami English', 'Mid-Atlantic or Transatlantic English', \n", " 'Middle Eastern English', 'Middle English', 'Midland American English', 'Milwaukee', 'Multicultural London', \n", " 'Namlish', 'Nepali English', 'New England English', 'New Orleans English', 'New York Latino English', \n", " 'Newfoundland English', 'Nigerian English', 'Norfolk', 'North-Central (Upper Midwestern) English', 'Northern', \n", " 'Northern American English', 'Northumberland', 'Northumbrian', 'Norwegian English', \n", " 'Older Southern American English', 'Ottawa Valley English', 'Pacific Northwest English', 'Pakistani English', \n", " 'Pennsylvania Dutch English', 'Philadelphia English', 'Philippine English', 'Pitmatic', 'Port Talbot', \n", " 'Potteries', 'Quebec English', 'Received Pronunciation', 'Regional and local Indian English', \n", " 'Rhode Island English', 'Saban English', 'Saint Helena', 'Scottish English', 'Scouse', 'Sierra Leonean English', \n", " 'Singapore English', 'Singlish', 'Smoggie', 'South African English', 'South Atlantic English', \n", " 'South-West Ireland', 'Southern', 'Southern American English', 'Southern Appalachian English', \n", " 'Sri Lankan English', 'Staffordshire', 'Canadian English', 'Suffolk', 'Sunderland', \n", " 'Sussex', 'Swedish English', 'Tanglish', 'Teesside', 'Tenglish', 'Texan English', 'Trinidadian English', \n", " 'Tristan da Cunha', 'Tyneside', 'Ulster', 'Ulster Scots', 'Vincentian Creole', 'Welsh English', \n", " 'West Country', 'West Midlands', 'Western American English', 'Western New England English', 'Western New York', \n", " 'Western Pennsylvania (Pittsburgh) English', 'Yeshiva English', 'Yorkshire',\n", "]" ] }, { "cell_type": "code", "execution_count": null, "id": "7e2ffa71", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "2013d5cd", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" } }, "nbformat": 4, "nbformat_minor": 5 }