{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "ce10c95c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 1, "id": "e696d5ee", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "\n", "%matplotlib inline\n", "\n", "import os\n", "import json\n", "import requests\n", "import urllib3\n", "\n", "from bs4 import BeautifulSoup, SoupStrainer\n", "from pytube import YouTube\n", "from pytube.exceptions import VideoUnavailable\n", "from vimeo_downloader import Vimeo\n", "import time\n", "\n", "import datetime\n", "import tqdm" ] }, { "cell_type": "code", "execution_count": 2, "id": "8a2f6481", "metadata": {}, "outputs": [], "source": [ "HEADERS = {\n", " \"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36\"\n", "}" ] }, { "cell_type": "code", "execution_count": null, "id": "cab3d85d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 3, "id": "1b8cf4f0", "metadata": {}, "outputs": [], "source": [ "def get_video_links(vid_resp):\n", " vimeo_links = []\n", " youtube_links = []\n", " if not vid_resp.ok:\n", " return \n", " for link in BeautifulSoup(vid_resp.text, parse_only=SoupStrainer('a')):\n", " if link.has_attr('href'):\n", " href = link[\"href\"]\n", " if \"youtube\" in href:\n", " youtube_links.append(href)\n", " elif \"vimeo\" in href:\n", " vimeo_links.append(href)\n", " return {\"youtube\": youtube_links, \"vimeo\": vimeo_links}\n", "\n", "\n", "def get_more_video_links(vid_resp):\n", " links = []\n", " for link in BeautifulSoup(vid_resp.text, parse_only=SoupStrainer('a')):\n", " if link.has_attr('href'):\n", " href = link[\"href\"]\n", " if href.startswith(\"?page=\"):\n", " links.append(href)\n", " return links" ] }, { "cell_type": "code", "execution_count": null, "id": "0a899743", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "822281c2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 26, "id": "6e9fbedb", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DateFrom=2017-11-07&Criteria.DateTo=2020-11-20&Criteria.ParliamentNumber=52&Criteria.PageNumber=1',\n", " 'https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DateFrom=2017-11-07&Criteria.DateTo=2020-11-20&Criteria.ParliamentNumber=52&Criteria.PageNumber=2',\n", " 'https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DateFrom=2017-11-07&Criteria.DateTo=2020-11-20&Criteria.ParliamentNumber=52&Criteria.PageNumber=3',\n", " 'https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DateFrom=2017-11-07&Criteria.DateTo=2020-11-20&Criteria.ParliamentNumber=52&Criteria.PageNumber=4',\n", " 'https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DateFrom=2017-11-07&Criteria.DateTo=2020-11-20&Criteria.ParliamentNumber=52&Criteria.PageNumber=5',\n", " 'https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DateFrom=2017-11-07&Criteria.DateTo=2020-11-20&Criteria.ParliamentNumber=52&Criteria.PageNumber=6',\n", " 'https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DateFrom=2017-11-07&Criteria.DateTo=2020-11-20&Criteria.ParliamentNumber=52&Criteria.PageNumber=7',\n", " 'https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DateFrom=2017-11-07&Criteria.DateTo=2020-11-20&Criteria.ParliamentNumber=52&Criteria.PageNumber=8']" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\n", "\n" ] }, { "cell_type": "code", "execution_count": 28, "id": "90870c12", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "found a total of 15 pages with preceedings of parliament\n" ] } ], "source": [ "parl_53_urls = [\n", " \"https://www.parliament.nz/en/pb/hansard-debates/rhr/\",\n", " \"https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DefaultParliamentNumber=53&Criteria.ParliamenStartDate=2020-11-24&Criteria.PageNumber=2\",\n", " \"https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DefaultParliamentNumber=53&Criteria.ParliamenStartDate=2020-11-24&Criteria.PageNumber=3\",\n", " \"https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DefaultParliamentNumber=53&Criteria.ParliamenStartDate=2020-11-24&Criteria.PageNumber=4\",\n", " \"https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DefaultParliamentNumber=53&Criteria.ParliamenStartDate=2020-11-24&Criteria.PageNumber=5\",\n", " \"https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DefaultParliamentNumber=53&Criteria.ParliamenStartDate=2020-11-24&Criteria.PageNumber=6\",\n", " \"https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DefaultParliamentNumber=53&Criteria.ParliamenStartDate=2020-11-24&Criteria.PageNumber=7\",\n", " # just need to know there are only 7 pages\n", "]\n", "\n", "base_url_parl_52 = (\n", " \"https://www.parliament.nz/en/pb/hansard-debates/rhr/\"\n", " + \"?Criteria.page=HansardReports&Criteria.DateFrom=2017-11-07\"\n", " + \"&Criteria.DateTo=2020-11-20&Criteria.ParliamentNumber=52\"\n", ")\n", "# just need to look up that vimeo stops after page 8\n", "parl_52_urls = [base_url_parl_52 + f\"&Criteria.PageNumber={i}\" for i in range(1, 9)]\n", "\n", "top_level_urls = parl_52_urls + parl_53_urls\n", "\n", "print(f\"found a total of {len(top_level_urls)} pages with preceedings of parliament\")" ] }, { "cell_type": "code", "execution_count": 29, "id": "dbbe1145", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "<>:8: SyntaxWarning: assertion is always true, perhaps remove parentheses?\n", "<>:8: SyntaxWarning: assertion is always true, perhaps remove parentheses?\n", "/tmp/ipykernel_111777/2304494439.py:8: SyntaxWarning: assertion is always true, perhaps remove parentheses?\n", " assert(all(successes), \"poop\")\n" ] } ], "source": [ "top_level_resps = []\n", "successes = []\n", "for tlu in top_level_urls:\n", " time.sleep(1)\n", " top_level_resp = requests.get(tlu, headers=HEADERS)\n", " successes.append(top_level_resp.ok)\n", " top_level_resps.append(top_level_resp)\n", "assert all(successes), \"poo\"\n", "assert len(successes) == len(top_level_urls), \"poop\"" ] }, { "cell_type": "code", "execution_count": 35, "id": "496053bc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d601aa81", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 36, "id": "0ebb44cc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " found 282 dates to process\n" ] }, { "data": { "text/plain": [ "['20200902',\n", " '20200901',\n", " '20200826',\n", " '20200825',\n", " '20200819',\n", " '20200818',\n", " '20200806',\n", " '20200804',\n", " '20200804',\n", " '20200730']" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "links = []\n", "for top_level_resp in top_level_resps:\n", " for link in BeautifulSoup(top_level_resp.text, parse_only=SoupStrainer('a')):\n", " if link.has_attr('href'):\n", " href = link[\"href\"]\n", " if href.startswith(\"/en/pb/hansard-debates/rhr/combined/HansD_\"):\n", " links.append(href)\n", "\n", "date_strs = [l.replace(\"/en/pb/hansard-debates/rhr/combined/HansD_\", \"\") for l in links]\n", "date_strs = [d.split(\"_\")[0] for d in date_strs]\n", "print(f\" found {len(date_strs)} dates to process\")\n", "date_strs[:10]" ] }, { "cell_type": "code", "execution_count": null, "id": "dfd828f9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 38, "id": "2cdb118d", "metadata": {}, "outputs": [], "source": [ "BASE_URL = \"https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_\"\n", "\n", "OUTPUT_DIR = \"/mnt/data-ssd-1/data/nz-parl\"" ] }, { "cell_type": "code", "execution_count": null, "id": "1af50cb4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 39, "id": "99166467", "metadata": {}, "outputs": [], "source": [ "# load manifest or initialize new\n", "#manifest = []\n", "with open(os.path.join(OUTPUT_DIR, \"manifest_bk.json\"), \"r\") as fi:\n", " manifest = json.load(fi)" ] }, { "cell_type": "code", "execution_count": 43, "id": "c7b8257f", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "257" ] }, "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(manifest)" ] }, { "cell_type": "code", "execution_count": 42, "id": "0d68a6cc", "metadata": { "scrolled": false }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "c9d0aebf87da4aedb8474cabd3589d5b", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/282 [00:00" ] }, "execution_count": 163, "metadata": {}, "output_type": "execute_result" } ], "source": [ "video_url = BASE_URL + \"/tab/video\"\n", "vid_resp = requests.get(video_url, headers=HEADERS)\n", "vid_resp" ] }, { "cell_type": "code", "execution_count": null, "id": "3f24ea2c", "metadata": { "scrolled": false }, "outputs": [], "source": [ "video_links = get_video_links(vid_resp)\n", "if video_links[\"youtube\"] and video_links[\"vimeo\"]:\n", " print(\"found both???\")\n", "pages_to_follow = get_more_video_links(vid_resp)\n", "for page in pages_to_follow:\n", " print(page)\n", " page_resp = requests.get(video_url + page, headers=HEADERS)\n", " if page_resp.ok:\n", " tmp = get_video_links(page_resp)\n", " video_links[\"youtube\"].extend(tmp[\"youtube\"])\n", " video_links[\"vimeo\"].extend(tmp[\"vimeo\"])\n", "video_links" ] }, { "cell_type": "code", "execution_count": null, "id": "a906c7a8", "metadata": {}, "outputs": [], "source": [ "metadata = {\"url\": orig_url, \"video_links\": video_links}" ] }, { "cell_type": "code", "execution_count": null, "id": "07fc71b9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7ca3c6f4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3f841b9f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "2f7916ca", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 153, "id": "0f871a50", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20170803_20170803/tab/video?page=5'" ] }, "execution_count": 153, "metadata": {}, "output_type": "execute_result" } ], "source": [ "video_url + more_video_links[3]" ] }, { "cell_type": "code", "execution_count": 142, "id": "a4d200fe", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "fetching https://www.youtube.com/watch?v=d_tnT-16Guw\n", "found file /Users/mikey/data/nz-parl/20170803/d_tnT-16Guw.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=VpwS8_idNeE\n", "found file /Users/mikey/data/nz-parl/20170803/VpwS8_idNeE.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=VpwS8_idNeE\n", "found file /Users/mikey/data/nz-parl/20170803/VpwS8_idNeE.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=F9rgZ1NMp3U\n", "found file /Users/mikey/data/nz-parl/20170803/F9rgZ1NMp3U.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=F9rgZ1NMp3U\n", "found file /Users/mikey/data/nz-parl/20170803/F9rgZ1NMp3U.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=-u5F-DP1R3c\n", "found file /Users/mikey/data/nz-parl/20170803/-u5F-DP1R3c.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=-u5F-DP1R3c\n", "found file /Users/mikey/data/nz-parl/20170803/-u5F-DP1R3c.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=QriIH5RUO0Y\n", "found file /Users/mikey/data/nz-parl/20170803/QriIH5RUO0Y.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=QriIH5RUO0Y\n", "found file /Users/mikey/data/nz-parl/20170803/QriIH5RUO0Y.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=cHhtiH_m6a0\n", "cant't find audio for {video_key}\n", "fetching https://www.youtube.com/watch?v=cHhtiH_m6a0\n", "cant't find audio for {video_key}\n", "fetching https://www.youtube.com/watch?v=VRPck9qf7Io\n", "found file /Users/mikey/data/nz-parl/20170803/VRPck9qf7Io.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=VRPck9qf7Io\n", "found file /Users/mikey/data/nz-parl/20170803/VRPck9qf7Io.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=1epY_NZp8SA\n", "found file /Users/mikey/data/nz-parl/20170803/1epY_NZp8SA.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=1epY_NZp8SA\n", "found file /Users/mikey/data/nz-parl/20170803/1epY_NZp8SA.mp4, skipping...\n" ] } ], "source": [ "output_dir = \"/Users/mikey/data/nz-parl/20170803/\"\n", "\n", "results = []\n", "\n", "for link in video_links[\"youtube\"]:\n", " print(f\"fetching {link}\")\n", " tmp = download_youtube(link, output_dir)\n", " time.sleep(1)\n", " results.append(tmp)\n", " with open(os.path.join(output_dir, \"metadata.json\"), \"w\") as fi:\n", " json.dump(results, fi)" ] }, { "cell_type": "code", "execution_count": 144, "id": "b23ea4f7", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 144, "metadata": {}, "output_type": "execute_result" } ], "source": [ "resp" ] }, { "cell_type": "code", "execution_count": null, "id": "008e9d10", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "62b0bbc5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5edbff1e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "dbdf2ac0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9ddc6b23", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 73, "id": "8170d138", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'20170802'" ] }, "execution_count": 73, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\n", "start_date = datetime.datetime.fromisoformat(\"2017-08-03\")\n", "one_day = datetime.timedelta(1)\n", "next_day = (start_date - one_day)\n", "next_day.strftime(\"%Y%m%d\")" ] }, { "cell_type": "code", "execution_count": 67, "id": "db8bc614", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "datetime.timedelta(days=1)" ] }, "execution_count": 67, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fe93b52d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8a97c300", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 6, "id": "4fcf18ee", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\n", "resp = requests.get(BASE_URL, headers=headers,)\n", "resp" ] }, { "cell_type": "code", "execution_count": 47, "id": "0bfd417b", "metadata": {}, "outputs": [], "source": [ "with open(\"/Users/mikey/data/nz-parl/20170803/response.text\", \"w\") as fi:\n", " fi.write(resp.text)" ] }, { "cell_type": "code", "execution_count": null, "id": "52ebd5ba", "metadata": {}, "outputs": [], "source": [ "video_url = BASE_URL + \"/tab/video\"\n", "vid_resp = requests.get(video_url, headers=headers)\n", "vid_resp" ] }, { "cell_type": "code", "execution_count": 49, "id": "2f9934c9", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 49, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": 50, "id": "d946dca2", "metadata": {}, "outputs": [], "source": [ "video_links = []\n", "for link in BeautifulSoup(vid_resp.text, parse_only=SoupStrainer('a')):\n", " if link.has_attr('href') and \"youtube\" in link[\"href\"]:\n", " video_links.append(link['href'])" ] }, { "cell_type": "code", "execution_count": 51, "id": "c503cc4b", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "20" ] }, "execution_count": 51, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(video_links)" ] }, { "cell_type": "code", "execution_count": 58, "id": "b3e5a9a5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 60, "id": "0c529042", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "https://www.youtube.com/watch?v=7ld6ZD8q0yM\n", "7ld6ZD8q0yM.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=7ld6ZD8q0yM\n", "7ld6ZD8q0yM.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=yPGZ6YjTwgE\n", "yPGZ6YjTwgE.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=yPGZ6YjTwgE\n", "yPGZ6YjTwgE.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=d_tnT-16Guw\n", "d_tnT-16Guw.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=d_tnT-16Guw\n", "d_tnT-16Guw.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=VpwS8_idNeE\n", "VpwS8_idNeE.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=VpwS8_idNeE\n", "VpwS8_idNeE.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=F9rgZ1NMp3U\n", "F9rgZ1NMp3U.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=F9rgZ1NMp3U\n", "F9rgZ1NMp3U.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=-u5F-DP1R3c\n", "-u5F-DP1R3c.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=-u5F-DP1R3c\n", "-u5F-DP1R3c.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=QriIH5RUO0Y\n", "QriIH5RUO0Y.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=QriIH5RUO0Y\n", "QriIH5RUO0Y.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=cHhtiH_m6a0\n", "cant't find audio for {video_key}\n", "https://www.youtube.com/watch?v=cHhtiH_m6a0\n", "cant't find audio for {video_key}\n", "https://www.youtube.com/watch?v=VRPck9qf7Io\n", "downloading https://www.youtube.com/watch?v=VRPck9qf7Io to VRPck9qf7Io.mp4\n", "https://www.youtube.com/watch?v=VRPck9qf7Io\n", "VRPck9qf7Io.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=1epY_NZp8SA\n", "downloading https://www.youtube.com/watch?v=1epY_NZp8SA to 1epY_NZp8SA.mp4\n", "https://www.youtube.com/watch?v=1epY_NZp8SA\n", "1epY_NZp8SA.mp4 exists, skipping...\n" ] } ], "source": [ "for link in video_links:\n", " print(link)\n", " video_key = link.replace(\"https://www.youtube.com/watch?v=\", \"\")\n", " fname = video_key + \".mp4\"\n", " output_path = \"/Users/mikey/data/nz-parl/20170803\"\n", " if os.path.exists(os.path.join(output_path, fname)):\n", " print(f\"{fname} exists, skipping...\")\n", " continue\n", " yt = YouTube(link)\n", " try:\n", " s = yt.streams.filter(type=\"audio\").first()\n", " except VideoUnavailable:\n", " print(\"cant't find audio for {video_key}\")\n", " try:\n", " s = yt.streams.first()\n", " except:\n", " continue\n", " metadata[video_key] = yt.description\n", " print(f\"downloading {link} to {fname}\")\n", " s.download(output_path=output_path, filename=fname)\n", " with open(\"/Users/mikey/data/nz-parl/20170803/metadata.json\", \"w\") as fi:\n", " json.dump(metadata, fi)\n", " time.sleep(4)" ] }, { "cell_type": "code", "execution_count": 61, "id": "69dfe246", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1" ] }, "execution_count": 61, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": 30, "id": "fe77196a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "s.download()" ] }, { "cell_type": "code", "execution_count": 34, "id": "f10a4723", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\n", "s" ] }, { "cell_type": "code", "execution_count": 102, "id": "b983a6bb", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 102, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": 43, "id": "72b6a5e4", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'2017-08-03 - Anti-Money Laundering and Countering Financing of Terrorism Amendment Bill- Third Reading - Video 12\\n\\nJono Naylor,Trevor Mallard,Matt Doocey\\n\\nHelp us caption & translate this video!\\n\\nhttp://amara.org/v/9FM2/'" ] }, "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ "yt.description" ] }, { "cell_type": "code", "execution_count": 44, "id": "10d3aa1c", "metadata": {}, "outputs": [], "source": [ "metadata = {}" ] }, { "cell_type": "code", "execution_count": 45, "id": "b0305ee3", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'/Users/mikey/data/nz-parl/20170803/7ld6ZD8q0yM.mp4'" ] }, "execution_count": 45, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": 46, "id": "fda7b6ea", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 20, "id": "3d7940d8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DefaultParliamentNumber=53&Criteria.ParliamenStartDate=2020-11-24Criteria.PageNumber=10'" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "base_url = \"https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports\"\n", "params = [\n", " \"Criteria.DefaultParliamentNumber=53\",\n", " \"Criteria.ParliamenStartDate=2020-11-24\"\n", " \"Criteria.PageNumber=10\",\n", "]\n", "base_url + \"&\" + \"&\".join(params)" ] }, { "cell_type": "code", "execution_count": 18, "id": "a6950578", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DefaultParliamentNumber=53&Criteria.ParliamenStartDate=2020-11-24&Criteria.PageNumber=6'" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\"https://www.parliament.nz/en/pb/hansard-debates/rhr/?Criteria.page=HansardReports&Criteria.DefaultParliamentNumber=53&Criteria.ParliamenStartDate=2020-11-24&Criteria.PageNumber=6\"" ] }, { "cell_type": "code", "execution_count": null, "id": "f9b04478", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }