{ "cells": [ { "cell_type": "markdown", "id": "0505f35c", "metadata": {}, "source": [ "## get location from twitter" ] }, { "cell_type": "code", "execution_count": 30, "id": "90a7efc5", "metadata": {}, "outputs": [], "source": [ "import json\n", "\n", "in_filepath = \"/home/georg/notebooks/tmp/twitter_links.jsonl\"\n", "out_filepath = \"/home/georg/notebooks/tmp/twitter_info.jsonl\"" ] }, { "cell_type": "code", "execution_count": 31, "id": "24abe598", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1793760 inputs found.\n", "159159 twitter pages found.\n" ] } ], "source": [ "input_links = []\n", "with open(in_filepath) as f:\n", " for e in f.read().strip().split(\"\\n\"):\n", " input_links.append(json.loads(e))\n", "print(len(input_links), \"inputs found.\")\n", "\n", "work_items = []\n", "for e in input_links:\n", " if e[\"social_links\"] is None:\n", " continue\n", " l = [ee.get(\"username\") for ee in e[\"social_links\"][\"userSocialUrls\"] if ee.get(\"type\") == \"twitter\"]\n", " if len(l) >= 1:\n", " work_items.append({\n", " \"uid\": e[\"uid\"],\n", " \"screen_name\": l[0],\n", " })\n", "print(len(work_items), \"twitter pages found.\")" ] }, { "cell_type": "code", "execution_count": 32, "id": "0942630d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "10 already done\n", "159150 work items left\n" ] } ], "source": [ "import os\n", "out = []\n", "if os.path.exists(out_filepath):\n", " with open(out_filepath) as f:\n", " for e in f.read().strip().split(\"\\n\"):\n", " out.append(json.loads(e))\n", "print(len(out), \"already done\")\n", "seen_uids = set([e[\"uid\"] for e in out])\n", "rel_work_items = [e for e in work_items if e[\"uid\"] not in seen_uids]\n", "print(len(rel_work_items), \"work items left\")" ] }, { "cell_type": "code", "execution_count": 33, "id": "f3c06a7d", "metadata": {}, "outputs": [], "source": [ "import requests\n", "import os\n", "import json\n", "import funcy\n", "import urllib\n", "import multiprocessing\n", "import tqdm\n", "import time\n", "import pandas as pd\n", "import numpy as np\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "BEARER_TOKEN = secrets[\"twitter_bearer_token\"]\n", "\n", "def _bearer_oauth(r):\n", " r.headers[\"Authorization\"] = f\"Bearer {BEARER_TOKEN}\"\n", " r.headers[\"User-Agent\"] = \"v2UserLookupPython\"\n", " return r\n", "\n", "def _connect_to_endpoint(url):\n", " response = requests.post(url, auth=_bearer_oauth)\n", " if response.status_code != 200:\n", " if response.json()[\"errors\"][0][\"code\"] == 17:\n", " return []\n", " raise Exception(\n", " \"Request returned an error: {} {}\".format(\n", " response.status_code, response.text\n", " )\n", " )\n", " return response.json()\n", "\n", "MAX_REQUEST_SIZE = 100\n", "N_REQUEST_MAX_PER_WAIT = 200 # 300\n", "WAIT_TIME_S = 15 * 60\n", "# max is ~30k per 15mins\n", "\n", "def _get_locations(work_items):\n", " screen_name_to_uid = {}\n", " for e in work_items:\n", " screen_name_to_uid[e[\"screen_name\"]] = e[\"uid\"]\n", " screen_name_str = \",\".join([urllib.parse.quote_plus(e[\"screen_name\"]) for e in work_items])\n", " url = f\"https://api.twitter.com/1.1/users/lookup.json?screen_name={screen_name_str}\"\n", " out = []\n", " try:\n", " for e in _connect_to_endpoint(url):\n", " out.append({\n", " \"uid\": screen_name_to_uid.get(e.get(\"screen_name\")),\n", " \"twitter_info\": {\n", " \"id\": e.get(\"id\"),\n", " \"name\": e.get(\"name\"),\n", " \"description\": e.get(\"description\"),\n", " \"screen_name\": e.get(\"screen_name\"),\n", " \"location\": e.get(\"location\"),\n", " },\n", " })\n", " except:\n", " pass\n", " return out\n", "\n", "def get_locations(work_items):\n", " out = []\n", " chunksize = MAX_REQUEST_SIZE * N_REQUEST_MAX_PER_WAIT\n", " p = multiprocessing.Pool(5)\n", " for work_items_chunk in tqdm.tqdm_notebook(list(funcy.chunks(chunksize, work_items))):\n", " work_items_chunk_chunk = list(funcy.chunks(MAX_REQUEST_SIZE, work_items_chunk))\n", " t0 = time.time()\n", " out_tmp = p.map(_get_locations, work_items_chunk_chunk, chunksize=5)\n", " for e in out_tmp:\n", " for ee in e:\n", " out.append(ee)\n", " with open(out_filepath, \"w\") as f:\n", " for e in out:\n", " f.write(json.dumps(e) + \"\\n\") \n", " t1 = time.time()\n", " time.sleep(np.max([WAIT_TIME_S - (t1 - t0) + 60, 0]))\n", " p.close()\n", " p.join()\n", " return out" ] }, { "cell_type": "code", "execution_count": null, "id": "f8313a92", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_9224/3834021751.py:66: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n", "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n", " for work_items_chunk in tqdm.tqdm_notebook(list(funcy.chunks(chunksize, work_items))):\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "92284e7d725b4b24a00bf1840e5cfdaa", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/8 [00:00