{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "112037d9", "metadata": {}, "outputs": [], "source": [ "import os\n", "import re\n", "import tqdm\n", "from collections import defaultdict, Counter\n", "import pickle\n", "import pandas as pd\n", "import numpy as np\n", "import plotly.graph_objects as go\n", "from ipywidgets import Output, VBox\n", "import webbrowser\n", "from IPython.display import display, HTML, Javascript" ] }, { "cell_type": "code", "execution_count": 2, "id": "8723ebbc", "metadata": {}, "outputs": [], "source": [ "meta_df = pd.read_json(\n", " \"/mnt/data-ssd-2/data/arxiv/custom_ml_meta.json\", \n", " dtype={\"id\": str},\n", " lines=True,\n", ")\n", "meta_df['submit_date'] = pd.to_datetime(meta_df['submit_date'], unit='ms')" ] }, { "cell_type": "code", "execution_count": 3, "id": "d510fe1c", "metadata": { "scrolled": false }, "outputs": [], "source": [ "def _make_df(search_term, min_date, min_citation, org_set):\n", " # format orgs\n", " meta_df['orgs_main'] = [\n", " \" & \".join([o for o in s.split(\" & \") if o in org_set]) \n", " for s in meta_df['orgs'].values\n", " ]\n", " meta_df['orgs_main'] = meta_df['orgs_main'].replace(\"\", \"other\")\n", " # filter papers\n", " plot_df = meta_df[\n", " (\n", " meta_df['title'].str.contains(r\"\\b{}\\b\".format(search_term), flags=re.IGNORECASE) |\n", " (meta_df['abstract'].str.count(r\"\\b{}\\b\".format(search_term), flags=re.IGNORECASE) >= 2)\n", " ) & (\n", " (\n", " (meta_df['submit_date'].dt.year >= int(min_date)) &\n", " (meta_df['n_citation'] >= min_citation)\n", " ) | (\n", " ((meta_df['submit_date'].max() - meta_df['submit_date']).dt.days <= 90) &\n", " (meta_df['orgs_main'] != \"other\")\n", " )\n", " )\n", " ].copy()\n", " plot_df.loc[(\n", " ((meta_df['submit_date'].max() - meta_df['submit_date']).dt.days <= 90) &\n", " (meta_df['orgs_main'] != \"other\")\n", " ), \"n_citation\"] += min_citation\n", " return plot_df\n", "\n", "def make_figure(search_term, min_date, min_citation, org_set):\n", "\n", " plot_df = _make_df(search_term, min_date, min_citation, org_set)\n", " \n", " fig = go.FigureWidget()\n", "\n", " dfs = []\n", " trace_names = list(plot_df['orgs_main'].unique())\n", " for name in trace_names:\n", " _df = plot_df[plot_df['orgs_main'] == name]\n", " dfs.append(_df)\n", " customdata = _df['orgs'].values\n", " # customdata = np.dstack((_df['title'].values, _df['orgs'].values))\n", " fig.add_scatter(\n", " x=_df[\"submit_date\"], \n", " y=_df[\"n_citation\"], \n", " name=name,\n", " text=_df['title'],\n", " customdata=customdata,\n", " hovertemplate=\n", " \"%{text}
\" +\n", " \"%{x|%Y %b} - %{customdata}
\" +\n", " \"\",\n", " )\n", "\n", " out = Output()\n", " @out.capture(clear_output=False)\n", " def handle_click(trace, points, state):\n", " if points.point_inds:\n", " out.clear_output()\n", "\n", " for n, scatter in enumerate(fig.data):\n", " s = [10] * dfs[n].shape[0]\n", " with fig.batch_update():\n", " scatter.marker.size = s\n", " scatter.marker.opacity = 0.75\n", "\n", " n_trace = points.trace_index\n", " n_point = points.point_inds[0]\n", "\n", " scatter = fig.data[n_trace]\n", " s = [10] * dfs[n_trace].shape[0]\n", " s[n_point] = 15\n", " with fig.batch_update():\n", " scatter.marker.size = s\n", " scatter.marker.opacity = 0.75\n", "\n", " row = dfs[n_trace].iloc[n_point]\n", " display(HTML(\"paper link\".format(row['id'])))\n", " display(HTML(\"\" + row['title'].replace(\"
\", \" \") + \"\"))\n", " print_str = row[\"submit_date\"].strftime(\"%Y %b\")\n", " if len(row[\"orgs\"]) > 0:\n", " print_str += \" - \" + row[\"orgs\"]\n", " print_str += \", {} citations\".format(row[\"n_citation\"])\n", " print(print_str)\n", " display(HTML(row['abstract'].strip()))\n", "\n", " for scatter in fig.data:\n", " scatter.on_click(handle_click)\n", "\n", " fig.update_traces(\n", " mode='markers',\n", " marker={\n", " 'size': 10,\n", " 'line': {\n", " 'width': 1,\n", " 'color': 'white',\n", " },\n", " 'opacity': 0.75,\n", " }\n", " )\n", "\n", " for n, scatter in enumerate(fig.data):\n", " if trace_names[n] == \"other\":\n", " scatter.marker.color = \"#888\"\n", "\n", " fig.update_layout(\n", " title={\n", " 'text': \"Most cited papers for
`{}`\".format(search_term),\n", " 'y':0.88,\n", " 'x':0.5,\n", " 'xanchor': 'center',\n", " 'yanchor': 'top'\n", " },\n", " xaxis={'title': 'Publication date'},\n", " yaxis={'type': 'log', 'visible': False, 'showticklabels': False}\n", " )\n", " return fig, out" ] }, { "cell_type": "code", "execution_count": 4, "id": "94fdfa6e", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "ce9aaf578d7c400bb0dd4814997553eb", "version_major": 2, "version_minor": 0 }, "text/plain": [ "VBox(children=(FigureWidget({\n", " 'data': [{'customdata': array(['Us.ibm', 'Ii.uni.wroc', '', '', 'Mails.tsing…" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "search_term = r\"speech recognition\"\n", "min_date = \"2015\"\n", "min_citation = 5\n", "org_set = set([\"Facebook\", \"Google\", \"Microsoft\", \"Amazon\", \"Nvidia\"])\n", "\n", "fig, out = make_figure(search_term, min_date, min_citation, org_set)\n", "VBox([fig, out])" ] }, { "cell_type": "code", "execution_count": null, "id": "e5921fa4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8ed225be", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9873b99e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "df2c1c53", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "31ae7238", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9f78a6c9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "014e44b5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8b917e43", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b0f33651", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a41e0904", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "62d161f4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "cf78ff26", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "edb02c46", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "84afb140", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "3aea125a", "metadata": {}, "source": [ "### Improvements" ] }, { "cell_type": "code", "execution_count": 5, "id": "54b70389", "metadata": {}, "outputs": [], "source": [ "# TODO: rank companies for multi-match\n", "# TODO: restrict extracted emails to first N characters to avoid references?\n", "# TODO: expand emails with companies for 1512.02595, 1910.10261, 1904.05862, 1703.02136, 2006.13979\n", "# TODO: additional metric for papers from last month?\n", "# TODO: normalize citations by age?\n", "# TODO: synonyms for filtering\n", "# TODO: filtering using fulltext?" ] }, { "cell_type": "code", "execution_count": null, "id": "ec07acf6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bd69ac1e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "29297f8d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3c42746e", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }