{ "cells": [ { "cell_type": "code", "execution_count": 85, "id": "ded6a806", "metadata": {}, "outputs": [], "source": [ "import os\n", "import json\n", "import tqdm\n", "import pandas as pd\n", "\n", "DATA_DIR = \"/data/suno/data/harvest/youtube_lg\"" ] }, { "cell_type": "code", "execution_count": 88, "id": "5344d5c5", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "245000it [00:43, 5582.40it/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ "225040 success\n", "19960 fail\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "fail_ids = []\n", "success_ids = []\n", "with open(os.path.join(DATA_DIR, \"youtube_metas.jsonl\")) as f:\n", " for l in tqdm.tqdm(f):\n", " if len(l.strip()) == 0:\n", " continue\n", " m = json.loads(l)\n", " if m[\"success\"]:\n", " success_ids.append(m[\"id\"])\n", " else:\n", " fail_ids.append(m[\"id\"])\n", "print(len(success_ids), \"success\")\n", "print(len(fail_ids), \"fail\")" ] }, { "cell_type": "code", "execution_count": 100, "id": "093054c7", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 100, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjoAAAGdCAYAAAAbudkLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAqPUlEQVR4nO3df1BV953/8Rfy4wos3ooMXG+CEefrGA1m62Jq0Gw1q4Jb0O1kprbBkGSaNWaNEoo20dpuSVrBaDTMysaq65hM0JLvjnE3qy4F+8MsA4LF0Iq6Zndq1KQQ0ngFjfZyA5/vHx3Ot1f8dfUK4ePzMeNM7jnve8/nvLxpXj33HogwxhgBAABYaMhALwAAAOB2oegAAABrUXQAAIC1KDoAAMBaFB0AAGAtig4AALAWRQcAAFiLogMAAKwVNdALGEg9PT36/e9/r4SEBEVERAz0cgAAwA0wxuj8+fPyer0aMuTa12zu6KLz+9//XqmpqQO9DAAAcBPOnDmju++++5ozd3TRSUhIkPSnoIYNGxa21w0EAqqurlZWVpaio6PD9rp3KvIMH7IML/IML/IMH9uz7OzsVGpqqvPf8Wu5o4tO78dVw4YNC3vRiYuL07Bhw6x8g/U38gwfsgwv8gwv8gyfOyXLG/naCV9GBgAA1qLoAAAAa1F0AACAtSg6AADAWhQdAABgLYoOAACwFkUHAABYi6IDAACsRdEBAADWougAAABrUXQAAIC1KDoAAMBaFB0AAGAtig4AALBW1EAvwGbpxT+Tv/vav0L+gzU5/bQaAADuPFzRAQAA1qLoAAAAa1F0AACAtSg6AADAWhQdAABgLYoOAACwFkUHAABYi6IDAACsRdEBAADWougAAABrUXQAAIC1KDoAAMBaFB0AAGAtig4AALAWRQcAAFiLogMAAKxF0QEAANai6AAAAGtRdAAAgLUoOgAAwFoUHQAAYC2KDgAAsBZFBwAAWIuiAwAArEXRAQAA1qLoAAAAa1F0AACAtSg6AADAWhQdAABgLYoOAACwVkhF5/PPP9f3v/99paWlKTY2VmPGjNFLL72knp4eZ8YYo+LiYnm9XsXGxmrGjBk6evRo0Ov4/X4tXbpUSUlJio+P17x58/Thhx8Gzfh8PuXn58vtdsvtdis/P1/nzp0Lmjl9+rTmzp2r+Ph4JSUlqaCgQF1dXSFGAAAAbBVS0Xn55Zf1k5/8ROXl5Tp+/LjWrl2rdevWaePGjc7M2rVrtWHDBpWXl+vQoUPyeDyaPXu2zp8/78wUFhZq9+7dqqysVG1trS5cuKDc3Fx1d3c7M3l5eWpublZVVZWqqqrU3Nys/Px8Z393d7dycnL02Wefqba2VpWVldq1a5eWLVt2K3kAAACLRIUyXF9fr7/7u79TTk6OJGn06NH66U9/ql//+teS/nQ1p6ysTKtWrdIjjzwiSXrjjTeUkpKinTt3atGiRero6NC2bdv05ptvatasWZKkiooKpaamav/+/crOztbx48dVVVWlgwcPasqUKZKkrVu3KjMzUydOnNC4ceNUXV2tY8eO6cyZM/J6vZKk9evX68knn9Tq1as1bNiw8CQEAAAGrZCu6Dz00EP6+c9/rvfff1+S9Jvf/Ea1tbX62te+Jkk6efKk2tralJWV5TzH5XJp+vTpqqurkyQ1NTUpEAgEzXi9XqWnpzsz9fX1crvdTsmRpAcffFButztoJj093Sk5kpSdnS2/36+mpqaQQgAAAHYK6YrOCy+8oI6ODt17772KjIxUd3e3Vq9erUcffVSS1NbWJklKSUkJel5KSopOnTrlzMTExGj48OF9Znqf39bWpuTk5D7HT05ODpq5/DjDhw9XTEyMM3M5v98vv9/vPO7s7JQkBQIBBQKBGwvhBvS+lmuIueFZXF1vRmR168gyvMgzvMgzfGzPMpTzCqnovPXWW6qoqNDOnTt13333qbm5WYWFhfJ6vXriiSecuYiIiKDnGWP6bLvc5TNXmr+ZmT9XWlqqF198sc/26upqxcXFXXN9N+NHk3uuO7Nv376wH9dWNTU1A70Ea5BleJFneJFn+Nia5cWLF294NqSi893vflcrVqzQt771LUnSxIkTderUKZWWluqJJ56Qx+OR9KerLSNHjnSe197e7lx98Xg86urqks/nC7qq097erqlTpzozH3/8cZ/jf/LJJ0Gv09DQELTf5/MpEAj0udLTa+XKlSoqKnIed3Z2KjU1VVlZWWH9Tk8gEFBNTY1+8Osh8vdcu+C1FGeH7bi26s1z9uzZio6OHujlDGpkGV7kGV7kGT62Z9n7icyNCKnoXLx4UUOGBH+tJzIy0rm9PC0tTR6PRzU1NZo0aZIkqaurSwcOHNDLL78sScrIyFB0dLRqamo0f/58SVJra6taWlq0du1aSVJmZqY6OjrU2Nior3zlK5KkhoYGdXR0OGUoMzNTq1evVmtrq1Oqqqur5XK5lJGRccX1u1wuuVyuPtujo6NvyxvB3xMhf/e1i46Nb8Db5Xb9Pd2JyDK8yDO8yDN8bM0ylHMKqejMnTtXq1ev1qhRo3Tffffpvffe04YNG/Ttb39b0p8+SiosLFRJSYnGjh2rsWPHqqSkRHFxccrLy5Mkud1uPfXUU1q2bJlGjBihxMRELV++XBMnTnTuwho/frzmzJmjhQsXavPmzZKkp59+Wrm5uRo3bpwkKSsrSxMmTFB+fr7WrVuns2fPavny5Vq4cCF3XAEAAEkhFp2NGzfqBz/4gRYvXqz29nZ5vV4tWrRI//iP/+jMPP/887p06ZIWL14sn8+nKVOmqLq6WgkJCc7Mq6++qqioKM2fP1+XLl3SzJkz9frrrysyMtKZ2bFjhwoKCpy7s+bNm6fy8nJnf2RkpPbu3avFixdr2rRpio2NVV5enl555ZWbDgMAANglpKKTkJCgsrIylZWVXXUmIiJCxcXFKi4uvurM0KFDtXHjxqAfNHi5xMREVVRUXHM9o0aN0p49e663bAAAcIfid10BAABrUXQAAIC1KDoAAMBaFB0AAGAtig4AALAWRQcAAFiLogMAAKxF0QEAANai6AAAAGtRdAAAgLUoOgAAwFoUHQAAYC2KDgAAsBZFBwAAWIuiAwAArEXRAQAA1qLoAAAAa1F0AACAtSg6AADAWhQdAABgLYoOAACwFkUHAABYi6IDAACsRdEBAADWougAAABrUXQAAIC1KDoAAMBaFB0AAGAtig4AALAWRQcAAFiLogMAAKxF0QEAANai6AAAAGtRdAAAgLUoOgAAwFoUHQAAYC2KDgAAsBZFBwAAWIuiAwAArEXRAQAA1qLoAAAAa1F0AACAtSg6AADAWhQdAABgLYoOAACwFkUHAABYi6IDAACsRdEBAADWougAAABrUXQAAIC1KDoAAMBaFB0AAGAtig4AALAWRQcAAFiLogMAAKxF0QEAANai6AAAAGtRdAAAgLUoOgAAwFoUHQAAYC2KDgAAsBZFBwAAWIuiAwAArEXRAQAA1qLoAAAAa1F0AACAtSg6AADAWhQdAABgLYoOAACwVshF56OPPtJjjz2mESNGKC4uTl/+8pfV1NTk7DfGqLi4WF6vV7GxsZoxY4aOHj0a9Bp+v19Lly5VUlKS4uPjNW/ePH344YdBMz6fT/n5+XK73XK73crPz9e5c+eCZk6fPq25c+cqPj5eSUlJKigoUFdXV6inBAAALBVS0fH5fJo2bZqio6P1n//5nzp27JjWr1+vL33pS87M2rVrtWHDBpWXl+vQoUPyeDyaPXu2zp8/78wUFhZq9+7dqqysVG1trS5cuKDc3Fx1d3c7M3l5eWpublZVVZWqqqrU3Nys/Px8Z393d7dycnL02Wefqba2VpWVldq1a5eWLVt2C3EAAACbRIUy/PLLLys1NVXbt293to0ePdr5Z2OMysrKtGrVKj3yyCOSpDfeeEMpKSnauXOnFi1apI6ODm3btk1vvvmmZs2aJUmqqKhQamqq9u/fr+zsbB0/flxVVVU6ePCgpkyZIknaunWrMjMzdeLECY0bN07V1dU6duyYzpw5I6/XK0lav369nnzySa1evVrDhg27pWAAAMDgF1LReeedd5Sdna1vfOMbOnDggO666y4tXrxYCxculCSdPHlSbW1tysrKcp7jcrk0ffp01dXVadGiRWpqalIgEAia8Xq9Sk9PV11dnbKzs1VfXy+32+2UHEl68MEH5Xa7VVdXp3Hjxqm+vl7p6elOyZGk7Oxs+f1+NTU16eGHH+6zfr/fL7/f7zzu7OyUJAUCAQUCgVCiuKbe13INMTc8i6vrzYisbh1Zhhd5hhd5ho/tWYZyXiEVnd/97nfatGmTioqK9L3vfU+NjY0qKCiQy+XS448/rra2NklSSkpK0PNSUlJ06tQpSVJbW5tiYmI0fPjwPjO9z29ra1NycnKf4ycnJwfNXH6c4cOHKyYmxpm5XGlpqV588cU+26urqxUXF3cjEYTkR5N7rjuzb9++sB/XVjU1NQO9BGuQZXiRZ3iRZ/jYmuXFixdveDakotPT06PJkyerpKREkjRp0iQdPXpUmzZt0uOPP+7MRUREBD3PGNNn2+Uun7nS/M3M/LmVK1eqqKjIedzZ2anU1FRlZWWF9aOuQCCgmpoa/eDXQ+TvufZ5txRnh+24turNc/bs2YqOjh7o5QxqZBle5Ble5Bk+tmfZ+4nMjQip6IwcOVITJkwI2jZ+/Hjt2rVLkuTxeCT96WrLyJEjnZn29nbn6ovH41FXV5d8Pl/QVZ329nZNnTrVmfn444/7HP+TTz4Jep2Ghoag/T6fT4FAoM+Vnl4ul0sul6vP9ujo6NvyRvD3RMjffe2iY+Mb8Ha5XX9PdyKyDC/yDC/yDB9bswzlnEK662ratGk6ceJE0Lb3339f99xzjyQpLS1NHo8n6FJZV1eXDhw44JSYjIwMRUdHB820traqpaXFmcnMzFRHR4caGxudmYaGBnV0dATNtLS0qLW11Zmprq6Wy+VSRkZGKKcFAAAsFdIVne985zuaOnWqSkpKNH/+fDU2NmrLli3asmWLpD99lFRYWKiSkhKNHTtWY8eOVUlJieLi4pSXlydJcrvdeuqpp7Rs2TKNGDFCiYmJWr58uSZOnOjchTV+/HjNmTNHCxcu1ObNmyVJTz/9tHJzczVu3DhJUlZWliZMmKD8/HytW7dOZ8+e1fLly7Vw4ULuuAIAAJJCLDoPPPCAdu/erZUrV+qll15SWlqaysrKtGDBAmfm+eef16VLl7R48WL5fD5NmTJF1dXVSkhIcGZeffVVRUVFaf78+bp06ZJmzpyp119/XZGRkc7Mjh07VFBQ4NydNW/ePJWXlzv7IyMjtXfvXi1evFjTpk1TbGys8vLy9Morr9x0GAAAwC4hFR1Jys3NVW5u7lX3R0REqLi4WMXFxVedGTp0qDZu3KiNGzdedSYxMVEVFRXXXMuoUaO0Z8+e664ZAADcmfhdVwAAwFoUHQAAYC2KDgAAsBZFBwAAWIuiAwAArEXRAQAA1qLoAAAAa1F0AACAtSg6AADAWhQdAABgLYoOAACwFkUHAABYi6IDAACsRdEBAADWougAAABrUXQAAIC1KDoAAMBaFB0AAGAtig4AALAWRQcAAFiLogMAAKxF0QEAANai6AAAAGtRdAAAgLUoOgAAwFoUHQAAYC2KDgAAsBZFBwAAWIuiAwAArEXRAQAA1qLoAAAAa1F0AACAtSg6AADAWlEDvYA73egVe68788GanH5YCQAA9uGKDgAAsBZFBwAAWIuiAwAArEXRAQAA1qLoAAAAa1F0AACAtSg6AADAWhQdAABgLYoOAACwFkUHAABYi6IDAACsRdEBAADWougAAABrUXQAAIC1KDoAAMBaFB0AAGAtig4AALAWRQcAAFiLogMAAKxF0QEAANai6AAAAGtRdAAAgLUoOgAAwFoUHQAAYC2KDgAAsBZFBwAAWIuiAwAArEXRAQAA1qLoAAAAa1F0AACAtSg6AADAWhQdAABgLYoOAACwFkUHAABY65aKTmlpqSIiIlRYWOhsM8aouLhYXq9XsbGxmjFjho4ePRr0PL/fr6VLlyopKUnx8fGaN2+ePvzww6AZn8+n/Px8ud1uud1u5efn69y5c0Ezp0+f1ty5cxUfH6+kpCQVFBSoq6vrVk4JAABY5KaLzqFDh7Rlyxbdf//9QdvXrl2rDRs2qLy8XIcOHZLH49Hs2bN1/vx5Z6awsFC7d+9WZWWlamtrdeHCBeXm5qq7u9uZycvLU3Nzs6qqqlRVVaXm5mbl5+c7+7u7u5WTk6PPPvtMtbW1qqys1K5du7Rs2bKbPSUAAGCZmyo6Fy5c0IIFC7R161YNHz7c2W6MUVlZmVatWqVHHnlE6enpeuONN3Tx4kXt3LlTktTR0aFt27Zp/fr1mjVrliZNmqSKigodOXJE+/fvlyQdP35cVVVV+pd/+RdlZmYqMzNTW7du1Z49e3TixAlJUnV1tY4dO6aKigpNmjRJs2bN0vr167V161Z1dnbeai4AAMACUTfzpGeffVY5OTmaNWuWfvzjHzvbT548qba2NmVlZTnbXC6Xpk+frrq6Oi1atEhNTU0KBAJBM16vV+np6aqrq1N2drbq6+vldrs1ZcoUZ+bBBx+U2+1WXV2dxo0bp/r6eqWnp8vr9Toz2dnZ8vv9ampq0sMPP9xn3X6/X36/33ncW4gCgYACgcDNRHFFva/lGmLC+np3qt7zv9NzCAeyDC/yDC/yDB/bswzlvEIuOpWVlTp8+LAOHTrUZ19bW5skKSUlJWh7SkqKTp065czExMQEXQnqnel9fltbm5KTk/u8fnJyctDM5ccZPny4YmJinJnLlZaW6sUXX+yzvbq6WnFxcVd8zq340eSesLzOvn37wvI6g11NTc1AL8EaZBle5Ble5Bk+tmZ58eLFG54NqeicOXNGzz33nKqrqzV06NCrzkVERAQ9Nsb02Xa5y2euNH8zM39u5cqVKioqch53dnYqNTVVWVlZGjZs2DXXF4pAIKCamhr94NdD5O+59nnfiJbi7DCsavDqzXP27NmKjo4e6OUMamQZXuQZXuQZPrZnGcpXVEIqOk1NTWpvb1dGRoazrbu7W++++67Ky8ud78+0tbVp5MiRzkx7e7tz9cXj8airq0s+ny/oqk57e7umTp3qzHz88cd9jv/JJ58EvU5DQ0PQfp/Pp0Ag0OdKTy+XyyWXy9Vne3R09G15I/h7IuTvvvWiY+Ob9Gbcrr+nOxFZhhd5hhd5ho+tWYZyTiF9GXnmzJk6cuSImpubnT+TJ0/WggUL1NzcrDFjxsjj8QRdKuvq6tKBAwecEpORkaHo6OigmdbWVrW0tDgzmZmZ6ujoUGNjozPT0NCgjo6OoJmWlha1trY6M9XV1XK5XEFFDAAA3LlCuqKTkJCg9PT0oG3x8fEaMWKEs72wsFAlJSUaO3asxo4dq5KSEsXFxSkvL0+S5Ha79dRTT2nZsmUaMWKEEhMTtXz5ck2cOFGzZs2SJI0fP15z5szRwoULtXnzZknS008/rdzcXI0bN06SlJWVpQkTJig/P1/r1q3T2bNntXz5ci1cuDCsH0MBAIDB66buurqW559/XpcuXdLixYvl8/k0ZcoUVVdXKyEhwZl59dVXFRUVpfnz5+vSpUuaOXOmXn/9dUVGRjozO3bsUEFBgXN31rx581ReXu7sj4yM1N69e7V48WJNmzZNsbGxysvL0yuvvBLuUwIAAIPULRedX/3qV0GPIyIiVFxcrOLi4qs+Z+jQodq4caM2btx41ZnExERVVFRc89ijRo3Snj17QlkuAAC4g/C7rgAAgLUoOgAAwFoUHQAAYC2KDgAAsBZFBwAAWIuiAwAArEXRAQAA1qLoAAAAa1F0AACAtSg6AADAWhQdAABgLYoOAACwFkUHAABYi6IDAACsRdEBAADWougAAABrUXQAAIC1KDoAAMBaFB0AAGAtig4AALAWRQcAAFiLogMAAKxF0QEAANai6AAAAGtRdAAAgLUoOgAAwFoUHQAAYC2KDgAAsFbUQC8A1zd6xd7rznywJqcfVgIAwODCFR0AAGAtig4AALAWRQcAAFiLogMAAKxF0QEAANai6AAAAGtRdAAAgLUoOgAAwFoUHQAAYC2KDgAAsBZFBwAAWIuiAwAArEXRAQAA1qLoAAAAa1F0AACAtSg6AADAWhQdAABgLYoOAACwFkUHAABYi6IDAACsRdEBAADWougAAABrUXQAAIC1KDoAAMBaFB0AAGAtig4AALAWRQcAAFiLogMAAKxF0QEAANai6AAAAGtRdAAAgLUoOgAAwFoUHQAAYC2KDgAAsBZFBwAAWIuiAwAArEXRAQAA1qLoAAAAa1F0AACAtUIqOqWlpXrggQeUkJCg5ORkff3rX9eJEyeCZowxKi4ultfrVWxsrGbMmKGjR48Gzfj9fi1dulRJSUmKj4/XvHnz9OGHHwbN+Hw+5efny+12y+12Kz8/X+fOnQuaOX36tObOnav4+HglJSWpoKBAXV1doZwSAACwWEhF58CBA3r22Wd18OBB1dTU6PPPP1dWVpY+++wzZ2bt2rXasGGDysvLdejQIXk8Hs2ePVvnz593ZgoLC7V7925VVlaqtrZWFy5cUG5urrq7u52ZvLw8NTc3q6qqSlVVVWpublZ+fr6zv7u7Wzk5Ofrss89UW1uryspK7dq1S8uWLbuVPAAAgEWiQhmuqqoKerx9+3YlJyerqalJX/3qV2WMUVlZmVatWqVHHnlEkvTGG28oJSVFO3fu1KJFi9TR0aFt27bpzTff1KxZsyRJFRUVSk1N1f79+5Wdna3jx4+rqqpKBw8e1JQpUyRJW7duVWZmpk6cOKFx48apurpax44d05kzZ+T1eiVJ69ev15NPPqnVq1dr2LBhtxwOAAAY3EIqOpfr6OiQJCUmJkqSTp48qba2NmVlZTkzLpdL06dPV11dnRYtWqSmpiYFAoGgGa/Xq/T0dNXV1Sk7O1v19fVyu91OyZGkBx98UG63W3V1dRo3bpzq6+uVnp7ulBxJys7Olt/vV1NTkx5++OE+6/X7/fL7/c7jzs5OSVIgEFAgELiVKIL0vpZriAnba97oMW3Ue242n2N/IcvwIs/wIs/wsT3LUM7rpouOMUZFRUV66KGHlJ6eLklqa2uTJKWkpATNpqSk6NSpU85MTEyMhg8f3mem9/ltbW1KTk7uc8zk5OSgmcuPM3z4cMXExDgzlystLdWLL77YZ3t1dbXi4uKue86h+tHknrC/5tXs27ev3441UGpqagZ6CdYgy/Aiz/Aiz/CxNcuLFy/e8OxNF50lS5bot7/9rWpra/vsi4iICHpsjOmz7XKXz1xp/mZm/tzKlStVVFTkPO7s7FRqaqqysrLC+lFXIBBQTU2NfvDrIfL3XPu8w6WlOLtfjjMQevOcPXu2oqOjB3o5gxpZhhd5hhd5ho/tWfZ+InMjbqroLF26VO+8847effdd3X333c52j8cj6U9XW0aOHOlsb29vd66+eDwedXV1yefzBV3VaW9v19SpU52Zjz/+uM9xP/nkk6DXaWhoCNrv8/kUCAT6XOnp5XK55HK5+myPjo6+LW8Ef0+E/N39U3RsfCNf7nb9Pd2JyDK8yDO8yDN8bM0ylHMK6a4rY4yWLFmit99+W7/4xS+UlpYWtD8tLU0ejyfoUllXV5cOHDjglJiMjAxFR0cHzbS2tqqlpcWZyczMVEdHhxobG52ZhoYGdXR0BM20tLSotbXVmamurpbL5VJGRkYopwUAACwV0hWdZ599Vjt37tS///u/KyEhwfkujNvtVmxsrCIiIlRYWKiSkhKNHTtWY8eOVUlJieLi4pSXl+fMPvXUU1q2bJlGjBihxMRELV++XBMnTnTuwho/frzmzJmjhQsXavPmzZKkp59+Wrm5uRo3bpwkKSsrSxMmTFB+fr7WrVuns2fPavny5Vq4cCF3XAEAAEkhFp1NmzZJkmbMmBG0ffv27XryySclSc8//7wuXbqkxYsXy+fzacqUKaqurlZCQoIz/+qrryoqKkrz58/XpUuXNHPmTL3++uuKjIx0Znbs2KGCggLn7qx58+apvLzc2R8ZGam9e/dq8eLFmjZtmmJjY5WXl6dXXnklpAAAAIC9Qio6xlz/dumIiAgVFxeruLj4qjNDhw7Vxo0btXHjxqvOJCYmqqKi4prHGjVqlPbs2XPdNQEAgDvTLf0cHXxxjF6x97ozH6zJ6YeVAADwxcEv9QQAANai6AAAAGtRdAAAgLUoOgAAwFoUHQAAYC2KDgAAsBZFBwAAWIuiAwAArEXRAQAA1qLoAAAAa1F0AACAtSg6AADAWhQdAABgLYoOAACwFkUHAABYi6IDAACsRdEBAADWougAAABrUXQAAIC1KDoAAMBaFB0AAGAtig4AALAWRQcAAFiLogMAAKxF0QEAANai6AAAAGtRdAAAgLUoOgAAwFoUHQAAYC2KDgAAsBZFBwAAWCtqoBeA/jN6xd7rznywJqcfVgIAQP/gig4AALAWRQcAAFiLogMAAKxF0QEAANai6AAAAGtRdAAAgLUoOgAAwFoUHQAAYC2KDgAAsBZFBwAAWIuiAwAArEXRAQAA1qLoAAAAa1F0AACAtSg6AADAWlEDvQB8sYxesfe6Mx+syemHlQAAcOu4ogMAAKxF0QEAANai6AAAAGtRdAAAgLUoOgAAwFrcdYWQcWcWAGCw4IoOAACwFkUHAABYi6IDAACsRdEBAADWougAAABrcdcVbgvuzAIAfBFwRQcAAFiLogMAAKxF0QEAANai6AAAAGvxZWQMGL6wDAC43biiAwAArEXRAQAA1hr0H1299tprWrdunVpbW3XfffeprKxMf/3Xfz3Qy0KYjF6xV65Io7VfkdKLfyZ/d8SAroeP0gBgcBnUReett95SYWGhXnvtNU2bNk2bN2/W3/7t3+rYsWMaNWrUQC8PFurP7xXxHSYAuHWDuuhs2LBBTz31lP7+7/9eklRWVqaf/exn2rRpk0pLSwd4dbhT3UhB6c9jUYYA3MkGbdHp6upSU1OTVqxYEbQ9KytLdXV1V3yO3++X3+93Hnd0dEiSzp49q0AgELa1BQIBXbx4UVGBIeruGdiPWmwQ1WN08WIPed6k/7P8/zr/7Bpi9P1JPfryqrflH6AsG1bODMvrTCn9eb8d62p6/13/9NNPFR0dfVuPdScgz/CxPcvz589Lkowx150dtEXnD3/4g7q7u5WSkhK0PSUlRW1tbVd8TmlpqV588cU+29PS0m7LGhE+eQO9AIsMdJZJ6+08FoD+d/78ebnd7mvODNqi0ysiIvj/lRpj+mzrtXLlShUVFTmPe3p6dPbsWY0YMeKqz7kZnZ2dSk1N1ZkzZzRs2LCwve6dijzDhyzDizzDizzDx/YsjTE6f/68vF7vdWcHbdFJSkpSZGRkn6s37e3tfa7y9HK5XHK5XEHbvvSlL92uJWrYsGFWvsEGCnmGD1mGF3mGF3mGj81ZXu9KTq9B+3N0YmJilJGRoZqamqDtNTU1mjp16gCtCgAAfJEM2is6klRUVKT8/HxNnjxZmZmZ2rJli06fPq1nnnlmoJcGAAC+AAZ10fnmN7+pTz/9VC+99JJaW1uVnp6uffv26Z577hnQdblcLv3whz/s8zEZbg55hg9Zhhd5hhd5hg9Z/n8R5kbuzQIAABiEBu13dAAAAK6HogMAAKxF0QEAANai6AAAAGtRdG6D1157TWlpaRo6dKgyMjL0X//1XwO9pAFVWlqqBx54QAkJCUpOTtbXv/51nThxImjGGKPi4mJ5vV7FxsZqxowZOnr0aNCM3+/X0qVLlZSUpPj4eM2bN08ffvhh0IzP51N+fr7cbrfcbrfy8/N17ty5232KA6a0tFQREREqLCx0tpFlaD766CM99thjGjFihOLi4vTlL39ZTU1Nzn7yvHGff/65vv/97ystLU2xsbEaM2aMXnrpJfX09Dgz5Hll7777rubOnSuv16uIiAj927/9W9D+/szt9OnTmjt3ruLj45WUlKSCggJ1dXXdjtPuHwZhVVlZaaKjo83WrVvNsWPHzHPPPWfi4+PNqVOnBnppAyY7O9ts377dtLS0mObmZpOTk2NGjRplLly44MysWbPGJCQkmF27dpkjR46Yb37zm2bkyJGms7PTmXnmmWfMXXfdZWpqaszhw4fNww8/bP7yL//SfP75587MnDlzTHp6uqmrqzN1dXUmPT3d5Obm9uv59pfGxkYzevRoc//995vnnnvO2U6WN+7s2bPmnnvuMU8++aRpaGgwJ0+eNPv37zf/+7//68yQ54378Y9/bEaMGGH27NljTp48af71X//V/MVf/IUpKytzZsjzyvbt22dWrVpldu3aZSSZ3bt3B+3vr9w+//xzk56ebh5++GFz+PBhU1NTY7xer1myZMltz+B2oeiE2Ve+8hXzzDPPBG279957zYoVKwZoRV887e3tRpI5cOCAMcaYnp4e4/F4zJo1a5yZP/7xj8btdpuf/OQnxhhjzp07Z6Kjo01lZaUz89FHH5khQ4aYqqoqY4wxx44dM5LMwYMHnZn6+nojyfz3f/93f5xavzl//rwZO3asqampMdOnT3eKDlmG5oUXXjAPPfTQVfeTZ2hycnLMt7/97aBtjzzyiHnssceMMeR5oy4vOv2Z2759+8yQIUPMRx995Mz89Kc/NS6Xy3R0dNyW873d+OgqjLq6utTU1KSsrKyg7VlZWaqrqxugVX3xdHR0SJISExMlSSdPnlRbW1tQbi6XS9OnT3dya2pqUiAQCJrxer1KT093Zurr6+V2uzVlyhRn5sEHH5Tb7bYu/2effVY5OTmaNWtW0HayDM0777yjyZMn6xvf+IaSk5M1adIkbd261dlPnqF56KGH9POf/1zvv/++JOk3v/mNamtr9bWvfU0Sed6s/sytvr5e6enpQb8sMzs7W36/P+gj3cFkUP9k5C+aP/zhD+ru7u7zS0VTUlL6/PLRO5UxRkVFRXrooYeUnp4uSU42V8rt1KlTzkxMTIyGDx/eZ6b3+W1tbUpOTu5zzOTkZKvyr6ys1OHDh3Xo0KE++8gyNL/73e+0adMmFRUV6Xvf+54aGxtVUFAgl8ulxx9/nDxD9MILL6ijo0P33nuvIiMj1d3drdWrV+vRRx+VxPvzZvVnbm1tbX2OM3z4cMXExAzabCk6t0FERETQY2NMn213qiVLlui3v/2tamtr++y7mdwun7nSvE35nzlzRs8995yqq6s1dOjQq86R5Y3p6enR5MmTVVJSIkmaNGmSjh49qk2bNunxxx935sjzxrz11luqqKjQzp07dd9996m5uVmFhYXyer164oknnDnyvDn9lZtt2fLRVRglJSUpMjKyT+ttb2/v05DvREuXLtU777yjX/7yl7r77rud7R6PR5KumZvH41FXV5d8Pt81Zz7++OM+x/3kk0+syb+pqUnt7e3KyMhQVFSUoqKidODAAf3TP/2ToqKinPMkyxszcuRITZgwIWjb+PHjdfr0aUm8N0P13e9+VytWrNC3vvUtTZw4Ufn5+frOd76j0tJSSeR5s/ozN4/H0+c4Pp9PgUBg0GZL0QmjmJgYZWRkqKamJmh7TU2Npk6dOkCrGnjGGC1ZskRvv/22fvGLXygtLS1of1pamjweT1BuXV1dOnDggJNbRkaGoqOjg2ZaW1vV0tLizGRmZqqjo0ONjY3OTENDgzo6OqzJf+bMmTpy5Iiam5udP5MnT9aCBQvU3NysMWPGkGUIpk2b1udHHbz//vvOLwbmvRmaixcvasiQ4P+sREZGOreXk+fN6c/cMjMz1dLSotbWVmemurpaLpdLGRkZt/U8b5t+/vKz9XpvL9+2bZs5duyYKSwsNPHx8eaDDz4Y6KUNmH/4h38wbrfb/OpXvzKtra3On4sXLzoza9asMW6327z99tvmyJEj5tFHH73irZN333232b9/vzl8+LD5m7/5myveOnn//feb+vp6U19fbyZOnDiobzm9EX9+15UxZBmKxsZGExUVZVavXm3+53/+x+zYscPExcWZiooKZ4Y8b9wTTzxh7rrrLuf28rffftskJSWZ559/3pkhzys7f/68ee+998x7771nJJkNGzaY9957z/nRJP2VW+/t5TNnzjSHDx82+/fvN3fffTe3lyPYP//zP5t77rnHxMTEmL/6q79ybqO+U0m64p/t27c7Mz09PeaHP/yh8Xg8xuVyma9+9avmyJEjQa9z6dIls2TJEpOYmGhiY2NNbm6uOX36dNDMp59+ahYsWGASEhJMQkKCWbBggfH5fP1wlgPn8qJDlqH5j//4D5Oenm5cLpe59957zZYtW4L2k+eN6+zsNM8995wZNWqUGTp0qBkzZoxZtWqV8fv9zgx5Xtkvf/nLK/7v5BNPPGGM6d/cTp06ZXJyckxsbKxJTEw0S5YsMX/84x9v5+nfVhHGGDMw15IAAABuL76jAwAArEXRAQAA1qLoAAAAa1F0AACAtSg6AADAWhQdAABgLYoOAACwFkUHAABYi6IDAACsRdEBAADWougAAABrUXQAAIC1/h/HjAm0/0GeLAAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "(pd.Series([duration_map[_id] for _id in success_ids if _id in duration_map])).hist(bins=50)" ] }, { "cell_type": "code", "execution_count": null, "id": "b864ac6c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 106, "id": "fbe720f8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "130485901\n", "58034835\n" ] } ], "source": [ "print(np.sum([duration_map[_id] for _id in success_ids if _id in duration_map]))\n", "print(np.sum([duration_map[_id] for _id in fail_ids if _id in duration_map]))" ] }, { "cell_type": "code", "execution_count": 101, "id": "4d9f7fc8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 101, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjEAAAGdCAYAAADjWSL8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAwoElEQVR4nO3df3AUdZ7/8deQTCYkl4wkKTKMBolVnLIGlQuKoLfgQRI5ArdF3bIuGtldTvEQMBtQQdbboGciXC1wF05UihKLyGJdKZ63cpjhdEEqyI9gVoIerrUIgsS4GhMwcTIm/f3DS38dQiCDPUk+k+ejiir70+/p+fR7RnjVZ7pnXJZlWQIAADDMoL6eAAAAwKUgxAAAACMRYgAAgJEIMQAAwEiEGAAAYCRCDAAAMBIhBgAAGIkQAwAAjBTf1xOIlo6ODn3yySdKSUmRy+Xq6+kAAIAesCxLZ86ckd/v16BBF15ridkQ88knnygrK6uvpwEAAC7Bxx9/rCuuuOKCNTEbYlJSUiR924TU1FTHjhsKhVRVVaX8/Hy53W7HjjtQ0U9n0U/n0Etn0U/nxHovm5ublZWVZf87fiExG2I6P0JKTU11PMQkJSUpNTU1Jt88vY1+Oot+OodeOot+Omeg9LInl4JwYS8AADASIQYAABiJEAMAAIxEiAEAAEYixAAAACMRYgAAgJEIMQAAwEiEGAAAYCRCDAAAMBIhBgAAGCniELN7925Nnz5dfr9fLpdLr7zySre18+bNk8vl0tq1a8PGg8GgFi5cqIyMDCUnJ2vGjBk6efJkWE1jY6OKiork9Xrl9XpVVFSkL7/8MtLpAgCAGBVxiPnqq690/fXXa926dRese+WVV7Rv3z75/f4u+4qLi7Vt2zZt3bpVe/bs0dmzZ1VYWKj29na7Zvbs2aqtrdWOHTu0Y8cO1dbWqqioKNLpAgCAGBXxD0BOnTpVU6dOvWDNqVOntGDBAr3++uuaNm1a2L6mpiZt3LhRmzdv1pQpUyRJlZWVysrK0s6dO1VQUKD3339fO3bs0Ntvv61x48ZJkjZs2KDx48fr6NGjuvrqqyOdNgAAiDGO/4p1R0eHioqK9OCDD+raa6/tsr+mpkahUEj5+fn2mN/vV05Ojqqrq1VQUKC9e/fK6/XaAUaSbr75Znm9XlVXV583xASDQQWDQXu7ublZ0re/9hkKhRw7v85jOXnMgYx+Oot+OodeOot+OifWexnJeTkeYlauXKn4+HgtWrTovPvr6+uVkJCgIUOGhI1nZmaqvr7erhk6dGiXxw4dOtSuOVd5eblWrFjRZbyqqkpJSUmRnsZFBQIBx485kNFPZ9FP59BLZ9FP58RqL1taWnpc62iIqamp0b/+67/q0KFDcrlcET3Wsqywx5zv8efWfNeyZctUUlJibzc3NysrK0v5+flKTU2NaC4XEgqFFAgE9OjBQQp2XPgc60oLHHveWNXZz7y8PLnd7r6ejvHop3PopbPop3NivZedn6T0hKMh5q233lJDQ4OGDx9uj7W3t2vx4sVau3atPvroI/l8PrW1tamxsTFsNaahoUETJkyQJPl8Pn366addjv/ZZ58pMzPzvM/t8Xjk8Xi6jLvd7qi8yMEOl4LtFw4xsfjmipZovU4DFf10Dr10Fv10Tqz2MpJzcvR7YoqKivTuu++qtrbW/uP3+/Xggw/q9ddflyTl5ubK7XaHLYOdPn1adXV1dogZP368mpqatH//frtm3759ampqsmsAAMDAFvFKzNmzZ/Xhhx/a28eOHVNtba3S0tI0fPhwpaenh9W73W75fD77Ylyv16u5c+dq8eLFSk9PV1pampYsWaLRo0fbdyuNGjVKt99+u+655x4988wzkqR7771XhYWF3JkEAAAkXUKIOXjwoG677TZ7u/M6lDlz5mjTpk09OsaaNWsUHx+vWbNmqbW1VZMnT9amTZsUFxdn17zwwgtatGiRfRfTjBkzLvrdNAAAYOCIOMRMmjRJlmX1uP6jjz7qMpaYmKiKigpVVFR0+7i0tDRVVlZGOj0AADBA8NtJAADASIQYAABgJEIMAAAwEiEGAAAYiRADAACMRIgBAABGIsQAAAAjEWIAAICRCDEAAMBIhBgAAGAkQgwAADASIQYAABiJEAMAAIxEiAEAAEYixAAAACMRYgAAgJEIMQAAwEiEGAAAYCRCDAAAMBIhBgAAGIkQAwAAjESIAQAARiLEAAAAIxFiAACAkeL7egLAQDRi6WsXrfnoyWm9MBMAMBcrMQAAwEiEGAAAYCRCDAAAMBIhBgAAGIkQAwAAjESIAQAARiLEAAAAIxFiAACAkQgxAADASIQYAABgJEIMAAAwEiEGAAAYiRADAACMRIgBAABGijjE7N69W9OnT5ff75fL5dIrr7xi7wuFQnr44Yc1evRoJScny+/36+6779Ynn3wSdoxgMKiFCxcqIyNDycnJmjFjhk6ePBlW09jYqKKiInm9Xnm9XhUVFenLL7+8pJMEAACxJ+IQ89VXX+n666/XunXruuxraWnRoUOH9Oijj+rQoUN6+eWX9cEHH2jGjBlhdcXFxdq2bZu2bt2qPXv26OzZsyosLFR7e7tdM3v2bNXW1mrHjh3asWOHamtrVVRUdAmnCAAAYlF8pA+YOnWqpk6det59Xq9XgUAgbKyiokI33XSTTpw4oeHDh6upqUkbN27U5s2bNWXKFElSZWWlsrKytHPnThUUFOj999/Xjh079Pbbb2vcuHGSpA0bNmj8+PE6evSorr766kinDQAAYkzEISZSTU1NcrlcuuyyyyRJNTU1CoVCys/Pt2v8fr9ycnJUXV2tgoIC7d27V16v1w4wknTzzTfL6/Wqurr6vCEmGAwqGAza283NzZK+/YgrFAo5dj6dx/IMsnpci+519mig9coTF533z0DtZzTQS2fRT+fEei8jOa+ohpivv/5aS5cu1ezZs5WamipJqq+vV0JCgoYMGRJWm5mZqfr6ertm6NChXY43dOhQu+Zc5eXlWrFiRZfxqqoqJSUlfd9T6eLxsR0Xrdm+fbvjzxurzl3Bi3Wrbrp4zfd5/wy0fkYTvXQW/XROrPaypaWlx7VRCzGhUEh33HGHOjo69NRTT1203rIsuVwue/u7/91dzXctW7ZMJSUl9nZzc7OysrKUn59vBygnhEIhBQIBPXpwkIId559Lp7rSAseeN1Z19jMvL09ut7uvp9Nrckpfv2jNpbx/Bmo/o4FeOot+OifWe9n5SUpPRCXEhEIhzZo1S8eOHdMbb7wRFiJ8Pp/a2trU2NgYthrT0NCgCRMm2DWffvppl+N+9tlnyszMPO9zejweeTyeLuNutzsqL3Kww6Vg+4VDTCy+uaIlWq9Tf3Wx9470/d4/A62f0UQvnUU/nROrvYzknBz/npjOAPPHP/5RO3fuVHp6etj+3Nxcud3usGWw06dPq66uzg4x48ePV1NTk/bv32/X7Nu3T01NTXYNAAAY2CJeiTl79qw+/PBDe/vYsWOqra1VWlqa/H6//v7v/16HDh3S7373O7W3t9vXsKSlpSkhIUFer1dz587V4sWLlZ6errS0NC1ZskSjR4+271YaNWqUbr/9dt1zzz165plnJEn33nuvCgsLuTMJAABIuoQQc/DgQd122232dud1KHPmzFFpaaleffVVSdINN9wQ9rg333xTkyZNkiStWbNG8fHxmjVrllpbWzV58mRt2rRJcXFxdv0LL7ygRYsW2XcxzZgx47zfTQMAAAamiEPMpEmTZFnd3x56oX2dEhMTVVFRoYqKim5r0tLSVFlZGen0AADAAMFvJwEAACMRYgAAgJEIMQAAwEiEGAAAYCRCDAAAMBIhBgAAGIkQAwAAjESIAQAARiLEAAAAIxFiAACAkQgxAADASIQYAABgJEIMAAAwEiEGAAAYiRADAACMRIgBAABGIsQAAAAjEWIAAICRCDEAAMBIhBgAAGAkQgwAADASIQYAABiJEAMAAIxEiAEAAEYixAAAACPF9/UEEJtGLH3tojUfPTmtF2YCAIhVrMQAAAAjEWIAAICRCDEAAMBIhBgAAGAkQgwAADASIQYAABiJEAMAAIzE98REEd+VAgBA9LASAwAAjESIAQAARiLEAAAAIxFiAACAkQgxAADASBGHmN27d2v69Ony+/1yuVx65ZVXwvZblqXS0lL5/X4NHjxYkyZN0pEjR8JqgsGgFi5cqIyMDCUnJ2vGjBk6efJkWE1jY6OKiork9Xrl9XpVVFSkL7/8MuITBAAAsSniEPPVV1/p+uuv17p16867f9WqVVq9erXWrVunAwcOyOfzKS8vT2fOnLFriouLtW3bNm3dulV79uzR2bNnVVhYqPb2drtm9uzZqq2t1Y4dO7Rjxw7V1taqqKjoEk4RAADEooi/J2bq1KmaOnXqefdZlqW1a9dq+fLlmjlzpiTp+eefV2ZmprZs2aJ58+apqalJGzdu1ObNmzVlyhRJUmVlpbKysrRz504VFBTo/fff144dO/T2229r3LhxkqQNGzZo/PjxOnr0qK6++upLPV8AABAjHP2yu2PHjqm+vl75+fn2mMfj0cSJE1VdXa158+appqZGoVAorMbv9ysnJ0fV1dUqKCjQ3r175fV67QAjSTfffLO8Xq+qq6vPG2KCwaCCwaC93dzcLEkKhUIKhUKOnWPnsTyDLEePF2s8cRfvz3dfm1jtQ3d62p9IDdR+RgO9dBb9dE6s9zKS83I0xNTX10uSMjMzw8YzMzN1/PhxuyYhIUFDhgzpUtP5+Pr6eg0dOrTL8YcOHWrXnKu8vFwrVqzoMl5VVaWkpKTIT+YiHh/b4chxtm/f7shx+ptVN1285rvnHggEojib/ifS/kRqoPUzmuils+inc2K1ly0tLT2ujcrPDrhcrrBty7K6jJ3r3Jrz1V/oOMuWLVNJSYm93dzcrKysLOXn5ys1NTWS6V9QKBRSIBDQowcHKdhx4XPqibrSAgdm1f/klL5+0Zq60gK7n3l5eXK73b0ws/6hp/2J1EDtZzTQS2fRT+fEei87P0npCUdDjM/nk/TtSsqwYcPs8YaGBnt1xufzqa2tTY2NjWGrMQ0NDZowYYJd8+mnn3Y5/meffdZllaeTx+ORx+PpMu52u6PyIgc7XAq2f/8QE4tvQEk96s13zz1ar1N/FWl/IjXQ+hlN9NJZ9NM5sdrLSM7J0RCTnZ0tn8+nQCCgMWPGSJLa2tq0a9curVy5UpKUm5srt9utQCCgWbNmSZJOnz6turo6rVq1SpI0fvx4NTU1af/+/brppm/X3fft26empiY76ACR4Mc4ASD2RBxizp49qw8//NDePnbsmGpra5WWlqbhw4eruLhYZWVlGjlypEaOHKmysjIlJSVp9uzZkiSv16u5c+dq8eLFSk9PV1pampYsWaLRo0fbdyuNGjVKt99+u+655x4988wzkqR7771XhYWF3JkEAAAkXUKIOXjwoG677TZ7u/M6lDlz5mjTpk166KGH1Nraqvnz56uxsVHjxo1TVVWVUlJS7MesWbNG8fHxmjVrllpbWzV58mRt2rRJcXFxds0LL7ygRYsW2XcxzZgxo9vvpgEAAANPxCFm0qRJsqzubw91uVwqLS1VaWlptzWJiYmqqKhQRUVFtzVpaWmqrKyMdHoAAGCA4LeTAACAkQgxAADASIQYAABgJEIMAAAwEiEGAAAYiRADAACMRIgBAABGIsQAAAAjReVXrIGeGLH0NXniLK266dtfdT7fjyLye0YAgO6wEgMAAIxEiAEAAEYixAAAACMRYgAAgJEIMQAAwEiEGAAAYCRCDAAAMBLfEwPjjVj6Wl9PAQDQB1iJAQAARiLEAAAAIxFiAACAkQgxAADASFzY28d6clEqP4IIAEBXrMQAAAAjEWIAAICRCDEAAMBIhBgAAGAkQgwAADASIQYAABiJEAMAAIxEiAEAAEYixAAAACMRYgAAgJEIMQAAwEiEGAAAYCRCDAAAMBIhBgAAGIkQAwAAjESIAQAARiLEAAAAIxFiAACAkRwPMd98841+9atfKTs7W4MHD9ZVV12lxx57TB0dHXaNZVkqLS2V3+/X4MGDNWnSJB05ciTsOMFgUAsXLlRGRoaSk5M1Y8YMnTx50unpAgAAQzkeYlauXKmnn35a69at0/vvv69Vq1bpX/7lX1RRUWHXrFq1SqtXr9a6det04MAB+Xw+5eXl6cyZM3ZNcXGxtm3bpq1bt2rPnj06e/asCgsL1d7e7vSUAQCAgeKdPuDevXv1d3/3d5o2bZokacSIEfrtb3+rgwcPSvp2FWbt2rVavny5Zs6cKUl6/vnnlZmZqS1btmjevHlqamrSxo0btXnzZk2ZMkWSVFlZqaysLO3cuVMFBQVOTxsAABjG8RBz66236umnn9YHH3ygv/zLv9Qf/vAH7dmzR2vXrpUkHTt2TPX19crPz7cf4/F4NHHiRFVXV2vevHmqqalRKBQKq/H7/crJyVF1dfV5Q0wwGFQwGLS3m5ubJUmhUEihUMix8+s8lmeQ5dgxe/qcJvHE9aw/nX3srp89OfeePtfF9GafezLnS5lP52NMfM/0N/TSWfTTObHey0jOy2VZlqP/GluWpUceeUQrV65UXFyc2tvb9cQTT2jZsmWSpOrqat1yyy06deqU/H6//bh7771Xx48f1+uvv64tW7bo5z//eVgokaT8/HxlZ2frmWee6fK8paWlWrFiRZfxLVu2KCkpyclTBAAAUdLS0qLZs2erqalJqampF6x1fCXmxRdfVGVlpbZs2aJrr71WtbW1Ki4ult/v15w5c+w6l8sV9jjLsrqMnetCNcuWLVNJSYm93dzcrKysLOXn51+0CZEIhUIKBAJ69OAgBTsuPF+n1JWa9/FZTunrParzDLL0+NiObvvZk3Pv6XNdTG/2uSdzvpT5dL4/8/Ly5Ha7L2Vq+D/00ln00zmx3svOT1J6wvEQ8+CDD2rp0qW64447JEmjR4/W8ePHVV5erjlz5sjn80mS6uvrNWzYMPtxDQ0NyszMlCT5fD61tbWpsbFRQ4YMCauZMGHCeZ/X4/HI4/F0GXe73VF5kYMdLgXbeyfEmPgmjbQ33fWzJ+fu1OvQm33uyZy/z3yi9b4fiOils+inc2K1l5Gck+N3J7W0tGjQoPDDxsXF2bdYZ2dny+fzKRAI2Pvb2tq0a9cuO6Dk5ubK7XaH1Zw+fVp1dXXdhhgAADCwOL4SM336dD3xxBMaPny4rr32Wr3zzjtavXq1fvGLX0j69mOk4uJilZWVaeTIkRo5cqTKysqUlJSk2bNnS5K8Xq/mzp2rxYsXKz09XWlpaVqyZIlGjx5t360EAAAGNsdDTEVFhR599FHNnz9fDQ0N8vv9mjdvnv7pn/7JrnnooYfU2tqq+fPnq7GxUePGjVNVVZVSUlLsmjVr1ig+Pl6zZs1Sa2urJk+erE2bNikuLs7pKQMAAAM5HmJSUlK0du1a+5bq83G5XCotLVVpaWm3NYmJiaqoqAj7kjwAAIBO/HYSAAAwEiEGAAAYiRADAACMRIgBAABGIsQAAAAjOX53EuCkEUtf6+spAAD6KVZiAACAkQgxAADASIQYAABgJEIMAAAwEiEGAAAYiRADAACMxC3WiBi3PQMA+gNWYgAAgJEIMQAAwEiEGAAAYCSuiQEcxjVDANA7CDFABAgoANB/EGKA/0NAAQCzcE0MAAAwEiEGAAAYiRADAACMRIgBAABG4sJeoJ/qyYXGHz05rRdmAgD9EysxAADASIQYAABgJEIMAAAwEiEGAAAYiRADAACMRIgBAABGIsQAAAAjEWIAAICRCDEAAMBIhBgAAGAkQgwAADASIQYAABiJEAMAAIxEiAEAAEYixAAAACNFJcScOnVKd911l9LT05WUlKQbbrhBNTU19n7LslRaWiq/36/Bgwdr0qRJOnLkSNgxgsGgFi5cqIyMDCUnJ2vGjBk6efJkNKYLAAAM5HiIaWxs1C233CK3263//u//1nvvvaff/OY3uuyyy+yaVatWafXq1Vq3bp0OHDggn8+nvLw8nTlzxq4pLi7Wtm3btHXrVu3Zs0dnz55VYWGh2tvbnZ4yAAAwULzTB1y5cqWysrL03HPP2WMjRoyw/9uyLK1du1bLly/XzJkzJUnPP/+8MjMztWXLFs2bN09NTU3auHGjNm/erClTpkiSKisrlZWVpZ07d6qgoMDpaQMAAMM4HmJeffVVFRQU6Mc//rF27dqlyy+/XPPnz9c999wjSTp27Jjq6+uVn59vP8bj8WjixImqrq7WvHnzVFNTo1AoFFbj9/uVk5Oj6urq84aYYDCoYDBobzc3N0uSQqGQQqGQY+fXeSzPIMuxY/b0OfsLT5xz597Zx97sZyw5973Rud3f3jMmopfOop/OifVeRnJejoeYP/3pT1q/fr1KSkr0yCOPaP/+/Vq0aJE8Ho/uvvtu1dfXS5IyMzPDHpeZmanjx49Lkurr65WQkKAhQ4Z0qel8/LnKy8u1YsWKLuNVVVVKSkpy4tTCPD62w/Fjdmf79u299lw9seom54/Zm/2MJd29NwKBQC/PJHbRS2fRT+fEai9bWlp6XOt4iOno6NDYsWNVVlYmSRozZoyOHDmi9evX6+6777brXC5X2OMsy+oydq4L1SxbtkwlJSX2dnNzs7KyspSfn6/U1NRLPZ0uQqGQAoGAHj04SMGOC8/XKXWl/evjs5zS1x07lmeQpcfHdvRqP2PJue+NzvdnXl6e3G53H80qNtBLZ9FP58R6Lzs/SekJx0PMsGHD9IMf/CBsbNSoUXrppZckST6fT9K3qy3Dhg2zaxoaGuzVGZ/Pp7a2NjU2NoatxjQ0NGjChAnnfV6PxyOPx9Nl3O12R+VFDna4FGzvnX90+9ubNBrn3Zv9jCXdvTei9b4fiOils+inc2K1l5Gck+N3J91yyy06evRo2NgHH3ygK6+8UpKUnZ0tn88XtgzW1tamXbt22QElNzdXbrc7rOb06dOqq6vrNsQAAICBxfGVmF/+8peaMGGCysrKNGvWLO3fv1/PPvusnn32WUnffoxUXFyssrIyjRw5UiNHjlRZWZmSkpI0e/ZsSZLX69XcuXO1ePFipaenKy0tTUuWLNHo0aPtu5UAAMDA5niIufHGG7Vt2zYtW7ZMjz32mLKzs7V27Vrdeeedds1DDz2k1tZWzZ8/X42NjRo3bpyqqqqUkpJi16xZs0bx8fGaNWuWWltbNXnyZG3atElxcXFOTxkAABjI8RAjSYWFhSosLOx2v8vlUmlpqUpLS7utSUxMVEVFhSoqKqIwQ3RnxNLX+noKAAD0CL+dBAAAjBSVlRj0T6yyAABiCSsxAADASIQYAABgJD5OMkBPPgb66MlpvTATAAD6D1ZiAACAkQgxAADASIQYAABgJEIMAAAwEiEGAAAYiRADAACMRIgBAABGIsQAAAAj8WV3gMHO/SJET5ylVTdJOaWvK9juksQXIQKIXazEAAAAIxFiAACAkQgxAADASIQYAABgJEIMAAAwEiEGAAAYiRADAACMRIgBAABGIsQAAAAjEWIAAICRCDEAAMBIhBgAAGAkfgAyRpz7Q4AAAMQ6VmIAAICRCDEAAMBIhBgAAGAkQgwAADASIQYAABiJEAMAAIxEiAEAAEYixAAAACMRYgAAgJEIMQAAwEiEGAAAYCRCDAAAMFLUQ0x5eblcLpeKi4vtMcuyVFpaKr/fr8GDB2vSpEk6cuRI2OOCwaAWLlyojIwMJScna8aMGTp58mS0pwsAAAwR1RBz4MABPfvss7ruuuvCxletWqXVq1dr3bp1OnDggHw+n/Ly8nTmzBm7pri4WNu2bdPWrVu1Z88enT17VoWFhWpvb4/mlAEAgCGiFmLOnj2rO++8Uxs2bNCQIUPsccuytHbtWi1fvlwzZ85UTk6Onn/+ebW0tGjLli2SpKamJm3cuFG/+c1vNGXKFI0ZM0aVlZU6fPiwdu7cGa0pAwAAg0QtxNx///2aNm2apkyZEjZ+7Ngx1dfXKz8/3x7zeDyaOHGiqqurJUk1NTUKhUJhNX6/Xzk5OXYNAAAY2OKjcdCtW7fq0KFDOnDgQJd99fX1kqTMzMyw8czMTB0/ftyuSUhICFvB6azpfPy5gsGggsGgvd3c3CxJCoVCCoVCl34y5+g8lmeQ5dgxB7LOPtJPZ5yvn06+/weSzr7RP2fQT+fEei8jOS/HQ8zHH3+sBx54QFVVVUpMTOy2zuVyhW1bltVl7FwXqikvL9eKFSu6jFdVVSkpKakHM4/M42M7HD/mQEY/nfXdfm7fvr0PZ2K+QCDQ11OIKfTTObHay5aWlh7XOh5iampq1NDQoNzcXHusvb1du3fv1rp163T06FFJ3662DBs2zK5paGiwV2d8Pp/a2trU2NgYthrT0NCgCRMmnPd5ly1bppKSEnu7ublZWVlZys/PV2pqqmPnFwqFFAgE9OjBQQp2XDh04eI8gyw9PraDfjokmv2sKy1w9Hj9Xef/63l5eXK73X09HePRT+fEei87P0npCcdDzOTJk3X48OGwsZ///Oe65ppr9PDDD+uqq66Sz+dTIBDQmDFjJEltbW3atWuXVq5cKUnKzc2V2+1WIBDQrFmzJEmnT59WXV2dVq1add7n9Xg88ng8XcbdbndUXuRgh0vBdv7RdQr9dFY0+hmLf1n2RLT+Dhmo6KdzYrWXkZyT4yEmJSVFOTk5YWPJyclKT0+3x4uLi1VWVqaRI0dq5MiRKisrU1JSkmbPni1J8nq9mjt3rhYvXqz09HSlpaVpyZIlGj16dJcLhQEAwMAUlQt7L+ahhx5Sa2ur5s+fr8bGRo0bN05VVVVKSUmxa9asWaP4+HjNmjVLra2tmjx5sjZt2qS4uLi+mDIAAOhneiXE/P73vw/bdrlcKi0tVWlpabePSUxMVEVFhSoqKqI7OQAAYCR+OwkAABiJEAMAAIxEiAEAAEYixAAAACMRYgAAgJEIMQAAwEiEGAAAYCRCDAAAMBIhBgAAGIkQAwAAjESIAQAARiLEAAAAI/XJr1gDwIWMWPraRWs+enJaL8wEQH/GSgwAADASKzEAHMMKCoDeRIgB0CM9CSgA0Jv4OAkAABiJEAMAAIxEiAEAAEYixAAAACMRYgAAgJG4OwmAkbidGwArMQAAwEisxADoVXzfDACnsBIDAACMRIgBAABGIsQAAAAjEWIAAICRCDEAAMBIhBgAAGAkQgwAADASIQYAABiJEAMAAIxEiAEAAEYixAAAACMRYgAAgJEIMQAAwEiEGAAAYCRCDAAAMBIhBgAAGMnxEFNeXq4bb7xRKSkpGjp0qH70ox/p6NGjYTWWZam0tFR+v1+DBw/WpEmTdOTIkbCaYDCohQsXKiMjQ8nJyZoxY4ZOnjzp9HQBAIChHA8xu3bt0v3336+3335bgUBA33zzjfLz8/XVV1/ZNatWrdLq1au1bt06HThwQD6fT3l5eTpz5oxdU1xcrG3btmnr1q3as2ePzp49q8LCQrW3tzs9ZQAAYKB4pw+4Y8eOsO3nnntOQ4cOVU1NjX74wx/KsiytXbtWy5cv18yZMyVJzz//vDIzM7VlyxbNmzdPTU1N2rhxozZv3qwpU6ZIkiorK5WVlaWdO3eqoKDA6WkDAADDOB5iztXU1CRJSktLkyQdO3ZM9fX1ys/Pt2s8Ho8mTpyo6upqzZs3TzU1NQqFQmE1fr9fOTk5qq6uPm+ICQaDCgaD9nZzc7MkKRQKKRQKOXY+ncfyDLIcO+ZA1tlH+ukM+hnu+/y/3/lYJ//+GMjop3NivZeRnFdUQ4xlWSopKdGtt96qnJwcSVJ9fb0kKTMzM6w2MzNTx48ft2sSEhI0ZMiQLjWdjz9XeXm5VqxY0WW8qqpKSUlJ3/tczvX42A7HjzmQ0U9n0c9vbd++/XsfIxAIODATdKKfzonVXra0tPS4NqohZsGCBXr33Xe1Z8+eLvtcLlfYtmVZXcbOdaGaZcuWqaSkxN5ubm5WVlaW8vPzlZqaegmzP79QKKRAIKBHDw5SsOPC88XFeQZZenxsB/10CP0MV1d66R89d/6/npeXJ7fb7eCsBib66ZxY72XnJyk9EbUQs3DhQr366qvavXu3rrjiCnvc5/NJ+na1ZdiwYfZ4Q0ODvTrj8/nU1tamxsbGsNWYhoYGTZgw4bzP5/F45PF4uoy73e6ovMjBDpeC7fwj4RT66Sz6+a2Rj1ZdtOajJ6ddcH+0/g4ZqOinc2K1l5Gck+N3J1mWpQULFujll1/WG2+8oezs7LD92dnZ8vl8YctgbW1t2rVrlx1QcnNz5Xa7w2pOnz6turq6bkMMAAAYWBxfibn//vu1ZcsW/ed//qdSUlLsa1i8Xq8GDx4sl8ul4uJilZWVaeTIkRo5cqTKysqUlJSk2bNn27Vz587V4sWLlZ6errS0NC1ZskSjR4+271YCAAADm+MhZv369ZKkSZMmhY0/99xz+tnPfiZJeuihh9Ta2qr58+ersbFR48aNU1VVlVJSUuz6NWvWKD4+XrNmzVJra6smT56sTZs2KS4uzukpAxjARix97bzjnjhLq26Sckpf19EnCnt5VgB6wvEQY1kXv7XT5XKptLRUpaWl3dYkJiaqoqJCFRUVDs4OAADECn47CQAAGIkQAwAAjBT1b+wFAHyru+tvvutit3wD+P9YiQEAAEYixAAAACMRYgAAgJG4JgYALqK/XcvS3+YD9BVCDAA4oCfBojcRdDAQ8HESAAAwEiEGAAAYiRADAACMRIgBAABGIsQAAAAjcXcSAPQj/e0uJ6A/I8QAALrFrdroz/g4CQAAGIkQAwAAjESIAQAARiLEAAAAI3FhLwAMUNwJBdOxEgMAAIxEiAEAAEbi4yQAwPfSk4+lPHGWVt0k5ZS+rmC767w1fN8MIsVKDAAAMBIhBgAAGIkQAwAAjESIAQAARiLEAAAAI3F3EgDAGPyqNr6LEAMA6Bf4BmFEihADABiQWNUxHyEGABBTentFhzDUdwgxAAD0A4ShyHF3EgAAMBIrMQAARJlTH3GNWPpaj36HygkmrPoQYgAA6AZ3TPVvhBgAANCFCdfocE0MAAAwEiEGAAAYqd+HmKeeekrZ2dlKTExUbm6u3nrrrb6eEgAA6Af6dYh58cUXVVxcrOXLl+udd97RX//1X2vq1Kk6ceJEX08NAAD0sX4dYlavXq25c+fqH/7hHzRq1CitXbtWWVlZWr9+fV9PDQAA9LF+e3dSW1ubampqtHTp0rDx/Px8VVdXd6kPBoMKBoP2dlNTkyTpiy++UCgUcmxeoVBILS0tig8NUntH9O7PHyjiOyy1tHTQT4fQT+fQS2fRT+f0p15+/vnnjh/zzJkzkiTLsi5a229DzJ///Ge1t7crMzMzbDwzM1P19fVd6svLy7VixYou49nZ2VGbI5wxu68nEGPop3PopbPop3P6Sy8zfhO9Y585c0Zer/eCNf02xHRyucJTpmVZXcYkadmyZSopKbG3Ozo69MUXXyg9Pf289ZequblZWVlZ+vjjj5WamurYcQcq+uks+ukceuks+umcWO+lZVk6c+aM/H7/RWv7bYjJyMhQXFxcl1WXhoaGLqszkuTxeOTxeMLGLrvssqjNLzU1NSbfPH2FfjqLfjqHXjqLfjonlnt5sRWYTv32wt6EhATl5uYqEAiEjQcCAU2YMKGPZgUAAPqLfrsSI0klJSUqKirS2LFjNX78eD377LM6ceKE7rvvvr6eGgAA6GP9OsT85Cc/0eeff67HHntMp0+fVk5OjrZv364rr7yyz+bk8Xj061//ustHV7g09NNZ9NM59NJZ9NM59PL/c1k9uYcJAACgn+m318QAAABcCCEGAAAYiRADAACMRIgBAABGIsRE6KmnnlJ2drYSExOVm5urt956q6+n1KfKy8t14403KiUlRUOHDtWPfvQjHT16NKzGsiyVlpbK7/dr8ODBmjRpko4cORJWEwwGtXDhQmVkZCg5OVkzZszQyZMnw2oaGxtVVFQkr9crr9eroqIiffnll9E+xT5TXl4ul8ul4uJie4xeRubUqVO66667lJ6erqSkJN1www2qqamx99PPnvvmm2/0q1/9StnZ2Ro8eLCuuuoqPfbYY+ro6LBr6Gf3du/erenTp8vv98vlcumVV14J29+bvTtx4oSmT5+u5ORkZWRkaNGiRWpra4vGaUefhR7bunWr5Xa7rQ0bNljvvfee9cADD1jJycnW8ePH+3pqfaagoMB67rnnrLq6Oqu2ttaaNm2aNXz4cOvs2bN2zZNPPmmlpKRYL730knX48GHrJz/5iTVs2DCrubnZrrnvvvusyy+/3AoEAtahQ4es2267zbr++uutb775xq65/fbbrZycHKu6utqqrq62cnJyrMLCwl49396yf/9+a8SIEdZ1111nPfDAA/Y4vey5L774wrryyiutn/3sZ9a+ffusY8eOWTt37rQ+/PBDu4Z+9tw///M/W+np6dbvfvc769ixY9Z//Md/WH/xF39hrV271q6hn93bvn27tXz5cuull16yJFnbtm0L299bvfvmm2+snJwc67bbbrMOHTpkBQIBy+/3WwsWLIh6D6KBEBOBm266ybrvvvvCxq655hpr6dKlfTSj/qehocGSZO3atcuyLMvq6OiwfD6f9eSTT9o1X3/9teX1eq2nn37asizL+vLLLy23221t3brVrjl16pQ1aNAga8eOHZZlWdZ7771nSbLefvttu2bv3r2WJOt///d/e+PUes2ZM2eskSNHWoFAwJo4caIdYuhlZB5++GHr1ltv7XY//YzMtGnTrF/84hdhYzNnzrTuuusuy7LoZyTODTG92bvt27dbgwYNsk6dOmXX/Pa3v7U8Ho/V1NQUlfONJj5O6qG2tjbV1NQoPz8/bDw/P1/V1dV9NKv+p6mpSZKUlpYmSTp27Jjq6+vD+ubxeDRx4kS7bzU1NQqFQmE1fr9fOTk5ds3evXvl9Xo1btw4u+bmm2+W1+uNuf7ff//9mjZtmqZMmRI2Ti8j8+qrr2rs2LH68Y9/rKFDh2rMmDHasGGDvZ9+RubWW2/V//zP/+iDDz6QJP3hD3/Qnj179Ld/+7eS6Of30Zu927t3r3JycsJ+XLGgoEDBYDDso1ZT9Otv7O1P/vznP6u9vb3Lj09mZmZ2+ZHKgcqyLJWUlOjWW29VTk6OJNm9OV/fjh8/btckJCRoyJAhXWo6H19fX6+hQ4d2ec6hQ4fGVP+3bt2qQ4cO6cCBA1320cvI/OlPf9L69etVUlKiRx55RPv379eiRYvk8Xh09913088IPfzww2pqatI111yjuLg4tbe364knntBPf/pTSbw/v4/e7F19fX2X5xkyZIgSEhKM7C8hJkIulyts27KsLmMD1YIFC/Tuu+9qz549XfZdSt/OrTlffSz1/+OPP9YDDzygqqoqJSYmdltHL3umo6NDY8eOVVlZmSRpzJgxOnLkiNavX6+7777brqOfPfPiiy+qsrJSW7Zs0bXXXqva2loVFxfL7/drzpw5dh39vHS91btY6i8fJ/VQRkaG4uLiuiTVhoaGLql2IFq4cKFeffVVvfnmm7riiivscZ/PJ0kX7JvP51NbW5saGxsvWPPpp592ed7PPvssZvpfU1OjhoYG5ebmKj4+XvHx8dq1a5f+7d/+TfHx8fZ50sueGTZsmH7wgx+EjY0aNUonTpyQxHszUg8++KCWLl2qO+64Q6NHj1ZRUZF++ctfqry8XBL9/D56s3c+n6/L8zQ2NioUChnZX0JMDyUkJCg3N1eBQCBsPBAIaMKECX00q75nWZYWLFigl19+WW+88Yays7PD9mdnZ8vn84X1ra2tTbt27bL7lpubK7fbHVZz+vRp1dXV2TXjx49XU1OT9u/fb9fs27dPTU1NMdP/yZMn6/Dhw6qtrbX/jB07Vnfeeadqa2t11VVX0csI3HLLLV1u9//ggw/sH5DlvRmZlpYWDRoU/k9GXFycfYs1/bx0vdm78ePHq66uTqdPn7Zrqqqq5PF4lJubG9XzjIpevpDYaJ23WG/cuNF67733rOLiYis5Odn66KOP+npqfeYf//EfLa/Xa/3+97+3Tp8+bf9paWmxa5588knL6/VaL7/8snX48GHrpz/96XlvHbziiiusnTt3WocOHbL+5m/+5ry3Dl533XXW3r17rb1791qjR482/rbLi/nu3UmWRS8jsX//fis+Pt564oknrD/+8Y/WCy+8YCUlJVmVlZV2Df3suTlz5liXX365fYv1yy+/bGVkZFgPPfSQXUM/u3fmzBnrnXfesd555x1LkrV69WrrnXfesb+io7d613mL9eTJk61Dhw5ZO3futK644gpusR4o/v3f/9268sorrYSEBOuv/uqv7FuJBypJ5/3z3HPP2TUdHR3Wr3/9a8vn81kej8f64Q9/aB0+fDjsOK2trdaCBQustLQ0a/DgwVZhYaF14sSJsJrPP//cuvPOO62UlBQrJSXFuvPOO63GxsZeOMu+c26IoZeR+a//+i8rJyfH8ng81jXXXGM9++yzYfvpZ881NzdbDzzwgDV8+HArMTHRuuqqq6zly5dbwWDQrqGf3XvzzTfP+3flnDlzLMvq3d4dP37cmjZtmjV48GArLS3NWrBggfX1119H8/SjxmVZltU3a0AAAACXjmtiAACAkQgxAADASIQYAABgJEIMAAAwEiEGAAAYiRADAACMRIgBAABGIsQAAAAjEWIAAICRCDEAAMBIhBgAAGAkQgwAADDS/wME6DSr7+hKtAAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "(pd.Series([duration_map[_id] for _id in fail_ids if _id in duration_map])).hist(bins=50)" ] }, { "cell_type": "code", "execution_count": null, "id": "bd53f771", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "93b5e850", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 93, "id": "deff679e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100000it [01:05, 1515.34it/s]\n" ] } ], "source": [ "def _parse_duration(s):\n", " if s is None:\n", " return 0\n", " parts = s.split(\":\")\n", " if len(parts) == 3:\n", " nh, nm, ns = parts\n", " elif len(parts) == 2:\n", " nm, ns = parts\n", " nh = 0\n", " else:\n", " raise ValueError(\"\")\n", " return int(nh) * 60**2 + int(nm) * 60 + int(ns)\n", "\n", "duration_map = {}\n", "with open(os.path.join(DATA_DIR, \"videos_by_domain_global.jsonl\")) as f:\n", " for l in tqdm.tqdm(f):\n", " if len(l.strip()) == 0:\n", " continue\n", " m = json.loads(l)\n", " if m is None:\n", " continue\n", " domain_str, search_results = m\n", " for search_result in search_results:\n", " _id = search_result[\"id\"]\n", " try:\n", " duration_map[search_result[\"id\"]] = _parse_duration(search_result[\"duration\"])\n", " except:\n", " continue\n", "# 6771507 unique videos\n", "# 60091 excluded\n", "# 1 failed\n", "# 1447996.6 hours of data" ] }, { "cell_type": "code", "execution_count": null, "id": "c24a5733", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5d81af32", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "21087d20", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "47915eb0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "e74204da", "metadata": {}, "source": [ "### check languages" ] }, { "cell_type": "code", "execution_count": 9, "id": "5c806d5f", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "1566000it [00:04, 376381.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "1541929 worked\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "data = []\n", "with open(os.path.join(DATA_DIR, \"youtube_metas.jsonl\")) as f:\n", " for l in tqdm.tqdm(f):\n", " if len(l.strip()) == 0:\n", " continue\n", " m = json.loads(l)\n", " if m is None or not m[\"success\"]:\n", " continue\n", " success_youtube_ids.append(m[\"youtube_id\"])\n", "print(len(success_youtube_ids), \"worked\")\n", "# 1541929 worked" ] }, { "cell_type": "code", "execution_count": 23, "id": "80503113", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "6146593 subtitles\n", "1541116 audios\n" ] } ], "source": [ "subtitle_filenames = os.listdir(os.path.join(DATA_DIR, \"subtitles\"))\n", "l_failed = []\n", "data = []\n", "for fn in subtitle_filenames:\n", " parts = fn.split(\".\")\n", " if len(parts) != 3:\n", " l_failed.append(fn)\n", " continue\n", " data.append(parts)\n", "print(len(l_failed), \"failed\")\n", "print(len(subtitle_filenames) - len(l_failed), \"subtitles\")\n", "df = pd.DataFrame(data, columns=[\"youtube_id\", \"lang\", \"ext\"])\n", "df = df[df[\"youtube_id\"].isin(set(success_youtube_ids))]\n", "df = df[df[\"ext\"] == \"vtt\"]\n", "print(df[\"youtube_id\"].nunique(), \"audios\")\n", "# 0 failed\n", "# 6146593 files worked\n", "# 1541116 audios" ] }, { "cell_type": "code", "execution_count": 34, "id": "974a1e35", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1176966 unique subtitle audios\n" ] } ], "source": [ "unique_cc_df = df[df[\"youtube_id\"].isin(set(df[\"youtube_id\"].value_counts().loc[lambda x: x == 1].index))]\n", "print(unique_cc_df.shape[0], \"unique subtitle audios\")" ] }, { "cell_type": "code", "execution_count": 45, "id": "c66b9504", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "en \t557929\n", "en-US\t 66086\n", "en-GB\t 39323\n", "es \t 28310\n", "ru \t 22321\n", "fr \t 18877\n", "hi \t 14989\n", "zh-TW\t 13321\n", "de \t 12358\n", "pt \t 12160\n", "ko \t 11655\n", "id \t 11527\n", "ja \t 9537\n", "it \t 9321\n", "pt-BR\t 9316\n", "en-IN\t 8837\n", "es-419\t 8656\n", "tr \t 8598\n", "pl \t 7775\n", "ar \t 6667\n", "vi \t 6456\n", "bn \t 6369\n", "zh-Hans\t 5812\n", "nl \t 5678\n", "zh \t 5392\n", "ur \t 4562\n", "en-CA\t 4362\n", "es-MX\t 4351\n", "ta \t 4286\n", "zh-CN\t 3817\n", "zh-Hant\t 3698\n", "da \t 3169\n", "cs \t 3028\n", "te \t 2931\n", "zh-HK\t 2817\n", "es-ES\t 2801\n", "sv \t 2783\n", "th \t 2667\n", "iw \t 2341\n", "el \t 2331\n", "fi \t 2071\n", "uk \t 1986\n", "hu \t 1906\n", "de-DE\t 1904\n", "ro \t 1790\n", "fil \t 1588\n", "fr-CA\t 1577\n", "ca \t 1493\n", "fr-FR\t 1454\n", "und \t 1422\n", "fa \t 1398\n", "no \t 1393\n", "ml \t 1143\n", "sr \t 1129\n", "sk \t 1105\n", "bg \t 1014\n", "hr \t 842\n", "ms \t 806\n", "si \t 791\n", "pt-PT\t 769\n", "nl-NL\t 692\n", "lt \t 619\n", "en-IE\t 571\n", "sl \t 529\n", "fa-IR\t 520\n", "hy \t 514\n", "lv \t 509\n", "es-US\t 467\n", "kk \t 452\n", "sq \t 403\n", "pa \t 385\n", "yue-HK\t 370\n", "mr \t 353\n", "et \t 336\n", "ne \t 319\n", "bs \t 291\n", "nl-BE\t 284\n", "tlh \t 274\n", "az \t 271\n", "sr-Latn\t 251\n", "ase \t 250\n", "de-CH\t 241\n", "kn \t 240\n", "zh-SG\t 223\n", "de-AT\t 209\n", "eu \t 205\n", "sw \t 201\n", "eo \t 197\n", "mk \t 189\n", "uz \t 182\n", "gu \t 178\n", "hi-Latn\t 167\n", "km \t 166\n", "ps \t 162\n", "bho \t 157\n", "or \t 146\n", "fr-BE\t 138\n", "is \t 129\n", "ka \t 120\n", "la \t 117\n", "iw-IL\t 117\n", "am \t 117\n", "fa-AF\t 110\n", "sv-SE\t 109\n", "yue \t 108\n", "so \t 104\n", "mn \t 103\n" ] } ], "source": [ "# look at language breakdown from unique ones\n", "for k, v in (unique_cc_df[\"lang\"].value_counts(normalize=True) * 1_000_000).round().astype(int).items():\n", " if v < 100:\n", " continue\n", " print(k.ljust(5) + \"\\t\" + str(v).rjust(6))" ] }, { "cell_type": "code", "execution_count": 48, "id": "ac76ef8c", "metadata": {}, "outputs": [], "source": [ "langs = []\n", "for k, v in (unique_cc_df[\"lang\"].value_counts(normalize=True) * 1_000_000).round().astype(int).items():\n", " if v < 1000:\n", " continue\n", " langs.append(k.split(\"-\")[0])" ] }, { "cell_type": "code", "execution_count": 49, "id": "d871594a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "56" ] }, "execution_count": 49, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(langs)" ] }, { "cell_type": "code", "execution_count": null, "id": "aec584a9", "metadata": {}, "outputs": [], "source": [ "# TODO: videos should have a language (even the ambiguous from subtitle ones)" ] }, { "cell_type": "code", "execution_count": 81, "id": "be01f3d0", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'/data/suno/data/harvest/youtube_lg/audio'" ] }, "execution_count": 81, "metadata": {}, "output_type": "execute_result" } ], "source": [ "d" ] }, { "cell_type": "code", "execution_count": 84, "id": "0c91a1fd", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "200900.00000000003" ] }, "execution_count": 84, "metadata": {}, "output_type": "execute_result" } ], "source": [ "41_000 * 4.9" ] }, { "cell_type": "code", "execution_count": 109, "id": "81e84450", "metadata": {}, "outputs": [], "source": [ "fns = os.listdir(os.path.join(DATA_DIR, \"audio\"))" ] }, { "cell_type": "code", "execution_count": 112, "id": "d3f0f2c4", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1278704" ] }, "execution_count": 112, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(fns)" ] }, { "cell_type": "code", "execution_count": null, "id": "af2e81ae", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f241cc87", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "44ce03b6", "metadata": {}, "source": [ "## get nlp corpus" ] }, { "cell_type": "code", "execution_count": null, "id": "b1b875fd", "metadata": {}, "outputs": [], "source": [ "# load all the english vtt files into a text corpus for searching" ] }, { "cell_type": "code", "execution_count": 2, "id": "e5154934", "metadata": {}, "outputs": [], "source": [ "import os\n", "import json\n", "import tqdm\n", "import pandas as pd\n", "import webvtt\n", "import re\n", "import html\n", "from suno_utils.utils.text import normalize_whitespace\n", "\n", "DATA_DIR = \"/data/suno/data/harvest/youtube_lg\"\n", "\n", "fns = os.listdir(os.path.join(DATA_DIR, \"audio\"))" ] }, { "cell_type": "code", "execution_count": 3, "id": "dcddd067", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 242050/242050 [00:08<00:00, 27975.52it/s]\n" ] } ], "source": [ "data = []\n", "for fn in fns:\n", " parts = fn.split(\".\")\n", " if len(parts) != 3:\n", " continue\n", " data.append((parts[0], parts[1]))\n", "df = pd.DataFrame(data)\n", "_df = df[df[1].str[:2] == \"en\"]\n", "_df = _df.drop_duplicates(subset=[0])\n", "en_vtt_fns = set()\n", "for _, row in tqdm.tqdm(_df.iterrows(), total=_df.shape[0]):\n", " en_vtt_fns.add(\".\".join([row[0], row[1], \"vtt\"]))\n", "en_vtt_fns = list(en_vtt_fns)" ] }, { "cell_type": "code", "execution_count": 26, "id": "3ee402ef", "metadata": {}, "outputs": [], "source": [ "def remove_tags(text):\n", " \"\"\"\n", " Remove vtt markup tags\n", " \"\"\"\n", " tags = [\n", " r'',\n", " r'',\n", " r'<\\d{2}:\\d{2}:\\d{2}\\.\\d{3}>',\n", "\n", " ]\n", "\n", " for pat in tags:\n", " text = re.sub(pat, '', text)\n", "\n", " # extract timestamp, only kep HH:MM\n", " text = re.sub(\n", " r'(\\d{2}:\\d{2}):\\d{2}\\.\\d{3} --> .* align:start position:0%',\n", " r'\\g<1>',\n", " text\n", " )\n", "\n", " text = re.sub(r'^\\s+$', '', text, flags=re.MULTILINE)\n", " return text\n", "\n", "def remove_header(lines):\n", " \"\"\"\n", " Remove vtt file header\n", " \"\"\"\n", " pos = -1\n", " for mark in ('##', 'Language: en',):\n", " if mark in lines:\n", " pos = lines.index(mark)\n", " lines = lines[pos+1:]\n", " return lines\n", "\n", "\n", "def merge_duplicates(lines):\n", " \"\"\"\n", " Remove duplicated subtitles. Duplacates are always adjacent.\n", " \"\"\"\n", " last_timestamp = ''\n", " last_cap = ''\n", " for line in lines:\n", " if line == \"\":\n", " continue\n", " if re.match('^\\d{2}:\\d{2}$', line):\n", " if line != last_timestamp:\n", " yield line\n", " last_timestamp = line\n", " else:\n", " if line != last_cap:\n", " yield line\n", " last_cap = line\n", "\n", "\n", "def merge_short_lines(lines):\n", " buffer = ''\n", " for line in lines:\n", " if line == \"\" or re.match('^\\d{2}:\\d{2}$', line):\n", " yield '\\n' + line\n", " continue\n", "\n", " if len(line+buffer) < 80:\n", " buffer += ' ' + line\n", " else:\n", " yield buffer.strip()\n", " buffer = line\n", " yield buffer" ] }, { "cell_type": "code", "execution_count": 47, "id": "98ba6027", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 242050/242050 [19:59<00:00, 201.76it/s]\n" ] } ], "source": [ "with open(\"/data/suno/data/harvest/nlp/captions.txt\", \"w\") as fw:\n", " for fn in tqdm.tqdm(en_vtt_fns):\n", " fp = os.path.join(DATA_DIR, \"audio\", fn)\n", " try:\n", "# for caption in webvtt.read(fp):\n", "# # print(caption.start)\n", "# # print(caption.end)\n", "# text = caption.text\n", "# text = re.sub(r\"\\s+-\\s*\", \" \", text)\n", "# text = re.sub(r\"\\s*-\\s+\", \" \", text)\n", "# text = normalize_whitespace(html.unescape(text))\n", "# fw.write(text + \"\\n\")\n", " \n", " with open(fp) as f:\n", " text = f.read()\n", " text = remove_tags(text)\n", " lines = text.splitlines()\n", " lines = remove_header(lines)\n", " lines = merge_duplicates(lines)\n", " lines = list(lines)\n", " lines = merge_short_lines(lines)\n", " clean_lines = []\n", " for line in lines:\n", " line = re.sub(r\"\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3} \\-\\-\\> \\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}\", \" \", line)\n", " line = normalize_whitespace(html.unescape(line))\n", " if len(line) == 0:\n", " continue\n", " clean_lines.append(line)\n", " for line in clean_lines:\n", " fw.write(line + \"\\n\")\n", " \n", " fw.write(\"\\n\")\n", " except webvtt.errors.MalformedCaptionError:\n", " pass\n", "# break" ] }, { "cell_type": "code", "execution_count": null, "id": "463c3175", "metadata": {}, "outputs": [], "source": [ "# TODO: parser has issues\n", "# an add an 01:26 line:0% echocardiogram. 01:26 line:0%\n", "# H: Hey, Ke names\n", "\n", "# TODO: what about leading dashes for new speaker?" ] }, { "cell_type": "code", "execution_count": 54, "id": "8d6337e7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "linked to PyTorch and it has an amazing API and\n", "frameworks. And so we started with PyTorch passing is built in\n", "PyTorch will be what pandas will be to rapidsai.\n", "I looked at when I started doing that was the PyTorch, Data\n", "like PyTorch and TensorFlow. 624\n", "in PyTorch? It would not be that that different,\n", "below even PyTorch and Keras itself. So mesh grid\n", "including TensorFlow, PyTorch and MXNet.\n", "like TensorFlow and PyTorch\n", "including Tensor Flow, MXNet and PyTorch.\n", "as well as dependencies from PyTorch and CD-DNN.\n", "In given data center may have some models in PyTorch, others in ONNX and others in TensorFlow, we can handle all of those and more so it's\n", "quality software right now is based on PyTorch and Python.\n", "including PyTorch MXNet and TensorFlow.\n", "for deep learning, TensorFlow, PyTorch, and MXNet.\n", "and 91% of cloud based PyTorch runs on AWS\n", "TorchServe now the default model serving library on PyTorch was built\n", "of additional code in your PyTorch and TensorFlow training scripts,\n", "that work on TensorFlow and PyTorch\n", "and 27 minutes on PyTorch. With our optimization\n", "and six minutes and 45 seconds for PyTorch.\n", "to fully trained model on PyTorch\n", "PyTorch, and scikit-learn. You can also add\n", "in TensorFlow or PyTorch. Other significant parts\n" ] } ], "source": [ "!cat /data/suno/data/harvest/nlp/captions.txt | grep PyTorch" ] }, { "cell_type": "code", "execution_count": 55, "id": "c7dd72d4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "pytorch. It's, you know, pytorch\n", "bitstream from pytorch, pull it out, do some data frame\n", "can't quite do a pytorch. I\n", "rewritten the pytorch Data\n", "the pytorch's data loader is one\n", "particular path. So the pytorch Data Loader, you know, what I\n", "then a code example in pytorch of how we can train such a\n" ] } ], "source": [ "!cat /data/suno/data/harvest/nlp/captions.txt | grep pytorch" ] }, { "cell_type": "code", "execution_count": null, "id": "0dd00a04", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3b8fda22", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d9579ff0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c9fac0da", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "2be3344e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ae86609d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d9ea25c4", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" } }, "nbformat": 4, "nbformat_minor": 5 }