{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5dabd6d3",
   "metadata": {},
   "source": [
    "## use open subtitles dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7a1958f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tqdm\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4b1aabe5",
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_DIR = \"/data/suno/data/FrequencyWords/content/2018\"\n",
    "\n",
    "lang_codes = [s for s in os.listdir(BASE_DIR) if os.path.isdir(os.path.join(BASE_DIR, s))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "06b033a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 62/62 [00:32<00:00,  1.92it/s]\n"
     ]
    }
   ],
   "source": [
    "word_counts = {}\n",
    "for lang_code in tqdm.tqdm(lang_codes):\n",
    "#     if lang_code in word_counts:\n",
    "#         continue\n",
    "    word_counts[lang_code] = []\n",
    "    with open(os.path.join(BASE_DIR, lang_code, f\"{lang_code}_full.txt\")) as f:\n",
    "        s = f.read()\n",
    "    lines = s.strip().split(\"\\n\")\n",
    "    # skip most common words\n",
    "    for line in lines:\n",
    "        word, n_count = line.split()\n",
    "        n_count = int(n_count)\n",
    "        if n_count >= 3:\n",
    "            word_counts[lang_code].append((word, n_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "eb17870c",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"/data/suno/data/FrequencyWords/word_counts.json\", \"w\") as f:\n",
    "    json.dump(word_counts, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89cf2fa9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f84014ee",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "625dd5bd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "d32bc1f0",
   "metadata": {},
   "source": [
    "# get full HF dataset (apache beam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f88cb0a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: this aborted, probably OOM\n",
    "# https://huggingface.co/datasets/wikipedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "03077499",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"HF_DATASETS_CACHE\"] = \"/data/suno/data/huggingface-datasets\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a90dac32",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "# languages here: https://meta.wikimedia.org/wiki/List_of_Wikipedias\n",
    "out = load_dataset(\"wikipedia\", language=\"en\", date=\"20221001\", beam_runner=\"DirectRunner\", split=\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1eeb9ed2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from datasets import load_dataset\n",
    "# def load_and_clean_wiki():\n",
    "#     dataset = load_dataset('wiki40b', 'sv', beam_runner='DirectRunner', split=\"train\")\n",
    "#     #dataset = load_dataset('wiki40b', 'sv', beam_runner='DirectRunner')\n",
    "#     dataset = dataset.remove_columns(['wikidata_id', 'version_id'])\n",
    "#     filtered_dataset = dataset.map(filter_wikipedia)\n",
    "#     # filtered_dataset[:3]\n",
    "#     # print(filtered_dataset[:3])\n",
    "#     return filtered_dataset\n",
    "\n",
    "# def filter_wikipedia(batch):\n",
    "#     batch[\"text\"] = \" \".join(batch[\"text\"].split(\"\\\n",
    "# _START_SECTION_\\\n",
    "# \"))\n",
    "#     batch[\"text\"] = \" \".join(batch[\"text\"].split(\"\\\n",
    "# _START_ARTICLE_\\\n",
    "# \"))\n",
    "#     batch[\"text\"] = \" \".join(batch[\"text\"].split(\"\\\n",
    "# _START_ARTICLE_\\\n",
    "# \"))\n",
    "#     batch[\"text\"] = \" \".join(batch[\"text\"].split(\"\\\n",
    "# _START_PARAGRAPH_\\\n",
    "# \"))\n",
    "#     batch[\"text\"] = \" \".join(batch[\"text\"].split(\"_NEWLINE_\"))\n",
    "#     batch[\"text\"] = \" \".join(batch[\"text\"].split(\"\\xa0\"))\n",
    "#     return batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c519251",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80a3dc46",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b32bc448",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "eab52792",
   "metadata": {},
   "source": [
    "## tensorflow wiki40b dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f14769ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-10 03:08:18.022578: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow_datasets as tfds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0d8217c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "WIKIPEDIA_LANGUAGES = [\n",
    "    \"en\", \"ar\", \"zh-cn\", \"zh-tw\", \"nl\", \"fr\", \"de\", \"it\", \"ja\", \"ko\", \"pl\",\n",
    "    \"pt\", \"ru\", \"es\", \"th\", \"tr\", \"bg\", \"ca\", \"cs\", \"da\", \"el\", \"et\", \"fa\",\n",
    "    \"fi\", \"he\", \"hi\", \"hr\", \"hu\", \"id\", \"lt\", \"lv\", \"ms\", \"no\", \"ro\", \"sk\",\n",
    "    \"sl\", \"sr\", \"sv\", \"tl\", \"uk\", \"vi\"\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6885b706",
   "metadata": {},
   "outputs": [],
   "source": [
    "/home/georg/tensorflow_datasets/wiki40b/en/1.3.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "875052ce",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "load() takes 1 positional argument but 2 positional arguments (and 2 keyword-only arguments) were given",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_526937/1603562771.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtfds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"wiki40b\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"en\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msplit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"train\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"/data/suno/data/tensorflow_datasets\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow_datasets/core/logging/__init__.py\u001b[0m in \u001b[0;36mdecorator\u001b[0;34m(function, unused_none_instance, args, kwargs)\u001b[0m\n\u001b[1;32m    248\u001b[0m     \u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0margs\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"name\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    249\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 250\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    251\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    252\u001b[0m       \u001b[0mmetadata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmark_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: load() takes 1 positional argument but 2 positional arguments (and 2 keyword-only arguments) were given"
     ]
    }
   ],
   "source": [
    "ds = tfds.load(\"wiki40b/en\", split=\"train\", data_dir=\"/data/suno/data/tensorflow_datasets\", bea)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18163e86",
   "metadata": {},
   "outputs": [],
   "source": [
    "https://huggingface.co/datasets/wikipedia\n",
    "    https://www.tensorflow.org/datasets/catalog/wiki40b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8e97a977",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting datasets\n",
      "  Downloading datasets-2.5.2-py3-none-any.whl (432 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m432.7/432.7 kB\u001b[0m \u001b[31m45.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: dill<0.3.6 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from datasets) (0.3.1.1)\n",
      "Requirement already satisfied: aiohttp in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from datasets) (3.8.1)\n",
      "Requirement already satisfied: fsspec[http]>=2021.11.1 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from datasets) (2022.7.1)\n",
      "Requirement already satisfied: pandas in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from datasets) (1.4.4)\n",
      "Collecting responses<0.19\n",
      "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
      "Requirement already satisfied: requests>=2.19.0 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from datasets) (2.28.1)\n",
      "Requirement already satisfied: pyarrow>=6.0.0 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from datasets) (7.0.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from datasets) (4.64.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from datasets) (1.21.5)\n",
      "Collecting xxhash\n",
      "  Downloading xxhash-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.1/212.1 kB\u001b[0m \u001b[31m38.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting huggingface-hub<1.0.0,>=0.2.0\n",
      "  Downloading huggingface_hub-0.10.0-py3-none-any.whl (163 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m163.5/163.5 kB\u001b[0m \u001b[31m30.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting multiprocess\n",
      "  Downloading multiprocess-0.70.13-py38-none-any.whl (131 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m131.4/131.4 kB\u001b[0m \u001b[31m27.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: packaging in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from datasets) (21.3)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (4.3.0)\n",
      "Requirement already satisfied: filelock in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (3.6.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (6.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from packaging->datasets) (3.0.9)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (3.3)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (1.26.11)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (2022.9.14)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from aiohttp->datasets) (5.2.0)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from aiohttp->datasets) (1.8.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from aiohttp->datasets) (21.4.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from aiohttp->datasets) (1.2.0)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from aiohttp->datasets) (4.0.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from aiohttp->datasets) (1.2.0)\n",
      "Collecting dill<0.3.6\n",
      "  Downloading dill-0.3.5.1-py2.py3-none-any.whl (95 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m95.8/95.8 kB\u001b[0m \u001b[31m21.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: python-dateutil>=2.8.1 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from pandas->datasets) (2022.1)\n",
      "Requirement already satisfied: six>=1.5 in /home/georg/anaconda3/envs/ml/lib/python3.8/site-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
      "Installing collected packages: xxhash, dill, responses, multiprocess, huggingface-hub, datasets\n",
      "  Attempting uninstall: dill\n",
      "    Found existing installation: dill 0.3.1.1\n",
      "    Uninstalling dill-0.3.1.1:\n",
      "      Successfully uninstalled dill-0.3.1.1\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "apache-beam 2.41.0 requires dill<0.3.2,>=0.3.1.1, but you have dill 0.3.5.1 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed datasets-2.5.2 dill-0.3.5.1 huggingface-hub-0.10.0 multiprocess-0.70.13 responses-0.18.0 xxhash-3.0.0\n"
     ]
    }
   ],
   "source": [
    "!pip install datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "534c3aea",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "125807f3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed884110",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0191ed4c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51aeba55",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}