{ "cells": [ { "cell_type": "markdown", "id": "2bdac5a0", "metadata": {}, "source": [ "### Semantic Prep" ] }, { "cell_type": "code", "execution_count": null, "id": "c8d7c3dc", "metadata": {}, "outputs": [], "source": [ "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'mert_25' \\\n", " --data-type 'youtube_music' \\\n", " --version 'v1' \\\n", " --chunksize '500' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '7200' \\\n", " --output-name 'mert_25_2x4k'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'mert_25' \\\n", " --data-type 'genius_hq' \\\n", " --version 'v1' \\\n", " --chunksize '500' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '480' \\\n", " --output-name 'mert_25_2x4k'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'mert_25' \\\n", " --data-type 'freesound' \\\n", " --version 'v1' \\\n", " --chunksize '500' \\\n", " --min-duration-s '4' \\\n", " --max-duration-s '480' \\\n", " --output-name 'mert_25_2x4k'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'mert_25' \\\n", " --data-type 'jamendo' \\\n", " --version 'v1' \\\n", " --chunksize '500' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '7200' \\\n", " --output-name 'mert_25_2x4k'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'mert_25' \\\n", " --data-type 'imslp' \\\n", " --version 'v1' \\\n", " --chunksize '500' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '7200' \\\n", " --output-name 'mert_25_2x4k'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'mert_25' \\\n", " --data-type 'pond5_music' \\\n", " --version 'v2' \\\n", " --chunksize '500' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '800' \\\n", " --output-name 'mert_25_2x4k'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'mert_25' \\\n", " --data-type 'deezer' \\\n", " --version 'v2' \\\n", " --chunksize '500' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '800' \\\n", " --output-name 'mert_25_2x4k'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'mert_25' \\\n", " --data-type 'ytm_tagged' \\\n", " --version 'v2' \\\n", " --chunksize '500' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '800' \\\n", " --output-name 'mert_25_2x4k'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'mert_25' \\\n", " --data-type 'musescore' \\\n", " --version 'v2' \\\n", " --chunksize '100' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '800' \\\n", " --output-name 'mert_25_2x4k'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'mert_25' \\\n", " --data-type 'ytm_mb' \\\n", " --version 'v2' \\\n", " --chunksize '500' \\\n", " --min-duration-s '60' \\\n", " --max-duration-s '480' \\\n", " --output-name 'mert_25_2x4k'\n", "\n", "# --first-only True \\\n", "# --force-overwrite True \\\n", "# --relax-len-check True \\" ] }, { "cell_type": "code", "execution_count": null, "id": "db71997f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "53e2fe31", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "9a66d9c6", "metadata": {}, "source": [ "### Codec Prep" ] }, { "cell_type": "code", "execution_count": null, "id": "5b55cecc", "metadata": {}, "outputs": [], "source": [ "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'dac_2c' \\\n", " --data-type 'youtube_music' \\\n", " --version 'v1' \\\n", " --chunksize '500' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '7200' \\\n", " --output-name 'dac_2c_25_8'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'dac_2c' \\\n", " --data-type 'genius_hq' \\\n", " --version 'v1' \\\n", " --chunksize '500' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '480' \\\n", " --output-name 'dac_2c_25_8'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'dac_2c' \\\n", " --data-type 'freesound' \\\n", " --version 'v1' \\\n", " --chunksize '500' \\\n", " --min-duration-s '4' \\\n", " --max-duration-s '480' \\\n", " --output-name 'dac_2c_25_8'\n", "\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'dac_2c' \\\n", " --data-type 'jamendo' \\\n", " --version 'v1' \\\n", " --chunksize '500' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '7200' \\\n", " --output-name 'dac_2c_25_8'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'dac_2c' \\\n", " --data-type 'imslp' \\\n", " --version 'v1' \\\n", " --chunksize '500' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '7200' \\\n", " --output-name 'dac_2c_25_8'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'dac_2c' \\\n", " --data-type 'pond5_music' \\\n", " --version 'v2' \\\n", " --chunksize '500' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '800' \\\n", " --output-name 'dac_2c_25_8'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'dac_2c' \\\n", " --data-type 'deezer' \\\n", " --version 'v2' \\\n", " --chunksize '500' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '800' \\\n", " --output-name 'dac_2c_25_8'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'dac_2c' \\\n", " --data-type 'ytm_tagged' \\\n", " --version 'v2' \\\n", " --chunksize '500' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '800' \\\n", " --output-name 'dac_2c_25_8'\n", "\n", "modal run ~/code/glockenspiel/suno_utils/suno_utils/scripts/gpt/modal_encode.py \\\n", " --embed-type 'dac_2c' \\\n", " --data-type 'musescore' \\\n", " --version 'v2' \\\n", " --chunksize '100' \\\n", " --min-duration-s '30' \\\n", " --max-duration-s '800' \\\n", " --output-name 'dac_2c_25_8'\n", "\n", "# --first-only True \\\n", "# --force-overwrite True \\\n", "# --relax-len-check True \\" ] }, { "cell_type": "code", "execution_count": null, "id": "32cdefd7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a5188609", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "28e4d3d5", "metadata": {}, "source": [ "## Misc" ] }, { "cell_type": "code", "execution_count": 1, "id": "00a53193", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " PRE audioset/\r\n", " PRE fma/\r\n", " PRE freesound/\r\n", " PRE genius_hq/\r\n", " PRE imslp/\r\n", " PRE jamendo/\r\n", " PRE models/\r\n", " PRE youtube_music/\r\n" ] } ], "source": [ "!aws s3 ls s3://suno-data/datasets/bundles/v1/" ] }, { "cell_type": "code", "execution_count": 4, "id": "0b8aa951", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " PRE d2v_test/\r\n", " PRE encodec/\r\n", " PRE hoot_logits/\r\n", " PRE mert_4x1k/\r\n", " PRE wavlm_8x10k/\r\n", "2023-06-10 21:00:37 1606253863 aligned_lyrics.jsonl\r\n", "2023-06-17 16:34:12 1476134780 aligned_lyrics_2.jsonl\r\n", "2023-06-03 20:20:28 5368870173 metas.jsonl\r\n", "2023-06-06 08:44:26 114266015 metas_encodec.jsonl\r\n", "2023-07-18 10:11:15 4979087277 metas_plus.jsonl\r\n", "2023-06-06 04:59:26 113489049 metas_wavlm_8x10k.jsonl\r\n", "2023-06-08 08:41:27 37207 tag_ids.jsonl\r\n" ] } ], "source": [ "!aws s3 ls s3://suno-data/datasets/bundles/v1/genius_hq/" ] }, { "cell_type": "code", "execution_count": null, "id": "6d283a15", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "49ff0dfe", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e00ad12d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "07692520", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "13b15e39", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": null, "id": "760eda7b", "metadata": {}, "outputs": [], "source": [ "# check max duration and put a metas.jsonl into bundles" ] }, { "cell_type": "code", "execution_count": 12, "id": "d050017e", "metadata": {}, "outputs": [], "source": [ "new_metas = [\n", " {\n", " \"id\": m[\"id\"],\n", " \"s3_filepath\": m[\"s3_filepath\"],\n", " \"duration_s\": m[\"duration_s\"],\n", " } for m in metas if \"s3_filepath\" in m\n", "]" ] }, { "cell_type": "code", "execution_count": 15, "id": "3092ce9b", "metadata": {}, "outputs": [], "source": [ "write_jsonl(new_metas, \"pond5_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 17, "id": "1e3d99c9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: ./pond5_metas.jsonl to s3://suno-data/datasets/bundles/v2/pond5_music/metas.jsonl\n" ] } ], "source": [ "!aws s3 cp pond5_metas.jsonl s3://suno-data/datasets/bundles/v2/pond5_music/metas.jsonl" ] }, { "cell_type": "code", "execution_count": 19, "id": "54098c9d", "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 20, "id": "7a0abfa2", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAGdCAYAAAD+JxxnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAu4klEQVR4nO3deXCUdZ7H8U8Skk4idMJh0mQJkFFXQC4NEuO1KCENZl0R1hJkHUSEgk1cQ3ZAcCFy6MbBkUONpmaUY0pYkaoRFdhAGwaQIYBEI5cwHlg4Ix0cMTSHJk3y7B9TeZYmmANytPm9X1Up6ef55vf8+vkm8PG5OsSyLEsAAAAGCm3tCQAAALQWghAAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFjtWnsCway6ulrffPONOnTooJCQkNaeDgAAaADLsnT69GklJCQoNLTuYz4EoTp88803SkxMbO1pAACAy/D111+rW7duddYQhOrQoUMHSX/fkU6ns0nH9vv92rx5s9LT0xUeHt6kY6Px6EdwoR/BhX4EH3pSN5/Pp8TERPvf8boQhOpQczrM6XQ2SxCKjo6W0+nkhzgI0I/gQj+CC/0IPvSkYRpyWQsXSwMAAGMRhAAAgLEIQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYq11rT8B0feduUkVVyE+u/+q5jBacDQAAZuGIEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFgEIQAAYKx2rT0B1K3nzA311nz1XEYLzAQAgLaHI0IAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMZqVBDKy8vTzTffrA4dOiguLk4jR47UkSNHAmqGDBmikJCQgK8pU6YE1Bw7dkwZGRmKjo5WXFycpk+frvPnzwfUbN26VTfddJMcDoeuvfZarVixotZ88vPz1bNnT0VGRiolJUV79uwJWP/jjz8qMzNTnTt3Vvv27TV69GiVlZU15i0DAIA2rFFBaNu2bcrMzNSuXbvk8Xjk9/uVnp6us2fPBtRNmjRJx48ft78WLlxor6uqqlJGRoYqKyu1c+dOrVy5UitWrFBubq5dc/ToUWVkZOiuu+5SaWmpsrOz9dhjj2nTpk12zZo1a5STk6Onn35aH330kQYMGCC3260TJ07YNdOmTdN7772ntWvXatu2bfrmm280atSoRu8kAADQNjXqgYqFhYUBr1esWKG4uDiVlJTozjvvtJdHR0fL5XJdcozNmzfr0KFDev/99xUfH6+BAwdqwYIFevLJJzV37lxFRESooKBASUlJeuGFFyRJvXv31o4dO7R48WK53W5J0qJFizRp0iRNmDBBklRQUKANGzZo2bJlmjlzpk6dOqXXX39dq1ev1t133y1JWr58uXr37q1du3bplltuacxbBwAAbdAVPVn61KlTkqROnToFLF+1apXeeOMNuVwu3XvvvZozZ46io6MlScXFxerXr5/i4+PterfbralTp+rgwYO68cYbVVxcrLS0tIAx3W63srOzJUmVlZUqKSnRrFmz7PWhoaFKS0tTcXGxJKmkpER+vz9gnF69eql79+4qLi6+ZBCqqKhQRUWF/drn80mS/H6//H5/o/dPXWrGc4RaTTYWLl/NPmRfBgf6EVzoR/ChJ3VrzH657CBUXV2t7Oxs3Xbbberbt6+9/KGHHlKPHj2UkJCgffv26cknn9SRI0f0hz/8QZLk9XoDQpAk+7XX662zxufz6YcfftD333+vqqqqS9YcPnzYHiMiIkKxsbG1amq2c7G8vDzNmzev1vLNmzfbQa6pLRhUfcVjbNy4sQlmAknyeDytPQVcgH4EF/oRfOjJpZ07d67BtZcdhDIzM3XgwAHt2LEjYPnkyZPtP/fr109du3bV0KFD9cUXX+iaa6653M21iFmzZiknJ8d+7fP5lJiYqPT0dDmdzibdlt/vl8fj0Zy9oaqoDrmisQ7MdTfRrMxV049hw4YpPDy8tadjPPoRXOhH8KEndas5o9MQlxWEsrKytH79em3fvl3dunWrszYlJUWS9Pnnn+uaa66Ry+WqdXdXzZ1cNdcVuVyuWnd3lZWVyel0KioqSmFhYQoLC7tkzYVjVFZWqry8POCo0IU1F3M4HHI4HLWWh4eHN9sPWkV1iCqqriwI8UvQdJqz12g8+hFc6EfwoSeX1ph90qi7xizLUlZWlt5++21t2bJFSUlJ9X5PaWmpJKlr166SpNTUVO3fvz/g7i6PxyOn06k+ffrYNUVFRQHjeDwepaamSpIiIiKUnJwcUFNdXa2ioiK7Jjk5WeHh4QE1R44c0bFjx+waAABgtkYdEcrMzNTq1av1zjvvqEOHDva1NjExMYqKitIXX3yh1atX65577lHnzp21b98+TZs2TXfeeaf69+8vSUpPT1efPn308MMPa+HChfJ6vZo9e7YyMzPtozFTpkzRyy+/rBkzZujRRx/Vli1b9NZbb2nDhg32XHJycjR+/HgNGjRIgwcP1pIlS3T27Fn7LrKYmBhNnDhROTk56tSpk5xOpx5//HGlpqZyxxgAAJDUyCD06quvSvr7QxMvtHz5cj3yyCOKiIjQ+++/b4eSxMREjR49WrNnz7Zrw8LCtH79ek2dOlWpqam66qqrNH78eM2fP9+uSUpK0oYNGzRt2jQtXbpU3bp102uvvWbfOi9JDz74oL799lvl5ubK6/Vq4MCBKiwsDLiAevHixQoNDdXo0aNVUVEht9utV155pVE7CAAAtF2NCkKWVfet3omJidq2bVu94/To0aPeO52GDBmijz/+uM6arKwsZWVl/eT6yMhI5efnKz8/v945AQAA8/BZYwAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIzVrrUngJbRc+aGemu+ei6jBWYCAEDw4IgQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxGhWE8vLydPPNN6tDhw6Ki4vTyJEjdeTIkYCaH3/8UZmZmercubPat2+v0aNHq6ysLKDm2LFjysjIUHR0tOLi4jR9+nSdP38+oGbr1q266aab5HA4dO2112rFihW15pOfn6+ePXsqMjJSKSkp2rNnT6PnAgAAzNWoILRt2zZlZmZq165d8ng88vv9Sk9P19mzZ+2aadOm6b333tPatWu1bds2ffPNNxo1apS9vqqqShkZGaqsrNTOnTu1cuVKrVixQrm5uXbN0aNHlZGRobvuukulpaXKzs7WY489pk2bNtk1a9asUU5Ojp5++ml99NFHGjBggNxut06cONHguQAAALM16oGKhYWFAa9XrFihuLg4lZSU6M4779SpU6f0+uuva/Xq1br77rslScuXL1fv3r21a9cu3XLLLdq8ebMOHTqk999/X/Hx8Ro4cKAWLFigJ598UnPnzlVERIQKCgqUlJSkF154QZLUu3dv7dixQ4sXL5bb7ZYkLVq0SJMmTdKECRMkSQUFBdqwYYOWLVummTNnNmguAADAbFd0jdCpU6ckSZ06dZIklZSUyO/3Ky0tza7p1auXunfvruLiYklScXGx+vXrp/j4eLvG7XbL5/Pp4MGDds2FY9TU1IxRWVmpkpKSgJrQ0FClpaXZNQ2ZCwAAMNtlf8RGdXW1srOzddttt6lv376SJK/Xq4iICMXGxgbUxsfHy+v12jUXhqCa9TXr6qrx+Xz64Ycf9P3336uqquqSNYcPH27wXC5WUVGhiooK+7XP55Mk+f1++f3+OvdHY9WM5wi1mmysujjC6t9OU7/Hn5Oa927yPggm9CO40I/gQ0/q1pj9ctlBKDMzUwcOHNCOHTsud4igk5eXp3nz5tVavnnzZkVHRzfLNhcMqr7iMTZu3FhvzcLBTTNOW+fxeFp7CrgA/Qgu9CP40JNLO3fuXINrLysIZWVlaf369dq+fbu6detmL3e5XKqsrFR5eXnAkZiysjK5XC675uK7u2ru5Lqw5uK7u8rKyuR0OhUVFaWwsDCFhYVdsubCMeqby8VmzZqlnJwc+7XP51NiYqLS09PldDobsmsazO/3y+PxaM7eUFVUh1zRWAfmuuut6Tt3U701DRmnrarpx7BhwxQeHt7a0zEe/Qgu9CP40JO61ZzRaYhGBSHLsvT444/r7bff1tatW5WUlBSwPjk5WeHh4SoqKtLo0aMlSUeOHNGxY8eUmpoqSUpNTdWzzz6rEydOKC4uTtLfE63T6VSfPn3smouPTng8HnuMiIgIJScnq6ioSCNHjpT091N1RUVFysrKavBcLuZwOORwOGotDw8Pb7YftIrqEFVUXVkQasjcGrINfpmat9doPPoRXOhH8KEnl9aYfdKoIJSZmanVq1frnXfeUYcOHexrbWJiYhQVFaWYmBhNnDhROTk56tSpk5xOpx5//HGlpqbad2mlp6erT58+evjhh7Vw4UJ5vV7Nnj1bmZmZdgiZMmWKXn75Zc2YMUOPPvqotmzZorfeeksbNmyw55KTk6Px48dr0KBBGjx4sJYsWaKzZ8/ad5E1ZC4AAMBsjQpCr776qiRpyJAhAcuXL1+uRx55RJK0ePFihYaGavTo0aqoqJDb7dYrr7xi14aFhWn9+vWaOnWqUlNTddVVV2n8+PGaP3++XZOUlKQNGzZo2rRpWrp0qbp166bXXnvNvnVekh588EF9++23ys3Nldfr1cCBA1VYWBhwAXV9cwEAAGZr9Kmx+kRGRio/P1/5+fk/WdOjR496L8wdMmSIPv744zprsrKy7FNhlzsXAABgLj5rDAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgrMv+0FXgSvScuaHemq+ey2iBmQAATMYRIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMZqdBDavn277r33XiUkJCgkJETr1q0LWP/II48oJCQk4Gv48OEBNSdPntS4cePkdDoVGxuriRMn6syZMwE1+/bt0x133KHIyEglJiZq4cKFteaydu1a9erVS5GRkerXr582btwYsN6yLOXm5qpr166KiopSWlqaPvvss8a+ZQAA0Ea1a+w3nD17VgMGDNCjjz6qUaNGXbJm+PDhWr58uf3a4XAErB83bpyOHz8uj8cjv9+vCRMmaPLkyVq9erUkyefzKT09XWlpaSooKND+/fv16KOPKjY2VpMnT5Yk7dy5U2PHjlVeXp7++Z//WatXr9bIkSP10UcfqW/fvpKkhQsX6sUXX9TKlSuVlJSkOXPmyO1269ChQ4qMjGzsW0cD9Zy5obWnAABAgzQ6CI0YMUIjRoyos8bhcMjlcl1y3aeffqrCwkJ9+OGHGjRokCTppZde0j333KPf/OY3SkhI0KpVq1RZWally5YpIiJCN9xwg0pLS7Vo0SI7CC1dulTDhw/X9OnTJUkLFiyQx+PRyy+/rIKCAlmWpSVLlmj27Nm67777JEm///3vFR8fr3Xr1mnMmDGNfesAAKCNaXQQaoitW7cqLi5OHTt21N13361nnnlGnTt3liQVFxcrNjbWDkGSlJaWptDQUO3evVv333+/iouLdeeddyoiIsKucbvd+vWvf63vv/9eHTt2VHFxsXJycgK263a77VN1R48eldfrVVpamr0+JiZGKSkpKi4uvmQQqqioUEVFhf3a5/NJkvx+v/x+/5XvmAvUjOcItZpsrLo4wurfTlO9x4ZsqyGu/6/19dYcmOtukm3VvPem7jMuD/0ILvQj+NCTujVmvzR5EBo+fLhGjRqlpKQkffHFF3rqqac0YsQIFRcXKywsTF6vV3FxcYGTaNdOnTp1ktfrlSR5vV4lJSUF1MTHx9vrOnbsKK/Xay+7sObCMS78vkvVXCwvL0/z5s2rtXzz5s2Kjo5u6C5olAWDqq94jIuvjbqUhYObZpyGaMi2mkpTzbmGx+Np0vFwZehHcKEfwYeeXNq5c+caXNvkQejCIy39+vVT//79dc0112jr1q0aOnRoU2+uSc2aNSvgKJPP51NiYqLS09PldDqbdFt+v18ej0dz9oaqojrkisZqyFGRvnM3Nck4DdGQbTWVpjwi5PF4NGzYMIWHhzfJmLh89CO40I/gQ0/qVnNGpyGa5dTYhX7xi1+oS5cu+vzzzzV06FC5XC6dOHEioOb8+fM6efKkfV2Ry+VSWVlZQE3N6/pqLlxfs6xr164BNQMHDrzkXB0OR60LuyUpPDy82X7QKqpDVFF1ZUGoIXNryDaa6j1e6ftpjKbuS3P2Go1HP4IL/Qg+9OTSGrNPmv05Qn/5y1/03Xff2WEkNTVV5eXlKikpsWu2bNmi6upqpaSk2DXbt28POMfn8Xh0/fXXq2PHjnZNUVFRwLY8Ho9SU1MlSUlJSXK5XAE1Pp9Pu3fvtmsAAIDZGh2Ezpw5o9LSUpWWlkr6+0XJpaWlOnbsmM6cOaPp06dr165d+uqrr1RUVKT77rtP1157rdzuv5/C6N27t4YPH65JkyZpz549+tOf/qSsrCyNGTNGCQkJkqSHHnpIERERmjhxog4ePKg1a9Zo6dKlAaetnnjiCRUWFuqFF17Q4cOHNXfuXO3du1dZWVmSpJCQEGVnZ+uZZ57Ru+++q/379+uXv/ylEhISNHLkyCvcbQAAoC1o9KmxvXv36q677rJf14ST8ePH69VXX9W+ffu0cuVKlZeXKyEhQenp6VqwYEHAKadVq1YpKytLQ4cOVWhoqEaPHq0XX3zRXh8TE6PNmzcrMzNTycnJ6tKli3Jzc+1b5yXp1ltv1erVqzV79mw99dRTuu6667Ru3Tr7GUKSNGPGDJ09e1aTJ09WeXm5br/9dhUWFvIMIQAAIOkygtCQIUNkWT99e/SmTfVfKNupUyf74Yk/pX///vrggw/qrHnggQf0wAMP/OT6kJAQzZ8/X/Pnz693TgAAwDx81hgAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIzVrrUngJ+XnjM3tPYUAABoMhwRAgAAxuKIEGwc7QEAmIYjQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxuL2+TaA294BALg8HBECAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMbiydL4WWvIU7W/ei6jBWYCAPg54ogQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjNToIbd++Xffee68SEhIUEhKidevWBay3LEu5ubnq2rWroqKilJaWps8++yyg5uTJkxo3bpycTqdiY2M1ceJEnTlzJqBm3759uuOOOxQZGanExEQtXLiw1lzWrl2rXr16KTIyUv369dPGjRsbPRcAAGCuRgehs2fPasCAAcrPz7/k+oULF+rFF19UQUGBdu/erauuukput1s//vijXTNu3DgdPHhQHo9H69ev1/bt2zV58mR7vc/nU3p6unr06KGSkhI9//zzmjt3rn7729/aNTt37tTYsWM1ceJEffzxxxo5cqRGjhypAwcONGouAADAXI1+jtCIESM0YsSIS66zLEtLlizR7Nmzdd9990mSfv/73ys+Pl7r1q3TmDFj9Omnn6qwsFAffvihBg0aJEl66aWXdM899+g3v/mNEhIStGrVKlVWVmrZsmWKiIjQDTfcoNLSUi1atMgOTEuXLtXw4cM1ffp0SdKCBQvk8Xj08ssvq6CgoEFzAQAAZmvSByoePXpUXq9XaWlp9rKYmBilpKSouLhYY8aMUXFxsWJjY+0QJElpaWkKDQ3V7t27df/996u4uFh33nmnIiIi7Bq3261f//rX+v7779WxY0cVFxcrJycnYPtut9s+VdeQuVysoqJCFRUV9mufzydJ8vv98vv9V7ZzLlIzniPUatJxUVtDeldT09R9xuWhH8GFfgQfelK3xuyXJg1CXq9XkhQfHx+wPD4+3l7n9XoVFxcXOIl27dSpU6eAmqSkpFpj1Kzr2LGjvF5vvdupby4Xy8vL07x582ot37x5s6Kjo3/iXV+ZBYOqm2Vc/L+Lrx2ri8fjacaZoLHoR3ChH8GHnlzauXPnGlzLR2xcYNasWQFHmXw+nxITE5Weni6n09mk2/L7/fJ4PJqzN1QV1SFNOjYCHZjrrremph/Dhg1TeHh4C8wKdaEfwYV+BB96UreaMzoN0aRByOVySZLKysrUtWtXe3lZWZkGDhxo15w4cSLg+86fP6+TJ0/a3+9yuVRWVhZQU/O6vpoL19c3l4s5HA45HI5ay8PDw5vtB62iOkQVVQSh5tSY3jVnr9F49CO40I/gQ08urTH7pEmfI5SUlCSXy6WioiJ7mc/n0+7du5WamipJSk1NVXl5uUpKSuyaLVu2qLq6WikpKXbN9u3bA87xeTweXX/99erYsaNdc+F2ampqttOQuQAAALM1OgidOXNGpaWlKi0tlfT3i5JLS0t17NgxhYSEKDs7W88884zeffdd7d+/X7/85S+VkJCgkSNHSpJ69+6t4cOHa9KkSdqzZ4/+9Kc/KSsrS2PGjFFCQoIk6aGHHlJERIQmTpyogwcPas2aNVq6dGnAaasnnnhChYWFeuGFF3T48GHNnTtXe/fuVVZWliQ1aC4AAMBsjT41tnfvXt11113265pwMn78eK1YsUIzZszQ2bNnNXnyZJWXl+v2229XYWGhIiMj7e9ZtWqVsrKyNHToUIWGhmr06NF68cUX7fUxMTHavHmzMjMzlZycrC5duig3NzfgWUO33nqrVq9erdmzZ+upp57Sddddp3Xr1qlv3752TUPmAgAAzNXoIDRkyBBZ1k/f8h0SEqL58+dr/vz5P1nTqVMnrV69us7t9O/fXx988EGdNQ888IAeeOCBK5oLAAAwF581BgAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGavRHbAA/Nz1nbqi35rMF6S0wEwBAsOGIEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADG4kNXAUl9527SwsF//29FVcgla756LqOFZwUAaG4cEQIAAMYiCAEAAGMRhAAAgLEIQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFhNHoTmzp2rkJCQgK9evXrZ63/88UdlZmaqc+fOat++vUaPHq2ysrKAMY4dO6aMjAxFR0crLi5O06dP1/nz5wNqtm7dqptuukkOh0PXXnutVqxYUWsu+fn56tmzpyIjI5WSkqI9e/Y09dsFAAA/Y81yROiGG27Q8ePH7a8dO3bY66ZNm6b33ntPa9eu1bZt2/TNN99o1KhR9vqqqiplZGSosrJSO3fu1MqVK7VixQrl5ubaNUePHlVGRobuuusulZaWKjs7W4899pg2bdpk16xZs0Y5OTl6+umn9dFHH2nAgAFyu906ceJEc7xlAADwM9QsQahdu3ZyuVz2V5cuXSRJp06d0uuvv65Fixbp7rvvVnJyspYvX66dO3dq165dkqTNmzfr0KFDeuONNzRw4ECNGDFCCxYsUH5+viorKyVJBQUFSkpK0gsvvKDevXsrKytL//qv/6rFixfbc1i0aJEmTZqkCRMmqE+fPiooKFB0dLSWLVvWHG8ZAAD8DLVrjkE/++wzJSQkKDIyUqmpqcrLy1P37t1VUlIiv9+vtLQ0u7ZXr17q3r27iouLdcstt6i4uFj9+vVTfHy8XeN2uzV16lQdPHhQN954o4qLiwPGqKnJzs6WJFVWVqqkpESzZs2y14eGhiotLU3FxcU/Oe+KigpVVFTYr30+nyTJ7/fL7/df0T65WM14jlCrScfF5anpQ139aOqfAfy0mn3NPg8O9CP40JO6NWa/NHkQSklJ0YoVK3T99dfr+PHjmjdvnu644w4dOHBAXq9XERERio2NDfie+Ph4eb1eSZLX6w0IQTXra9bVVePz+fTDDz/o+++/V1VV1SVrDh8+/JNzz8vL07x582ot37x5s6Kjoxu2AxppwaDqZhkXl6eufmzcuLEFZwJJ8ng8rT0FXIB+BB96cmnnzp1rcG2TB6ERI0bYf+7fv79SUlLUo0cPvfXWW4qKimrqzTWpWbNmKScnx37t8/mUmJio9PR0OZ3OJt2W3++Xx+PRnL2hqqgOadKx0XiOUEsLBlXX2Y8Dc90tPCtz1fx+DBs2TOHh4a09HePRj+BDT+pWc0anIZrl1NiFYmNj9Y//+I/6/PPPNWzYMFVWVqq8vDzgqFBZWZlcLpckyeVy1bq7q+ausgtrLr7TrKysTE6nU1FRUQoLC1NYWNgla2rGuBSHwyGHw1FreXh4eLP9oFVUh6iiiiAULOrqB3/ZtLzm/N1D49GP4ENPLq0x+6TZnyN05swZffHFF+ratauSk5MVHh6uoqIie/2RI0d07NgxpaamSpJSU1O1f//+gLu7PB6PnE6n+vTpY9dcOEZNTc0YERERSk5ODqiprq5WUVGRXQMAANDkQehXv/qVtm3bpq+++ko7d+7U/fffr7CwMI0dO1YxMTGaOHGicnJy9Mc//lElJSWaMGGCUlNTdcstt0iS0tPT1adPHz388MP65JNPtGnTJs2ePVuZmZn20ZopU6boyy+/1IwZM3T48GG98soreuuttzRt2jR7Hjk5Ofrd736nlStX6tNPP9XUqVN19uxZTZgwoanfMgAA+Jlq8lNjf/nLXzR27Fh99913uvrqq3X77bdr165duvrqqyVJixcvVmhoqEaPHq2Kigq53W698sor9veHhYVp/fr1mjp1qlJTU3XVVVdp/Pjxmj9/vl2TlJSkDRs2aNq0aVq6dKm6deum1157TW73/1/D8eCDD+rbb79Vbm6uvF6vBg4cqMLCwloXUAMAAHM1eRB6880361wfGRmp/Px85efn/2RNjx496r1DZ8iQIfr444/rrMnKylJWVladNQAAwFzNfrE00Fb0nLmh3pqvnstogZkAAJoKH7oKAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIzFk6WBJsTTpwHg54UjQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMxafPAy2MT6gHgODBESEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFg8UBEIQjx0EQBaBkeEAACAsQhCAADAWJwaA36mOH0GAFeOI0IAAMBYBCEAAGAsghAAADAW1wgBAC6p5jo0R5ilhYOlvnM3qaIqpFYd16Lh54wjQgAAwFgcEQLaMO4sA4C6cUQIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxuGsMMBx3lgEwGUeEAACAsTgiBKBeHDUC0FYRhAA0CcISgJ8jI4JQfn6+nn/+eXm9Xg0YMEAvvfSSBg8e3NrTAoxDWAIQbNp8EFqzZo1ycnJUUFCglJQULVmyRG63W0eOHFFcXFxrTw/ARXrO3FDvh3w2BIEKQEO0+SC0aNEiTZo0SRMmTJAkFRQUaMOGDVq2bJlmzpzZyrMD0Fw4+lS3huwfwARtOghVVlaqpKREs2bNspeFhoYqLS1NxcXFteorKipUUVFhvz516pQk6eTJk/L7/U06N7/fr3PnzqmdP1RV1Zf3f7xoOu2qLZ07V00/gkRL9ePaX73VbGMHu8b85V9fPxqyH3fPGtqILaI+Nf+GfPfddwoPD2/t6QSd06dPS5Isy6q3tk0Hob/97W+qqqpSfHx8wPL4+HgdPny4Vn1eXp7mzZtXa3lSUlKzzRHB46HWngAC0I/gcqX96PJCk0wDaJTTp08rJiamzpo2HYQaa9asWcrJybFfV1dX6+TJk+rcubNCQpr2/0p9Pp8SExP19ddfy+l0NunYaDz6EVzoR3ChH8GHntTNsiydPn1aCQkJ9da26SDUpUsXhYWFqaysLGB5WVmZXC5XrXqHwyGHwxGwLDY2tjmnKKfTyQ9xEKEfwYV+BBf6EXzoyU+r70hQjTb9ZOmIiAglJyerqKjIXlZdXa2ioiKlpqa24swAAEAwaNNHhCQpJydH48eP16BBgzR48GAtWbJEZ8+ete8iAwAA5mrzQejBBx/Ut99+q9zcXHm9Xg0cOFCFhYW1LqBuaQ6HQ08//XStU3FoHfQjuNCP4EI/gg89aTohVkPuLQMAAGiD2vQ1QgAAAHUhCAEAAGMRhAAAgLEIQgAAwFgEoVaQn5+vnj17KjIyUikpKdqzZ09rT6lNysvL080336wOHTooLi5OI0eO1JEjRwJqfvzxR2VmZqpz585q3769Ro8eXesBnMeOHVNGRoaio6MVFxen6dOn6/z58y35Vtqk5557TiEhIcrOzraX0Y+W9de//lX/9m//ps6dOysqKkr9+vXT3r177fWWZSk3N1ddu3ZVVFSU0tLS9NlnnwWMcfLkSY0bN05Op1OxsbGaOHGizpw509Jv5WevqqpKc+bMUVJSkqKionTNNddowYIFAZ+VRT+aiYUW9eabb1oRERHWsmXLrIMHD1qTJk2yYmNjrbKystaeWpvjdrut5cuXWwcOHLBKS0ute+65x+revbt15swZu2bKlClWYmKiVVRUZO3du9e65ZZbrFtvvdVef/78eatv375WWlqa9fHHH1sbN260unTpYs2aNas13lKbsWfPHqtnz55W//79rSeeeMJeTj9azsmTJ60ePXpYjzzyiLV7927ryy+/tDZt2mR9/vnnds1zzz1nxcTEWOvWrbM++eQT61/+5V+spKQk64cffrBrhg8fbg0YMMDatWuX9cEHH1jXXnutNXbs2NZ4Sz9rzz77rNW5c2dr/fr11tGjR621a9da7du3t5YuXWrX0I/mQRBqYYMHD7YyMzPt11VVVVZCQoKVl5fXirMyw4kTJyxJ1rZt2yzLsqzy8nIrPDzcWrt2rV3z6aefWpKs4uJiy7Isa+PGjVZoaKjl9XrtmldffdVyOp1WRUVFy76BNuL06dPWddddZ3k8Huuf/umf7CBEP1rWk08+ad1+++0/ub66utpyuVzW888/by8rLy+3HA6H9T//8z+WZVnWoUOHLEnWhx9+aNf87//+rxUSEmL99a9/bb7Jt0EZGRnWo48+GrBs1KhR1rhx4yzLoh/NiVNjLaiyslIlJSVKS0uzl4WGhiotLU3FxcWtODMznDp1SpLUqVMnSVJJSYn8fn9AP3r16qXu3bvb/SguLla/fv0CHsDpdrvl8/l08ODBFpx925GZmamMjIyA/S7Rj5b27rvvatCgQXrggQcUFxenG2+8Ub/73e/s9UePHpXX6w3oR0xMjFJSUgL6ERsbq0GDBtk1aWlpCg0N1e7du1vuzbQBt956q4qKivTnP/9ZkvTJJ59ox44dGjFihCT60Zza/JOlg8nf/vY3VVVV1XqqdXx8vA4fPtxKszJDdXW1srOzddttt6lv376SJK/Xq4iIiFofrBsfHy+v12vXXKpfNevQOG+++aY++ugjffjhh7XW0Y+W9eWXX+rVV19VTk6OnnrqKX344Yf6j//4D0VERGj8+PH2/rzU/r6wH3FxcQHr27Vrp06dOtGPRpo5c6Z8Pp969eqlsLAwVVVV6dlnn9W4ceMkiX40I4IQjJCZmakDBw5ox44drT0VY3399dd64okn5PF4FBkZ2drTMV51dbUGDRqk//7v/5Yk3XjjjTpw4IAKCgo0fvz4Vp6ded566y2tWrVKq1ev1g033KDS0lJlZ2crISGBfjQzTo21oC5duigsLKzWXTBlZWVyuVytNKu2LysrS+vXr9cf//hHdevWzV7ucrlUWVmp8vLygPoL++FyuS7Zr5p1aLiSkhKdOHFCN910k9q1a6d27dpp27ZtevHFF9WuXTvFx8fTjxbUtWtX9enTJ2BZ7969dezYMUn/vz/r+vvK5XLpxIkTAevPnz+vkydP0o9Gmj59umbOnKkxY8aoX79+evjhhzVt2jTl5eVJoh/NiSDUgiIiIpScnKyioiJ7WXV1tYqKipSamtqKM2ubLMtSVlaW3n77bW3ZskVJSUkB65OTkxUeHh7QjyNHjujYsWN2P1JTU7V///6Av1w8Ho+cTmetf0RQt6FDh2r//v0qLS21vwYNGqRx48bZf6YfLee2226r9TiJP//5z+rRo4ckKSkpSS6XK6AfPp9Pu3fvDuhHeXm5SkpK7JotW7aourpaKSkpLfAu2o5z584pNDTwn+SwsDBVV1dLoh/NqrWv1jbNm2++aTkcDmvFihXWoUOHrMmTJ1uxsbEBd8GgaUydOtWKiYmxtm7dah0/ftz+OnfunF0zZcoUq3v37taWLVusvXv3WqmpqVZqaqq9vuZ27fT0dKu0tNQqLCy0rr76am7XbiIX3jVmWfSjJe3Zs8dq166d9eyzz1qfffaZtWrVKis6Otp644037JrnnnvOio2Ntd555x1r37591n333XfJ27VvvPFGa/fu3daOHTus6667jtu1L8P48eOtf/iHf7Bvn//DH/5gdenSxZoxY4ZdQz+aB0GoFbz00ktW9+7drYiICGvw4MHWrl27WntKbZKkS34tX77crvnhhx+sf//3f7c6duxoRUdHW/fff791/PjxgHG++uora8SIEVZUVJTVpUsX6z//8z8tv9/fwu+mbbo4CNGPlvXee+9Zffv2tRwOh9WrVy/rt7/9bcD66upqa86cOVZ8fLzlcDisoUOHWkeOHAmo+e6776yxY8da7du3t5xOpzVhwgTr9OnTLfk22gSfz2c98cQTVvfu3a3IyEjrF7/4hfVf//VfAY+FoB/NI8SyLnhsJQAAgEG4RggAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAY/0fvrPaAuAtZTsAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "pd.Series([m[\"duration_s\"] for m in new_metas]).hist(bins=50)" ] }, { "cell_type": "code", "execution_count": 1, "id": "c0279185", "metadata": {}, "outputs": [], "source": [ "# from tokenizers import Tokenizer\n", "from tokenizers import AddedToken\n", "from transformers import T5Tokenizer, BertTokenizer\n", "\n", "from suno_utils.utils.tokenizers import tokenize, detokenize" ] }, { "cell_type": "code", "execution_count": 2, "id": "73bcced9", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "You are using the legacy behaviour of the . This means that tokens that come after special tokens will not be properly handled. We recommend you to read the related pull request available at https://github.com/huggingface/transformers/pull/24565\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "119548\n", "32101\n" ] } ], "source": [ "bert_tokenizer = BertTokenizer.from_pretrained(\"bert-base-multilingual-cased\", model_max_length=256)\n", "bert_tokenizer.add_special_tokens({\"additional_special_tokens\": [AddedToken(\"\\n\")]})\n", "t5_tokenizer = T5Tokenizer.from_pretrained(\"t5-base\", model_max_length=256)\n", "t5_tokenizer.add_special_tokens({\"additional_special_tokens\": [AddedToken(\"\\n\")]})\n", "# TODO: for T5 there is a weird bug https://github.com/huggingface/transformers/pull/24565\n", "\n", "print(len(bert_tokenizer.get_vocab()))\n", "print(len(t5_tokenizer.get_vocab()))" ] }, { "cell_type": "code", "execution_count": 4, "id": "0d3dde1a", "metadata": {}, "outputs": [], "source": [ "text = \"[rap] This is a test.\\nAnotheronething line.\\n\\n New verse.\"" ] }, { "cell_type": "code", "execution_count": 7, "id": "5896c630", "metadata": {}, "outputs": [], "source": [ "# bert_tokenizer.encode(text, add_special_tokens=False, truncation=True, max_length=10)" ] }, { "cell_type": "code", "execution_count": 8, "id": "9cdf8f9d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "16\n", "['[', 'rap', ']', 'This', 'is', 'a', 'test', '.', 'Another', '##onet', '##hing', 'line', '.', 'New', 'verse', '.']\n", "[ rap ] This is a test. Anotheronething line. New verse.\n" ] } ], "source": [ "enc = bert_tokenizer.encode(text, add_special_tokens=False)\n", "print(len(enc))\n", "print(bert_tokenizer.convert_ids_to_tokens(enc))\n", "print(bert_tokenizer.decode(enc))" ] }, { "cell_type": "code", "execution_count": 43, "id": "984d63d6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "20\n", "['▁[', 'rap', ']', '▁This', '▁is', '▁', 'a', '▁test', '.', '\\n', '▁Another', 'one', 'thing', '▁line', '.', '\\n', '\\n', '▁New', '▁verse', '.']\n", "[rap] This is a test. \n", " Anotheronething line. \n", " \n", " New verse.\n" ] } ], "source": [ "enc = t5_tokenizer.encode(text, add_special_tokens=False)\n", "print(len(enc))\n", "print(t5_tokenizer.convert_ids_to_tokens(enc))\n", "print(t5_tokenizer.decode(enc))" ] }, { "cell_type": "code", "execution_count": 51, "id": "5eb02904", "metadata": {}, "outputs": [], "source": [ "text = \"\"\"\n", "In einem Bächlein helle,\n", "Da schoß in froher Eil\n", "Die launische Forelle\n", "Vorüber wie ein Pfeil\n", "\"\"\".strip()" ] }, { "cell_type": "code", "execution_count": 52, "id": "2ff9aea6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "33\n", "['In', 'einem', 'B', '##ä', '##ch', '##lein', 'hell', '##e', ',', '\\n', 'Da', 'scho', '##ß', 'in', 'fr', '##oh', '##er', 'Ei', '##l', '\\n', 'Die', 'lau', '##nische', 'Forel', '##le', '\\n', 'Vor', '##über', 'wie', 'ein', 'P', '##fe', '##il']\n", "In einem Bächlein helle, \n", " Da schoß in froher Eil \n", " Die launische Forelle \n", " Vorüber wie ein Pfeil\n" ] } ], "source": [ "enc = bert_tokenizer.encode(text, add_special_tokens=False)\n", "print(len(enc))\n", "print(bert_tokenizer.convert_ids_to_tokens(enc))\n", "print(bert_tokenizer.decode(enc))" ] }, { "cell_type": "code", "execution_count": 53, "id": "64e6a303", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "37\n", "['▁In', '▁einem', '▁B', 'äch', 'le', 'in', '▁hell', 'e', ',', '\\n', '▁Da', '▁', 's', 'cho', 'ß', '▁in', '▁', 'fro', 'her', '▁E', 'il', '\\n', '▁Die', '▁la', 'un', 'ische', '▁For', 'elle', '\\n', '▁Vor', 'über', '▁wie', '▁ein', '▁P', 'f', 'e', 'il']\n", "In einem Bächlein helle, \n", " Da schoß in froher Eil \n", " Die launische Forelle \n", " Vorüber wie ein Pfeil\n" ] } ], "source": [ "enc = t5_tokenizer.encode(text, add_special_tokens=False)\n", "print(len(enc))\n", "print(t5_tokenizer.convert_ids_to_tokens(enc))\n", "print(t5_tokenizer.decode(enc))" ] }, { "cell_type": "code", "execution_count": 54, "id": "390d1053", "metadata": {}, "outputs": [], "source": [ "text = \"\"\"\n", "在一条明亮的小溪里，\n", "达高兴地匆忙开枪\n", "喜怒无常的鳟鱼\n", "如箭般离去\n", "\"\"\".strip()" ] }, { "cell_type": "code", "execution_count": 55, "id": "eb7242cd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "33\n", "['在', '一', '条', '明', '亮', '的', '小', '溪', '里', '，', '\\n', '达', '高', '兴', '地', '匆', '忙', '开', '枪', '\\n', '喜', '怒', '无', '常', '的', '[UNK]', '鱼', '\\n', '如', '箭', '般', '离', '去']\n", "在 一 条 明 亮 的 小 溪 里 ， \n", " 达 高 兴 地 匆 忙 开 枪 \n", " 喜 怒 无 常 的 [UNK] 鱼 \n", " 如 箭 般 离 去\n" ] } ], "source": [ "enc = bert_tokenizer.encode(text, add_special_tokens=False)\n", "print(len(enc))\n", "print(bert_tokenizer.convert_ids_to_tokens(enc))\n", "print(bert_tokenizer.decode(enc))" ] }, { "cell_type": "code", "execution_count": 56, "id": "ba33d332", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "12\n", "['▁', '', ',', '\\n', '▁', '', '\\n', '▁', '', '\\n', '▁', '']\n", ", \n", " \n", " \n", " \n" ] } ], "source": [ "enc = t5_tokenizer.encode(text, add_special_tokens=False)\n", "print(len(enc))\n", "print(t5_tokenizer.convert_ids_to_tokens(enc))\n", "print(t5_tokenizer.decode(enc))" ] }, { "cell_type": "code", "execution_count": 57, "id": "df71f34c", "metadata": {}, "outputs": [], "source": [ "text = \"\"\"\n", "इनेम बैचलीन हेले,\n", "दा स्कोß इन फ्रहर एइल\n", "मरो launische Forelle\n", "वोरुबेर एक पफील के साथ\n", "\"\"\".strip()" ] }, { "cell_type": "code", "execution_count": 58, "id": "f0a64241", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "41\n", "['इन', '##ेम', 'ब', '##ैच', '##ली', '##न', 'हे', '##ले', ',', '\\n', 'द', '##ा', 'स', '##्क', '##ो', '##ß', 'इन', 'फ', '##्रह', '##र', 'ए', '##इल', '\\n', 'म', '##रो', 'lau', '##nische', 'Forel', '##le', '\\n', 'व', '##ोर', '##ु', '##बे', '##र', 'एक', 'प', '##फ', '##ील', 'के', 'साथ']\n", "इनेम बैचलीन हेले, \n", " दा स्कोß इन फ्रहर एइल \n", " मरो launische Forelle \n", " वोरुबेर एक पफील के साथ\n" ] } ], "source": [ "enc = bert_tokenizer.encode(text, add_special_tokens=False)\n", "print(len(enc))\n", "print(bert_tokenizer.convert_ids_to_tokens(enc))\n", "print(bert_tokenizer.decode(enc))" ] }, { "cell_type": "code", "execution_count": 59, "id": "5f98f53f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "38\n", "['▁', '', '▁', '', '▁', '', ',', '\\n', '▁', '', '▁', '', 'ß', '▁', '', '▁', '', '▁', '', '\\n', '▁', '', '▁la', 'un', 'ische', '▁For', 'elle', '\\n', '▁', '', '▁', '', '▁', '', '▁', '', '▁', '']\n", " , \n", " ß \n", " launische Forelle \n", " \n" ] } ], "source": [ "enc = t5_tokenizer.encode(text, add_special_tokens=False)\n", "print(len(enc))\n", "print(t5_tokenizer.convert_ids_to_tokens(enc))\n", "print(t5_tokenizer.decode(enc))" ] }, { "cell_type": "code", "execution_count": 24, "id": "cb3fbcd8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'blabla\\n\\nas d'" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "text = \"blabla \\n \\n\\nas d \\n\"\n", "\n", "def _space_repl(m):\n", " s = m.group()\n", " n_newline = s.count(\"\\n\")\n", " if n_newline >= 2:\n", " return \"\\n\\n\"\n", " elif n_newline == 1:\n", " return \"\\n\"\n", " return \" \"\n", "\n", "def simplify_whitespace(text):\n", " \"\"\"simplify while respecting up to 2 newlines\"\"\"\n", " return re.sub(r\"\\s+\", _space_repl, text).strip()\n", "\n", "simplify_whitespace(text)" ] }, { "cell_type": "code", "execution_count": null, "id": "479be3ca", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e9278d80", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "98d1a1ec", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ae8a6bfe", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "57fb5a0f", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 5 }