{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", "from suno_utils.utils.text import (\n", " write_jsonl,\n", " read_jsonl,\n", ")" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 1460846/1460846 [00:38<00:00, 37998.70it/s]\n" ] } ], "source": [ "METAS_DIR = \"/app/suno/data/diffusion_mix/metadata\"\n", "all_metas = read_jsonl(os.path.join(METAS_DIR, \"metas.jsonl\"), progress=True)" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Counter({'en': 679316, None: 448226, 'es': 72557, 'fr': 35783, 'pt': 33209, 'de': 29648, 'ru': 22797, 'pl': 19711, 'ko': 16627, 'it': 16530, 'tr': 14611, 'ar': 7157, 'hi': 6282, 'nl': 5274, 'id': 5085, 'sv': 3633, 'da': 3217, 'tl': 3065, 'ja': 2773, 'ro': 2706, 'fi': 2221, 'pa': 1940, 'sr': 1934, 'iw': 1847, 'ta': 1757, 'no': 1686, 'hr': 1676, 'te': 1601, 'uk': 1405, 'cs': 1253, 'zh': 1227, 'fa': 1103, 'th': 1078, 'sk': 904, 'bs': 881, 'hu': 847, 'vi': 845, 'sq': 737, 'romanization': 624, 'bg': 546, 'ms': 471, 'ml': 388, 'az': 364, 'kn': 349, 'et': 341, 'sw': 288, 'is': 249, 'lt': 209, 'lv': 208, 'ca': 184, 'af': 182, 'eu': 166, 'he': 159, 'ur': 158, 'mk': 158, 'zu': 158, 'sl': 142, 'la': 133, 'co': 129, 'sco': 127, 'arz': 124, 'gl': 119, 'ht': 109, 'war': 94, 'eo': 81, 'cy': 75, 'mr': 75, 'xh': 74, 'ceb': 72, 'sn': 58, 'el': 55, 'ga': 50, 'yo': 44, 'ln': 42, 'pnb': 38, 'sa': 37, 'gu': 36, 'jbo': 30, 'su': 30, 'mn': 28, 'be': 27, 'bn': 27, 'st': 27, 'ne': 22, 'jv': 20, 'gd': 20, 'ia': 20, 'jw': 20, 'ak': 19, 'wo': 18, 'rw': 18, 'ku': 17, 'uz': 16, 'lb': 16, 'mi': 16, 'br': 15, 'so': 15, 'rm': 15, 'nn': 13, 'bh': 12, 'oc': 12, 'mg': 12, 'kw': 11, 'ka': 11, 'aa': 10, 'ie': 10, 'hy': 9, 'qu': 9, 'ha': 9, 've': 9, 'ts': 9, 'sm': 9, 'tn': 9, 'bi': 9, 'fo': 9, 'fy': 8, 'tt': 8, 'haw': 8, 'lg': 7, 'om': 7, 'tlh': 7, 'ny': 6, 'ast': 5, 'mt': 5, 'vo': 5, 'crs': 5, 'to': 5, 'kk': 4, 'gn': 4, 'new': 4, 'nds': 4, 'als': 4, 'diq': 4, 'ss': 4, 'ig': 4, 'kha': 4, 'or': 3, 'io': 3, 'si': 3, 'km': 3, 'yi': 3, 'ilo': 3, 'am': 3, 'tk': 3, 'fj': 3, 'tg': 3, 'pam': 2, 'pms': 2, 'scn': 2, 'lmo': 2, 'nso': 2, 'kl': 2, 'gv': 2, 'an': 1, 'cbk': 1, 'xmf': 1, 'sh': 1, 'ckb': 1, 'sc': 1, 'wa': 1, 'lo': 1, 'nap': 1, 'mzn': 1, 'ps': 1, 'vec': 1, 'mfe': 1, 'ky': 1, 'ab': 1, 'iu': 1, 'nr': 1})\n" ] } ], "source": [ "from collections import Counter\n", "\n", "c = Counter()\n", "for m in all_metas:\n", " c[m.get(\"lang\")] += 1" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "en 679316\n", "None 448226\n", "es 72557\n", "fr 35783\n", "pt 33209\n", "de 29648\n", "ru 22797\n", "pl 19711\n", "ko 16627\n", "it 16530\n", "tr 14611\n", "ar 7157\n", "hi 6282\n", "nl 5274\n", "id 5085\n", "sv 3633\n", "da 3217\n", "tl 3065\n", "ja 2773\n", "ro 2706\n", "fi 2221\n", "pa 1940\n", "sr 1934\n", "iw 1847\n", "ta 1757\n", "no 1686\n", "hr 1676\n", "te 1601\n", "uk 1405\n", "cs 1253\n", "zh 1227\n", "fa 1103\n", "th 1078\n", "sk 904\n", "bs 881\n", "hu 847\n", "vi 845\n", "sq 737\n", "romanization 624\n", "bg 546\n", "ms 471\n", "ml 388\n", "az 364\n", "kn 349\n", "et 341\n", "sw 288\n", "is 249\n", "lt 209\n", "lv 208\n", "ca 184\n", "af 182\n", "eu 166\n", "he 159\n", "ur 158\n", "mk 158\n", "zu 158\n", "sl 142\n", "la 133\n", "co 129\n", "sco 127\n", "arz 124\n", "gl 119\n", "ht 109\n", "war 94\n", "eo 81\n", "cy 75\n", "mr 75\n", "xh 74\n", "ceb 72\n", "sn 58\n", "el 55\n", "ga 50\n", "yo 44\n", "ln 42\n", "pnb 38\n", "sa 37\n", "gu 36\n", "jbo 30\n", "su 30\n", "mn 28\n", "be 27\n", "bn 27\n", "st 27\n", "ne 22\n", "jv 20\n", "gd 20\n", "ia 20\n", "jw 20\n", "ak 19\n", "wo 18\n", "rw 18\n", "ku 17\n", "uz 16\n", "lb 16\n", "mi 16\n", "br 15\n", "so 15\n", "rm 15\n", "nn 13\n", "bh 12\n", "oc 12\n", "mg 12\n", "kw 11\n", "ka 11\n", "aa 10\n", "ie 10\n", "hy 9\n", "qu 9\n", "ha 9\n", "ve 9\n", "ts 9\n", "sm 9\n", "tn 9\n", "bi 9\n", "fo 9\n", "fy 8\n", "tt 8\n", "haw 8\n", "lg 7\n", "om 7\n", "tlh 7\n", "ny 6\n", "ast 5\n", "mt 5\n", "vo 5\n", "crs 5\n", "to 5\n", "kk 4\n", "gn 4\n", "new 4\n", "nds 4\n", "als 4\n", "diq 4\n", "ss 4\n", "ig 4\n", "kha 4\n", "or 3\n", "io 3\n", "si 3\n", "km 3\n", "yi 3\n", "ilo 3\n", "am 3\n", "tk 3\n", "fj 3\n", "tg 3\n", "pam 2\n", "pms 2\n", "scn 2\n", "lmo 2\n", "nso 2\n", "kl 2\n", "gv 2\n", "an 1\n", "cbk 1\n", "xmf 1\n", "sh 1\n", "ckb 1\n", "sc 1\n", "wa 1\n", "lo 1\n", "nap 1\n", "mzn 1\n", "ps 1\n", "vec 1\n", "mfe 1\n", "ky 1\n", "ab 1\n", "iu 1\n", "nr 1\n" ] } ], "source": [ "for k, v in sorted(c.items(), key=lambda x: x[1], reverse=True):\n", " print(k, v)" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1227\n" ] } ], "source": [ "selected_metas = [m for m in all_metas if \"zh\" in m.get(\"lang\", \"\").lower()]\n", "print(len(selected_metas))" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 1227/1227 [00:00<00:00, 44662.85it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "write_jsonl(\n", " selected_metas,\n", " os.path.join(\"/app/suno/data/diff_dpo/ft/metadata\", \"metas.jsonl\"),\n", " progress=True,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" } }, "nbformat": 4, "nbformat_minor": 2 }