{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:48:26.486470Z",
"start_time": "2024-01-26T15:48:26.484985Z"
}
},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\""
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:48:41.159922Z",
"start_time": "2024-01-26T15:48:26.488090Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/tony/anaconda3/envs/suno_env/lib/python3.10/site-packages/whisper/timing.py:57: NumbaDeprecationWarning: \u001b[1mThe 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.\u001b[0m\n",
" def backtrace(trace: np.ndarray):\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"WILL USE FLASH ATTN: True\n",
"model loaded: /app/suno/checkpoints/2024-01-20_22-01-25/last_ckpt_infer.pt\n",
"model loaded: 5869.4M params\n"
]
}
],
"source": [
"import numpy as np\n",
"import torch.nn.functional as F\n",
"from suno_utils.audio import Audio\n",
"from suno_utils.gpt.chirp_v2_5 import (\n",
" codec_decode,\n",
" GenerationConfig,\n",
" _get_model_if_needed,\n",
" decode_stream,\n",
" # prep_gconf,\n",
" preload_models,\n",
" prep_gconf,\n",
" GPT_CKPT_PATH,\n",
" generate_audio,\n",
" generate_audio_stream,\n",
")\n",
"from suno_utils.gpt.generation_engine import (\n",
" make_prompt,\n",
" align_codes,\n",
" make_request,\n",
" unshift_arrays,\n",
")\n",
"from suno_utils.gpt.engine import Engine, Request, Job\n",
"import torch\n",
"from tqdm import tqdm\n",
"\n",
"FT_GPT_CKPT_PATH = \"s3://suno-data/tony/fine_tune_3b/xxl_ft_v7_t1_35k.pt\"\n",
"\n",
"preload_models(\n",
" load_gpt=False,\n",
" load_semantic=False,\n",
" # gpt_ckpt_path=\"s3://suno-data/georg/checkpoints/chirp_v2_5/7b.pt\",\n",
" # codec_ckpt_path=\"s3://suno-data/georg/models/codec/dac_2c_25x12.pt\",\n",
")\n",
"\n",
"# gpt_ckpt_path = _get_model_if_needed(\n",
"# \"s3://suno-data/georg/checkpoints/chirp_v2_5/7b.pt\"\n",
"# )\n",
"gpt_ckpt_path = \"/app/suno/checkpoints/2024-01-20_22-01-25/last_ckpt_infer.pt\"\n",
"engine = Engine(\n",
" gpt_ckpt_path,\n",
" \"/home/victor/data/models/chirp_v2/tokenizer_60k.json\",\n",
" max_sequences=8,\n",
" # compile=True,\n",
")\n",
"# engine.start()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:48:41.169489Z",
"start_time": "2024-01-26T15:48:41.162522Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Request(id='1', streams=[CfgStreamConfig(prompt=tensor([[ 60, 76, 14, ..., 20633, 10, 60001],\n",
" [ 4000, 4000, 4000, ..., 4000, 4000, 4001],\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049],\n",
" ...,\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049],\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049],\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049]]), max_n_steps=None, weight=1.1), CfgStreamConfig(prompt=tensor([[60001, 60001, 60001, ..., 60001, 60001, 60001],\n",
" [ 4000, 4000, 4000, ..., 4000, 4000, 4001],\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049],\n",
" ...,\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049],\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049],\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049]]), max_n_steps=None, weight=1.4), CfgStreamConfig(prompt=tensor([[ 60, 76, 14, ..., 60001, 60001, 60001],\n",
" [ 4000, 4000, 4000, ..., 4000, 4000, 4001],\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049],\n",
" ...,\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049],\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049],\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049]]), max_n_steps=250, weight=2.5), CfgStreamConfig(prompt=tensor([[ 60, 20553, 23324, ..., 60001, 60001, 60001],\n",
" [ 4000, 4000, 4000, ..., 4000, 4000, 4001],\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049],\n",
" ...,\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049],\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049],\n",
" [ 2048, 2048, 2048, ..., 2048, 2048, 2049]]), max_n_steps=250, weight=-4)], max_gen_duration_s=120, min_gen_duration_s=2.5, temp_semantic=0.9, temp_coarse=0.9, top_k_semantic=1500, top_k_coarse=1500, top_p_semantic=0.98, top_p_coarse=0.98, allow_eos=True, min_eos_p=0.25, eos_pad_duration_s=0.24, history_arr=None, random_seed=42, debug=True)\n",
"p.shape=torch.Size([14, 172])\n",
"p.shape=torch.Size([14, 172])\n",
"p.shape=torch.Size([14, 172])\n",
"p.shape=torch.Size([14, 172])\n"
]
}
],
"source": [
"text = \"\"\"\n",
"[Verse]\n",
"In the morning, when I wake up\n",
"I see the sun, shining bright\n",
"I hear the birds, singing their song\n",
"Makes me wanna dance, all day long (ooh-yeah)\n",
"\n",
"[Chorus]\n",
"Sunshine in the rain (sunshine in the rain)\n",
"Makes me feel so alive (so alive)\n",
"With you by my side (with you by my side)\n",
"I'm never gonna complain (no-no-no)\n",
"Sunshine in the rain (sunshine in the rain)\n",
"Every cloud has a silver lining (silver lining)\n",
"With you, life is divine (so divine)\n",
"Together, we're never gonna fade (we'll never fade)\n",
"Sunshine in the rain (sunshine in the rain)\n",
"\"\"\"\n",
"\n",
"cfg = GenerationConfig(\n",
" text=text,\n",
" # cfg_coef=1, # uncomment, you trun OFF cfg_text -- 2nd stream\n",
" text_tags=\"k-pop\", # uncomment, you turn ON cfg_tag -- 3rd stream\n",
" # cfg_coef_tags_max_steps=None,\n",
" text_neg_tags=\"instrumental noise\",\n",
" n_batch=1,\n",
" max_gen_duration_s=120,\n",
" allow_eos=True,\n",
" random_seed=42,\n",
" debug=True\n",
")\n",
"cfg = prep_gconf(cfg)\n",
"\n",
"model_conf = engine.model.config\n",
"request = make_request(\"1\", cfg, engine.model.config, engine.tokenizer)\n",
"print(request)\n",
"for stream in request.streams:\n",
" p = stream.prompt\n",
" print(f\"{p.shape=}\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:48:41.226505Z",
"start_time": "2024-01-26T15:48:41.171531Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"GenerationConfig(text=[\"[Verse]\\nIn the morning, when I wake up\\nI see the sun, shining bright\\nI hear the birds, singing their song\\nMakes me wanna dance, all day long (ooh-yeah)\\n\\n[Chorus]\\nSunshine in the rain (sunshine in the rain)\\nMakes me feel so alive (so alive)\\nWith you by my side (with you by my side)\\nI'm never gonna complain (no-no-no)\\nSunshine in the rain (sunshine in the rain)\\nEvery cloud has a silver lining (silver lining)\\nWith you, life is divine (so divine)\\nTogether, we're never gonna fade (we'll never fade)\\nSunshine in the rain (sunshine in the rain)\"], text_tags='k-pop', text_neg_tags='instrumental noise', text_start_control_tags=None, text_end_control_tags=None, history_arr=None, history_text=None, n_batch=1, max_gen_duration_s=120, min_gen_duration_s=2.5, temp_semantic=0.9, temp_coarse=0.9, top_k_semantic=1500, top_k_coarse=1500, top_p_semantic=0.98, top_p_coarse=0.98, cfg_coef=1.1, cfg_coef_max_steps=None, cfg_coef_tags=2.5, cfg_coef_neg_tags=-4, cfg_coef_tags_max_steps=250, n_repeat_tags=3, rep_penality=0, allow_eos=True, min_eos_p=0.25, min_text_offset=128, eos_pad_duration_s=0.24, respect_newlines=True, oracle_arr=None, return_semantic=True, silent=False, stream=False, n_stride_tokens=10, random_seed=42, use_whisper=True, history_do_asr=True, debug=True)\n"
]
}
],
"source": [
"print(cfg)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:49:28.678130Z",
"start_time": "2024-01-26T15:48:41.228154Z"
},
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Job 1 found EOS at step 1385\n",
"Job 1 completed at step 1496\n",
"Job 1 completed\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"1496it [00:01, 1041.29it/s]\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"job = engine.run_request(request)\n",
"\n",
"stream = engine.token_generator(job)\n",
"\n",
"audios = []\n",
"for audio in decode_stream(align_codes(tqdm(stream), model_conf)):\n",
" audios.append(audio[0])\n",
"\n",
"audio_continued = Audio.concatenate(audios)\n",
"audio_continued.play()\n",
"# print(job.debug_info)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:49:28.688863Z",
"start_time": "2024-01-26T15:49:28.680440Z"
}
},
"outputs": [],
"source": [
"torch.cuda.empty_cache()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:50:16.567511Z",
"start_time": "2024-01-26T15:49:28.690177Z"
},
"scrolled": false
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
" 48%|███████████████████████████████████████████████████▌ | 1496/3105 [00:45<00:49, 32.84it/s]\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"out = {}\n",
"audios = generate_audio_stream(cfg, out=out)\n",
"audio = Audio.concatenate([a[0] for a in audios])\n",
"audio.play()\n",
"# print(out)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:50:16.648827Z",
"start_time": "2024-01-26T15:50:16.569072Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"prefill_last_k_cache\n",
"torch.Size([4, 128]) torch.Size([4, 128])\n",
"tensor(0., dtype=torch.bfloat16)\n",
"first_semantic_stream_logits\n",
"\n",
"first_semantic_probs\n",
"tensor([[0., 0., 0., ..., 0., 0., 0.]])\n",
"tensor([[0., 0., 0., ..., 0., 0., 0.]])\n"
]
}
],
"source": [
"# check debug info match\n",
"engine_debug = job.debug_info\n",
"base_debug = out[\"debug_info\"]\n",
"engine_debug.move_to_device(\"cpu\")\n",
"base_debug.move_to_device(\"cpu\")\n",
"\n",
"# for base_prompt, engine_prompt in zip(\n",
"# base_debug.stream_prompts, engine_debug.stream_prompts\n",
"# ):\n",
"# assert torch.allclose(base_prompt, engine_prompt)\n",
"\n",
"print(\"prefill_last_k_cache\")\n",
"print(base_debug.prefill_last_k_cache.shape, engine_debug.prefill_last_k_cache.shape)\n",
"# print(base_debug.prefill_last_k_cache[0, :-10])\n",
"# print(engine_debug.prefill_last_k_cache[0, :-10])\n",
"print(\n",
" torch.max(\n",
" torch.abs(base_debug.prefill_last_k_cache - engine_debug.prefill_last_k_cache)\n",
" )\n",
")\n",
"# assert torch.allclose(\n",
"# base_debug.prefill_last_k_cache, engine_debug.prefill_last_k_cache, atol=0.2\n",
"# )\n",
"\n",
"print(\"first_semantic_stream_logits\")\n",
"# print(base_debug.first_semantic_stream_logits)\n",
"# print(engine_debug.first_semantic_stream_logits[:, :-1])\n",
"# check topk\n",
"# print(torch.topk(base_debug.first_semantic_stream_logits, 10))\n",
"# print(torch.topk(engine_debug.first_semantic_stream_logits[:, :-1], 10))\n",
"\n",
"# (\n",
"# base_debug.first_semantic_stream_logits\n",
"# - engine_debug.first_semantic_stream_logits[:, :-1]\n",
"# )[:, :100]\n",
"# assert torch.allclose(base_debug.first_semantic_stream_logits, engine_debug.first_semantic_stream_logits[:,:-1])\n",
"\n",
"\n",
"print()\n",
"print(\"first_semantic_probs\")\n",
"print(base_debug.first_semantic_probs)\n",
"print(engine_debug.first_semantic_probs[:, :-1])\n",
"# check topk\n",
"# print(torch.topk(base_debug.first_semantic_probs, 10))\n",
"# print(torch.topk(engine_debug.first_semantic_probs[:, :-1], 10))\n",
"# assert torch.allclose(\n",
"# base_debug.first_semantic_probs,\n",
"# engine_debug.first_semantic_probs[:, :-1],\n",
"# atol=0.04,\n",
"# )"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"no text cfg, no tag cfg\n",
"```\n",
"non-engine, engine\n",
"decode_one_logits decode_one_logits js: 1.2505552149377763e-12\n",
"decode_one_logits prefill_logits js: 8.857222564984113e-05\n",
"prefill_logits decode_one_logits js: 8.857222564984113e-05\n",
"prefill_logits prefill_logits js: 0.0\n",
"kv caches\n",
"prefill_last_k_cache prefill_last_k_cache rms: 0.0\n",
"prefill_last_k_cache decode_one_last_k_cache rms: 0.0\n",
"decode_one_last_k_cache prefill_last_k_cache rms: 0.0\n",
"decode_one_last_k_cache decode_one_last_k_cache rms: 0.0\n",
"```\n",
"\n",
"with only text cfg\n",
"```\n",
"non-engine, engine\n",
"decode_one_logits decode_one_logits js: 0.015353244729340076\n",
"decode_one_logits prefill_logits js: 0.05821094661951065\n",
"prefill_logits decode_one_logits js: 0.04050331562757492\n",
"prefill_logits prefill_logits js: 4.576613719109446e-07\n",
"kv caches\n",
"prefill_last_k_cache prefill_last_k_cache rms: 0.0\n",
"prefill_last_k_cache decode_one_last_k_cache rms: 0.0\n",
"decode_one_last_k_cache prefill_last_k_cache rms: 0.0\n",
"decode_one_last_k_cache decode_one_last_k_cache rms: 0.0\n",
"```\n",
"\n",
"3 streams\n",
"```\n",
"non-engine, engine\n",
"decode_one_logits decode_one_logits js: 1.359115481376648\n",
"decode_one_logits prefill_logits js: 1.4671876430511475\n",
"prefill_logits decode_one_logits js: 0.08467227220535278\n",
"prefill_logits prefill_logits js: 9.379164112033322e-13\n",
"kv caches\n",
"prefill_last_k_cache prefill_last_k_cache rms: 0.0\n",
"prefill_last_k_cache decode_one_last_k_cache rms: 0.0\n",
"decode_one_last_k_cache prefill_last_k_cache rms: 0.0\n",
"decode_one_last_k_cache decode_one_last_k_cache rms: 0.0\n",
"```\n",
"\n",
"with flash, without text cfg\n",
"```\n",
"non-engine, engine\n",
"decode_one_logits decode_one_logits js: 8.856291242409497e-05\n",
"decode_one_logits prefill_logits js: 0.00017808671691454947\n",
"prefill_logits decode_one_logits js: 0.00026702857576310635\n",
"prefill_logits prefill_logits js: 0.0\n",
"kv caches\n",
"prefill_last_k_cache prefill_last_k_cache rms: 0.00513291172683239\n",
"prefill_last_k_cache decode_one_last_k_cache rms: 0.00513291172683239\n",
"decode_one_last_k_cache prefill_last_k_cache rms: 0.00513291172683239\n",
"decode_one_last_k_cache decode_one_last_k_cache rms: 0.00513291172683239\n",
"```\n",
"with flash, with only text cfg\n",
"```\n",
"non-engine, engine\n",
"decode_one_logits decode_one_logits js: 0.011832738295197487\n",
"decode_one_logits prefill_logits js: 0.016643138602375984\n",
"prefill_logits decode_one_logits js: 0.004897898994386196\n",
"prefill_logits prefill_logits js: 0.0\n",
"kv caches\n",
"prefill_last_k_cache prefill_last_k_cache rms: 0.0032161902636289597\n",
"prefill_last_k_cache decode_one_last_k_cache rms: 0.0032161902636289597\n",
"decode_one_last_k_cache prefill_last_k_cache rms: 0.0032161902636289597\n",
"decode_one_last_k_cache decode_one_last_k_cache rms: 0.0032161902636289597\n",
"```\n",
"with flash, 3 streams\n",
"```\n",
"non-engine, engine\n",
"decode_one_logits decode_one_logits js: 0.9808357357978821\n",
"decode_one_logits prefill_logits js: 1.0738568305969238\n",
"prefill_logits decode_one_logits js: 0.07982449233531952\n",
"prefill_logits prefill_logits js: 0.0\n",
"kv caches\n",
"prefill_last_k_cache prefill_last_k_cache rms: 0.010091752745211124\n",
"prefill_last_k_cache decode_one_last_k_cache rms: 0.010091752745211124\n",
"decode_one_last_k_cache prefill_last_k_cache rms: 0.010091752745211124\n",
"decode_one_last_k_cache decode_one_last_k_cache rms: 0.010091752745211124\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:50:16.655225Z",
"start_time": "2024-01-26T15:50:16.650809Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"non-engine, engine\n",
"decode_one_logits decode_one_logits js: 3.237398147583008\n",
"decode_one_logits prefill_logits js: 3.243633270263672\n",
"prefill_logits decode_one_logits js: 0.0002987093757838011\n",
"prefill_logits prefill_logits js: 5.271967893349938e-07\n",
"kv caches\n",
"prefill_last_k_cache prefill_last_k_cache rms: 0.0\n",
"prefill_last_k_cache decode_one_last_k_cache rms: 0.012756001204252243\n",
"decode_one_last_k_cache prefill_last_k_cache rms: 0.012756001204252243\n",
"decode_one_last_k_cache decode_one_last_k_cache rms: 0.0\n",
"gpt debug out\n"
]
},
{
"data": {
"text/plain": [
"{'decode_one_logits_decode_one_logits_rms': 8.701568603515625,\n",
" 'decode_one_logits_decode_one_logits_kl': 15.217329025268555,\n",
" 'decode_one_logits_decode_one_logits_js': 3.237398147583008,\n",
" 'decode_one_logits_prefill_logits_rms': 8.58690071105957,\n",
" 'decode_one_logits_prefill_logits_kl': 15.238426208496094,\n",
" 'decode_one_logits_prefill_logits_js': 3.243633270263672,\n",
" 'prefill_logits_decode_one_logits_rms': 0.1580256223678589,\n",
" 'prefill_logits_decode_one_logits_kl': 0.0011787901166826487,\n",
" 'prefill_logits_decode_one_logits_js': 0.0002987093757838011,\n",
" 'prefill_logits_prefill_logits_rms': 0.0,\n",
" 'prefill_logits_prefill_logits_kl': 5.271967893349938e-07,\n",
" 'prefill_logits_prefill_logits_js': 5.271967893349938e-07,\n",
" 'prefill_last_k_cache_prefill_last_k_cache_rms': 0.0,\n",
" 'prefill_last_k_cache_decode_one_last_k_cache_rms': 0.012756001204252243,\n",
" 'decode_one_last_k_cache_prefill_last_k_cache_rms': 0.012756001204252243,\n",
" 'decode_one_last_k_cache_decode_one_last_k_cache_rms': 0.0}"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"base_debug.compare(engine_debug)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:50:16.712693Z",
"start_time": "2024-01-26T15:50:16.656767Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"torch.Size([1, 4032]) torch.Size([1, 4032])\n",
"torch.FloatTensor torch.FloatTensor\n",
"torch.Size([1, 4000]) torch.Size([1, 4000])\n",
"torch.FloatTensor torch.FloatTensor\n"
]
}
],
"source": [
"print(base_debug.first_semantic_stream_logits.shape, engine_debug.first_semantic_stream_logits.shape)\n",
"print(base_debug.first_semantic_stream_logits.type(), engine_debug.first_semantic_stream_logits.type())\n",
"print(base_debug.first_semantic_probs.shape, engine_debug.first_semantic_probs.shape)\n",
"print(base_debug.first_semantic_probs.type(), engine_debug.first_semantic_probs.type())"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:50:16.773635Z",
"start_time": "2024-01-26T15:50:16.714303Z"
}
},
"outputs": [],
"source": [
"# for i in range(4000):\n",
"# if (\n",
"# abs(\n",
"# base_debug.first_semantic_stream_logits[0, i].item()\n",
"# - engine_debug.first_semantic_stream_logits[0, i].item()\n",
"# )\n",
"# > 0\n",
"# ):\n",
"# print(\n",
"# i,\n",
"# base_debug.first_semantic_probs[0, i],\n",
"# engine_debug.first_semantic_probs[0, i],\n",
"# )\n",
"# print(\n",
"# \"--\",\n",
"# base_debug.first_semantic_stream_logits[0, i].item(),\n",
"# engine_debug.first_semantic_stream_logits[0, i].item(),\n",
"# )\n",
"# print(\"OOOPs\")\n",
"# for i in range(4000):\n",
"# if (\n",
"# abs(\n",
"# base_debug.first_semantic_probs[0, i].item()\n",
"# - engine_debug.first_semantic_probs[0, i].item()\n",
"# )\n",
"# > 0\n",
"# ):\n",
"# print(\n",
"# i,\n",
"# base_debug.first_semantic_probs[0, i],\n",
"# engine_debug.first_semantic_probs[0, i],\n",
"# )\n",
"# print(\n",
"# \"--\",\n",
"# base_debug.first_semantic_stream_logits[0, i].item(),\n",
"# engine_debug.first_semantic_stream_logits[0, i].item(),\n",
"# )"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:50:16.836751Z",
"start_time": "2024-01-26T15:50:16.775460Z"
}
},
"outputs": [],
"source": [
"# semantic at 1687 tensor([601]) tensor(4000)\n",
"# semantic at 1688 tensor([54]) tensor(4000)\n",
"# semantic at 1689 tensor([3797]) tensor(4000)\n",
"# semantic at 1690 tensor([1552]) tensor(4000)\n",
"# semantic at 1691 tensor([2082]) tensor(4000)\n",
"# semantic at 1692 tensor([3797]) tensor(4000)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:50:16.919617Z",
"start_time": "2024-01-26T15:50:16.838360Z"
}
},
"outputs": [],
"source": [
"for i, (base_token, engine_token) in enumerate(zip(base_debug.pred_semantic_tokens, engine_debug.pred_semantic_tokens)):\n",
" if base_token[0] != engine_token:\n",
" print(f\"semantic at {i}\", base_token[0], engine_token)\n",
"\n",
"for i, (base_token, engine_token) in enumerate(zip(base_debug.pred_coarse_tokens, engine_debug.pred_coarse_tokens)):\n",
" base_token = base_token[0]\n",
" if not torch.equal(base_token, engine_token.reshape(-1, 1)):\n",
" print(f\"coarse at {i}\", base_token, engine_token)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:50:16.970337Z",
"start_time": "2024-01-26T15:50:16.921130Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"59.88"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(base_debug.pred_coarse_tokens) / 25"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Check"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:50:17.031504Z",
"start_time": "2024-01-26T15:50:16.972216Z"
}
},
"outputs": [],
"source": [
"# from collections import Counter\n",
"# test_list = [v.item() for v in engine_debug.first_semantic_stream_logits[0, :]]\n",
"# print(Counter(test_list))\n",
"\n",
"# test_tensor = torch.tensor(test_list).reshape(1, -1)\n",
"# print(\"this is a normal tensor\", test_tensor.shape, test_tensor.type())\n",
"# test_tensor_b = torch.tensor(test_list).bfloat16().reshape(1, -1)\n",
"# print(\"this is bfloat16\", test_tensor_b.shape, test_tensor_b.type())\n",
"# print(job.request.temp_semantic, type(job.request.temp_semantic))\n",
"# test_tensor_probs = F.softmax(test_tensor / job.request.temp_semantic, dim=-1)\n",
"# test_tensor_b_probs = F.softmax(test_tensor_b / job.request.temp_semantic, dim=-1)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-26T15:50:17.104549Z",
"start_time": "2024-01-26T15:50:17.032734Z"
}
},
"outputs": [],
"source": [
"# for i in range(len(test_list)):\n",
"# if abs(test_tensor_probs[0, i].item() - test_tensor_b_probs[0, i].item()) > 0.001:\n",
"# print(\n",
"# \"WTF\",\n",
"# i,\n",
"# test_list[i],\n",
"# \"\\n float tensor:\",\n",
"# test_tensor[0, i],\n",
"# test_tensor[0, i] / job.request.temp_semantic,\n",
"# test_tensor_probs[0, i].item(),\n",
"# \"\\n bfloat tensor:\",\n",
"# test_tensor_b[0, i],\n",
"# test_tensor_b[0, i] / job.request.temp_semantic,\n",
"# test_tensor_b_probs[0, i].item(),\n",
"# )"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": true,
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}