{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "470c7030",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sinfo - showing the compute nodes\n",
    "# squeue - shows the jobs and state in the queue\n",
    "# scontrol - for managing the compute nodes (needs sudo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ad5c8b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "cat /etc/hosts | grep compute-hpc-node-166 | gawk -F' ' '{ print $2 \" \" $4}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f11a4b83",
   "metadata": {},
   "outputs": [],
   "source": [
    "compute-hpc-node-166-rdma.local.rdma inst-qsajg-a100-pkix01-535-02-rdma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bcd68149",
   "metadata": {},
   "outputs": [],
   "source": [
    "oci compute instance list --query \"data[?contains(\\\"display-name\\\",'inst-qsajg-a100-pkix01-535-02')]\".{\"ocid:id\"} --compartment-id ocid1.compartment.oc1..aaaaaaaa276be2q76ht2d66vzb6mga727vltmmzfxdz2j7dm6ummkmaqvfbq --auth instance_principal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7eaabf3f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7448d78f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23a33987",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51da4c6c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0550644c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # rack 1\n",
    "# compute-hpc-node-294 (georg, main gpt)\n",
    "# compute-hpc-node-378 (georg, main gpt)\n",
    "# # rack 2\n",
    "# compute-hpc-node-365 (victor, codec)\n",
    "# compute-hpc-node-597 (victor, codec)\n",
    "# # rack 3\n",
    "# compute-hpc-node-54  (georg, main gpt)\n",
    "# # rack 4\n",
    "# compute-hpc-node-78  (tony, hoot)\n",
    "# # rack 6\n",
    "# compute-hpc-node-310 (victor, codec)\n",
    "# # rack 7\n",
    "# compute-hpc-node-327 (victor, codec)\n",
    "# # rack 8\n",
    "# compute-hpc-node-409 (georg, dac)\n",
    "# # rack 9\n",
    "# compute-hpc-node-541 (georg, dac)\n",
    "\n",
    "\n",
    "# # rack 5\n",
    "# compute-hpc-node-166 (georg, main gpt) -- (borked)\n",
    "# # rack 10\n",
    "# compute-hpc-node-658 (georg, dac)\n",
    "# # rack 11\n",
    "# compute-hpc-node-692 (mikey, misc)\n",
    "# # rack 12\n",
    "# compute-hpc-node-703 (mikey, misc)\n",
    "# # rack 13\n",
    "# compute-hpc-node-742 (victor, misc)\n",
    "# # rack 14\n",
    "# compute-hpc-node-793 (tony, misc)\n",
    "# # rack 15\n",
    "# compute-hpc-node-868 (victor, diffusion)\n",
    "# # rack 16\n",
    "# compute-hpc-node-958 (tony, hoot)\n",
    "# # rack 17\n",
    "# compute-hpc-node-998 (victor, diffusion)\n",
    "\n",
    "# a10compute535-gpu-node-314 (victor, pat)\n",
    "# a10compute535-gpu-node-780 (tony)\n",
    "# a10compute535-gpu-node-805\n",
    "# a10compute535-gpu-node-927 (victor)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "368c2622",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bc61f0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pytorch-triton==2.1.0+6e4932cda8\n",
    "# torch==2.2.0.dev20231027+cu118\n",
    "# torchaudio==2.2.0.dev20231027+cu118"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81660452",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokens_per_sample*batch_size*n_gpu*n_nodes*n_tot_steps/token_hz/secs_per_min/mins_per_hour\n",
    "# f\"{3008*8*8*6*100_000/25/60/60:,.0f} hours\"\n",
    "# 1,283,413 hours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a454a9ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "python train.py \\\n",
    "    --out_dir=\"/app/suno/checkpoints\" \\\n",
    "    --data_dir=\"/mnt/localdisk/data\" \\\n",
    "    --debug_val_only=True \\\n",
    "    --eval_iters=50 \\\n",
    "    --block_size=4288 \\\n",
    "    --t_text=1152 \\\n",
    "    --t_memmap=3008 \\\n",
    "    --t_audio=3136 \\\n",
    "    --gradient_accumulation_steps=1 \\\n",
    "    --batch_size=4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75f473f5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85c10079",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "a972848a",
   "metadata": {},
   "source": [
    "### lg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a643ae56",
   "metadata": {},
   "outputs": [],
   "source": [
    "OMP_NUM_THREADS=1 torchrun \\\n",
    "    --nnodes=8 --nproc_per_node=8 \\\n",
    "    --rdzv_backend=c10d --rdzv_id=123 --rdzv_endpoint=\"compute-hpc-node-294:1234\" \\\n",
    "    train.py \\\n",
    "    \\\n",
    "    --out_dir=\"/app/suno/checkpoints\" \\\n",
    "    --data_dir=\"/mnt/localdisk/data\" \\\n",
    "    \\\n",
    "    --learning_rate=5e-4 \\\n",
    "    --max_iters=100_000 \\\n",
    "    \\\n",
    "    --block_size=4288 \\\n",
    "    --t_text=1152 \\\n",
    "    --t_memmap=3008 \\\n",
    "    --t_audio=3136 \\\n",
    "    \\\n",
    "    --min_text_offs=1152 \\\n",
    "    \\\n",
    "    --n_layer=36 \\\n",
    "    --n_head=20 \\\n",
    "    --d_head=64 \\\n",
    "    \\\n",
    "    --gradient_accumulation_steps=2 \\\n",
    "    --batch_size=4 \\\n",
    "    \\\n",
    "    --fsdp=True \\\n",
    "    --sharding_strategy=\"full_shard\" \\\n",
    "    \\\n",
    "    --wandb_log=True \\\n",
    "    --wandb_project=\"chirp-v2\" \\\n",
    "    --wandb_run_name=\"lg_fixText\"\n",
    "\n",
    "#     \\\n",
    "#     --dropout=0.1 \\\n",
    "#     --n_kv_head=4 \\\n",
    "#     --d_head=64 \\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc4756af",
   "metadata": {},
   "outputs": [],
   "source": [
    "#     \\\n",
    "#     --preload_checkpoint=\"/app/suno/checkpoints/2023-11-09_23-46-02/last_ckpt.pt\" \\\n",
    "#     --preload_optimizer=True \\\n",
    "#     --custom_seed_offset=2 \\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "365909d1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c24ba9eb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2137cd3c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "be0a48d4",
   "metadata": {},
   "source": [
    "### xxl 3b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15150c72",
   "metadata": {},
   "outputs": [],
   "source": [
    "# batch: ~2M tokens: 3k(context)*2(batch_size)*2(grad_accum)*8(n_gpu)*16(n_nodes)\n",
    "#  -> for 1T - 500_000 steps needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afba228c",
   "metadata": {},
   "outputs": [],
   "source": [
    "OMP_NUM_THREADS=1 torchrun \\\n",
    "    --nnodes=16 --nproc_per_node=8 \\\n",
    "    --rdzv_backend=c10d --rdzv_id=123 --rdzv_endpoint=\"compute-hpc-node-294:1234\" \\\n",
    "    train.py \\\n",
    "    \\\n",
    "    --out_dir=\"/app/suno/checkpoints\" \\\n",
    "    --data_dir=\"/mnt/localdisk/data\" \\\n",
    "    \\\n",
    "    --learning_rate=5e-4 \\\n",
    "    --max_iters=600_000 \\\n",
    "    --eval_interval=1000 \\\n",
    "    --log_interval=10 \\\n",
    "    \\\n",
    "    --block_size=4288 \\\n",
    "    --t_text=1152 \\\n",
    "    --t_memmap=3008 \\\n",
    "    --t_audio=3136 \\\n",
    "    \\\n",
    "    --n_layer=48 \\\n",
    "    --n_head=32 \\\n",
    "    --n_embd=2048 \\\n",
    "    \\\n",
    "    --gradient_accumulation_steps=2 \\\n",
    "    --batch_size=2 \\\n",
    "    \\\n",
    "    --fsdp=True \\\n",
    "    --sharding_strategy=\"full_shard\" \\\n",
    "    \\\n",
    "    --wandb_log=True \\\n",
    "    --wandb_project=\"chirp-v2\" \\\n",
    "    --wandb_run_name=\"xxl\"\n",
    "\n",
    "#     \\\n",
    "#     --preload_checkpoint=\"/app/suno/checkpoints/2023-10-16_01-02-51/last_ckpt.pt\" \\\n",
    "#     --preload_optimizer=True \\\n",
    "#     --custom_seed_offset=4 \\"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5b1748c",
   "metadata": {},
   "source": [
    "### Larger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "637ddafc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 7B\n",
    "# --n_layer=48 \\\n",
    "# --n_head=32 \\\n",
    "# --n_embd=3328 \\\n",
    "# or \n",
    "# --n_layer=32 \\\n",
    "# --n_head=32 \\\n",
    "# --n_embd=4096 \\\n",
    "\n",
    "# # 13B (12,680M)\n",
    "# --n_layer=48 \\\n",
    "# --n_head=36 \\\n",
    "# --n_embd=4608 \\\n",
    "# or (13,082M)\n",
    "# --n_layer=40 \\\n",
    "# --n_head=40 \\\n",
    "# --n_embd=5120 \\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84e3b052",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "60bfa587",
   "metadata": {},
   "source": [
    "### finetune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bd235ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/ft_info_val.json /app/suno/data/chirp_v2/\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/ft_info_tr.json /app/suno/data/chirp_v2/\n",
    "\n",
    "# pdsh -w compute-hpc-node-[54,78,294,310,327,365,378,409,541,597,658,692,703,742,793,868,958,998] \"cp /app/suno/data/chirp_v2/ft_info_val.json /mnt/localdisk/data\"\n",
    "# pdsh -w compute-hpc-node-[54,78,294,310,327,365,378,409,541,597,658,692,703,742,793,868,958,998] \"cp /app/suno/data/chirp_v2/ft_info_tr.json /mnt/localdisk/data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cef7970",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train\n",
    "#  genius_hq_lyrics: 1,412.7 hours, 58,067 samples\n",
    "#  pond5_music: 209.9 hours, 10,149 samples\n",
    "\n",
    "# val\n",
    "#  genius_hq_lyrics: 157.9 hours, 6,452 samples\n",
    "#  pond5_music: 23.1 hours, 1,128 samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4fc04e8e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "137 steps is one epoch\n"
     ]
    }
   ],
   "source": [
    "print(int(round(70_000/(8*8*8))), \"steps is one epoch\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da8d97eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: test dropout etc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebb2a838",
   "metadata": {},
   "outputs": [],
   "source": [
    "OMP_NUM_THREADS=1 torchrun \\\n",
    "    --nnodes=8 --nproc_per_node=8 \\\n",
    "    --rdzv_backend=c10d --rdzv_id=123 --rdzv_endpoint=\"compute-hpc-node-294:1234\" \\\n",
    "    train.py \\\n",
    "    \\\n",
    "    --out_dir=\"/app/suno/checkpoints\" \\\n",
    "    --data_dir=\"/mnt/localdisk/data\" \\\n",
    "    \\\n",
    "    --train_filename=\"data_tr.bin\" \\\n",
    "    --train_metas_filename=\"metas_tr.jsonl\" \\\n",
    "    --train_info_filename=\"ft_info_tr.json\" \\\n",
    "    \\\n",
    "    --val_filename=\"data_tr.bin\" \\\n",
    "    --val_metas_filename=\"metas_tr.jsonl\" \\\n",
    "    --val_info_filename=\"ft_info_val.json\" \\\n",
    "    \\\n",
    "    --preload_checkpoint=\"/app/suno/checkpoints/2023-10-17_16-01-32/last_ckpt.pt\" \\\n",
    "    --weights_multiplier=\"genius_hq_lyrics:1;pond5_music:1\" \\\n",
    "    --is_finetune=True \\\n",
    "    \\\n",
    "    --learning_rate=2e-5 \\\n",
    "    --max_iters=5_000 \\\n",
    "    --eval_interval=100 \\\n",
    "    --eval_iters=500 \\\n",
    "    --log_interval=10 \\\n",
    "    \\\n",
    "    --block_size=4288 \\\n",
    "    --t_text=1152 \\\n",
    "    --t_memmap=3008 \\\n",
    "    --t_audio=3136 \\\n",
    "    \\\n",
    "    --n_layer=48 \\\n",
    "    --n_head=32 \\\n",
    "    --d_head=64 \\\n",
    "    \\\n",
    "    --gradient_accumulation_steps=2 \\\n",
    "    --batch_size=2 \\\n",
    "    \\\n",
    "    --fsdp=True \\\n",
    "    --sharding_strategy=\"full_shard\" \\\n",
    "    \\\n",
    "    --wandb_log=True \\\n",
    "    --wandb_project=\"chirp-v2-ft\" \\\n",
    "    --wandb_run_name=\"xxl_ft\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eeb5ca37",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5adfb1a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d2c2e9e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "4bd925fd",
   "metadata": {},
   "source": [
    "### misc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "051ed6f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#     --preload_checkpoint=\"/app/suno/checkpoints/2023-06-09_19-46-20/last_ckpt.pt\" \\\n",
    "#     --custom_seed_offset=2 \\\n",
    "#     --preload_optimizer=True \\\n",
    "\n",
    "#     --preload_strict=False \\\n",
    "\n",
    "#     --dummy_data=True \\\n",
    "#     --debug_val_only=True \\\n",
    "#     --eval_iters=50 \\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8ebb9cc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91a17875",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "406881e2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "dbf7befa",
   "metadata": {},
   "source": [
    "## Speed test large"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08e2c64b",
   "metadata": {},
   "outputs": [],
   "source": [
    "OMP_NUM_THREADS=1 NCCL_DEBUG=WARN NCCL_IB_SL=0 NCCL_IB_TC=41 NCCL_IB_QPS_PER_CONNECTION=4 UCX_TLS=ud,self,sm \\\n",
    "UCX_NET_DEVICES=mlx5_0:1 coll_hcoll_enable=0 \\\n",
    "NCCL_IB_HCA=\"=mlx5_5,mlx5_6,mlx5_7,mlx5_8,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_14,mlx5_15,mlx5_16,mlx5_17,mlx5_9,mlx5_10,mlx5_11,mlx5_12\" \\\n",
    "HCOLL_ENABLE_MCAST_ALL=0 NCCL_IB_GID_INDEX=3 \\\n",
    "torchrun --standalone --nnodes=1 --nproc-per-node=8 \\\n",
    "    train.py \\\n",
    "    \\\n",
    "    --out_dir=\"/app/suno/checkpoints\" \\\n",
    "    --data_dir=\"/mnt/localdisk/data_2\" \\\n",
    "    \\\n",
    "    --eval_interval=1000 \\\n",
    "    --log_interval=10 \\\n",
    "    --debug_val_only=True \\\n",
    "    --eval_iters=50 \\\n",
    "    \\\n",
    "    --block_size=4288 \\\n",
    "    --t_text=1152 \\\n",
    "    --t_memmap=3008 \\\n",
    "    --t_audio=3136 \\\n",
    "    \\\n",
    "    --n_layer=26 \\\n",
    "    --n_head=20 \\\n",
    "    --n_embd=1280 \\\n",
    "    \\\n",
    "    --gradient_accumulation_steps=2 \\\n",
    "    --batch_size=2 \\\n",
    "    \\\n",
    "    --fsdp=True \\\n",
    "    --sharding_strategy=\"full_shard\"\n",
    "    \n",
    "# step_time 486.7ms, mfu 62.4%, throughput 280k tok/s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28f09e11",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a298f51e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96bbe306",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "30f09ed0",
   "metadata": {},
   "source": [
    "## Transfer data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a5426d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# mkdir -p /app/suno/data/chirp_v2/\n",
    "\n",
    "# aws s3 cp s3://suno-data/georg/trained_models/chirp_v2/tokenizer_60k.json /app/suno/data/chirp_v2/\n",
    "\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/info_val.json /app/suno/data/chirp_v2/\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/metas_val.jsonl /app/suno/data/chirp_v2/\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/data_val.bin /app/suno/data/chirp_v2/\n",
    "\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/info_tr.json /app/suno/data/chirp_v2/\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/metas_tr.jsonl /app/suno/data/chirp_v2/\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/data_tr.bin /app/suno/data/chirp_v2/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac93eb0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pdsh -w compute-hpc-node-[54,78,166,294,310,327,365,378,409,541,597,658,692,703,742,793,868,958,998] \\\n",
    "#     \"rsync -ah --progress /app/suno/data/chirp_v2/ /mnt/localdisk/data/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9b615eb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c445df79",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "fe1a89b4",
   "metadata": {},
   "source": [
    "### misc other data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1db332b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# aws s3 cp s3://suno-data/georg/trained_models/chirp_v2/tokenizer_60k.json /app/suno/data/chirp_v2_5/\n",
    "\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/info_val.json /app/suno/data/chirp_v2_5/\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/metas_val.jsonl /app/suno/data/chirp_v2_5/\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/data_val.bin /app/suno/data/chirp_v2_5/\n",
    "\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/info_tr.json /app/suno/data/chirp_v2_5/\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/metas_tr.jsonl /app/suno/data/chirp_v2_5/\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/data_tr.bin /app/suno/data/chirp_v2_5/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64aee051",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ssh compute-hpc-node-13\n",
    "# mkdir -p /mnt/localdisk/data_2/\n",
    "\n",
    "# aws s3 cp s3://suno-data/georg/trained_models/chirp_v2/tokenizer_60k.json /mnt/localdisk/data_2/\n",
    "\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/info_val.json /mnt/localdisk/data/\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/metas_val.jsonl /mnt/localdisk/data/\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/data_val.bin /mnt/localdisk/data/\n",
    "\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/info_tr.json /mnt/localdisk/data/\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/metas_tr.jsonl /mnt/localdisk/data/\n",
    "# aws s3 cp s3://suno-data/georg/data/chirp_v2/data_tr.bin /mnt/localdisk/data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57d45d8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 13,75,335,454, 510,576,584,646,711,794,857,891,982"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45211256",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pdsh -w compute-hpc-node-[335,454] \"cp -r /app/suno_tmp/data /mnt/localdisk/data\"\n",
    "# pdsh -w compute-hpc-node-[510,576,584,646,711,794,857,891,982] \"cp -r /app/suno_tmp/data /mnt/localdisk/data\"\n",
    "\n",
    "# pdsh -w compute-hpc-node-[335,454] \"rsync -ah --progress /app/suno_tmp/data/ /mnt/localdisk/data/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "989be84a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# rsync -a -e ssh user@remote:/path/to/destination /path/to/source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfd60e06",
   "metadata": {},
   "outputs": [],
   "source": [
    "rsync -a --progress -e ssh /app/suno_tmp/data_2/ /mnt/localdisk/data_2/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "078796e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pdsh -w compute-hpc-node-[111,222] \\\n",
    "#     \"rsync -a --progress -e ssh compute-hpc-node-13:/mnt/localdisk/data_2/ /mnt/localdisk/data_2/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9d8e4cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "/mnt/localdisk/data_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1736f52",
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute-hpc-node-982 (tony)\n",
    "# 330,438,440,540,736"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74bfa442",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pdsh -w compute-hpc-node-[330,438,440,540,736] \\\n",
    "#     \"rsync -a --progress -e ssh compute-hpc-node-13:/mnt/localdisk/data/ /mnt/localdisk/data/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "841c91a9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1096d3fe",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "fb784c34",
   "metadata": {},
   "source": [
    "## Playground"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "860f593a",
   "metadata": {},
   "outputs": [],
   "source": [
    "default (xl, fsdp 2/2, dummy_data)\n",
    "55.2% mfu, 98k tok/s, 34/40Gb\n",
    "\n",
    "real data\n",
    "53.7% mfu, 95k tok/s\n",
    "\n",
    "xformers mem_eff (torch 2.0)\n",
    "62.0% mfu, 110k tok/s\n",
    "\n",
    "xformers sdpa (torch 2.0)\n",
    "70.5% mfu, 124k tok/s\n",
    "\n",
    "pytorch nightly (flash 2)\n",
    "64.3% mfu, 114k tok/s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d3d0231",
   "metadata": {},
   "outputs": [],
   "source": [
    "default (xl, fsdp 2/2, dummy_data)\n",
    "55.2% mfu, 98k tok/s, 34/40Gb\n",
    "\n",
    "xformers (torch 2.0)\n",
    "63.0% mfu, 112k tok/s\n",
    "\n",
    "pytorch nightly (flash 2)\n",
    "64.3% mfu, 114k tok/s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb43f948",
   "metadata": {},
   "outputs": [],
   "source": [
    "iter 25: avg_loss 1.621, step_time 1193.5ms, mfu 62.0%, throughput 110k tok/s, total time 37s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58984be9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# batch 2\n",
    "mfu 63.3%, throughput 112k tok/s\n",
    "with compile:\n",
    "65.3%, throughput 116k tok/s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38030efc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31307c76",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "659ee599",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4302b0c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}