working from /home/tony/Work/glockenspiel/sunoGPT working from /home/tony/Work/glockenspiel/sunoGPT working from /home/tony/Work/glockenspiel/sunoGPT working from /home/tony/Work/glockenspiel/sunoGPT working from /home/tony/Work/glockenspiel/sunoGPT working from /home/tony/Work/glockenspiel/sunoGPT working from /home/tony/Work/glockenspiel/sunoGPT working from /home/tony/Work/glockenspiel/sunoGPT Overriding: out_dir = /app/suno/checkpoints Overriding: data_dir = /app/suno/data/dpo/7v_v20_full/ Overriding: train_filename = data_tr.bin Overriding: train_metas_filename = meta_tr.jsonl Overriding: train_info_filename = info_tr.json Overriding: val_filename = data_val.bin Overriding: val_metas_filename = meta_val.jsonl Overriding: val_info_filename = info_val.json Overriding: learning_rate = 5e-07 Overriding: min_lr = 1e-08 Overriding: grad_clip = 0.1 Overriding: warmup_iters = 100 Overriding: max_iters = 1250 Overriding: eval_interval = 500 Overriding: eval_iters = 25 Overriding: step_save_iters = 2000 Overriding: step_save_infer = False Overriding: block_size = 4288 Overriding: t_text = 1152 Overriding: t_memmap = 3008 Overriding: t_audio = 3136 Overriding: coarse_vocab_size = 2112 Overriding: coarse_codebook_size = 2048 Overriding: coarse_pad_token = 2048 Overriding: coarse_infer_token = 2049 Overriding: n_layer = 32 Overriding: n_head = 32 Overriding: d_head = 128 Overriding: n_kv_head = 4 Overriding: dpo_beta = 5.0 Overriding: semantic_codebook_weight = 4.0 Overriding: first_codebook_weight = 1.0 Overriding: last_codebook_weight = 0.5 Overriding: attention_type = tao Overriding: gradient_accumulation_steps = 1 Overriding: batch_size = 4 Overriding: fsdp = True Overriding: sharding_strategy = full_shard Overriding: grad_checkpointing = True Overriding: preload_checkpoint = /app/suno/checkpoints/2024-01-20_22-01-25/last_ckpt.pt Overriding: preload_strict = False Overriding: wandb_log = True Overriding: wandb_project = chirp-v3-dpo Overriding: wandb_run_name = 7b_dpo_t20_beta5_bt4_large8 [2024-03-19_03:10:35]: ddp init, rank 40, local_rank 0 [2024-03-19_03:10:35]: loading data... [2024-03-19_03:10:35]: 1,524 lines of val loaded. [2024-03-19_03:10:35]: ddp init, rank 48, local_rank 0 [2024-03-19_03:10:35]: loading data... [2024-03-19_03:10:35]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 42, local_rank 2 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 4, local_rank 4 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 1, local_rank 1 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 3, local_rank 3 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 5, local_rank 5 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 6, local_rank 6 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 2, local_rank 2 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 7, local_rank 7 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 41, local_rank 1 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 46, local_rank 6 [2024-03-19_03:10:36]: ddp init, rank 47, local_rank 7 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 45, local_rank 5 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 43, local_rank 3 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 44, local_rank 4 [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded.[2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 49, local_rank 1 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 50, local_rank 2 [2024-03-19_03:10:36]: ddp init, rank 51, local_rank 3 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 53, local_rank 5 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 54, local_rank 6 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 55, local_rank 7 [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 52, local_rank 4 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 24, local_rank 0 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 32, local_rank 0 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 8, local_rank 0 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 56, local_rank 0 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 16, local_rank 0 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 0, local_rank 0 loss discounts for codebooks: [0.308 0.077 0.073 0.07 0.066 0.063 0.059 0.056 0.052 0.049 0.045 0.042 0.038] [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 25, local_rank 1 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 37, local_rank 5 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 28, local_rank 4 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 27, local_rank 3 [2024-03-19_03:10:36]: ddp init, rank 33, local_rank 1 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 39, local_rank 7 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 30, local_rank 6 [2024-03-19_03:10:36]: ddp init, rank 34, local_rank 2 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 29, local_rank 5 [2024-03-19_03:10:36]: ddp init, rank 26, local_rank 2 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 31, local_rank 7 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 36, local_rank 4 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 38, local_rank 6 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 35, local_rank 3 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 12, local_rank 4 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 23, local_rank 7 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 15, local_rank 7 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 13, local_rank 5 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 57, local_rank 1 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 10, local_rank 2 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 14, local_rank 6 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 9, local_rank 1 [2024-03-19_03:10:36]: ddp init, rank 11, local_rank 3 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 17, local_rank 1 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 18, local_rank 2 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: 1,524 lines of val loaded. [2024-03-19_03:10:36]: ddp init, rank 22, local_rank 6 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 20, local_rank 4 [2024-03-19_03:10:36]: loading data... [2024-03-19_03:10:36]: ddp init, rank 21, local_rank 5 [2024-03-19_03:10:37]: loading data... [2024-03-19_03:10:37]: ddp init, rank 19, local_rank 3 [2024-03-19_03:10:37]: loading data... [2024-03-19_03:10:37]: ddp init, rank 58, local_rank 2 [2024-03-19_03:10:37]: 1,524 lines of val loaded. [2024-03-19_03:10:37]: loading data... [2024-03-19_03:10:37]: ddp init, rank 63, local_rank 7 [2024-03-19_03:10:37]: loading data... [2024-03-19_03:10:37]: 1,524 lines of val loaded. [2024-03-19_03:10:37]: ddp init, rank 62, local_rank 6 [2024-03-19_03:10:37]: loading data... [2024-03-19_03:10:37]: ddp init, rank 60, local_rank 4 [2024-03-19_03:10:37]: loading data... [2024-03-19_03:10:37]: 1,524 lines of val loaded. [2024-03-19_03:10:37]: ddp init, rank 59, local_rank 3 [2024-03-19_03:10:37]: loading data... [2024-03-19_03:10:37]: ddp init, rank 61, local_rank 5 [2024-03-19_03:10:37]: 1,524 lines of val loaded. [2024-03-19_03:10:37]: loading data... [2024-03-19_03:10:37]: 1,524 lines of val loaded. [2024-03-19_03:10:37]: 1,524 lines of val loaded. [2024-03-19_03:10:37]: 1,524 lines of val loaded. [2024-03-19_03:10:37]: 1,524 lines of val loaded. [2024-03-19_03:10:37]: 1,524 lines of val loaded. [2024-03-19_03:10:37]: 1,524 lines of val loaded. [2024-03-19_03:10:37]: 1,524 lines of val loaded. [2024-03-19_03:10:37]: 1,524 lines of val loaded. [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded.[2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch[2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded.[2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch[2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:37]: 150,818 lines of train loaded. [2024-03-19_03:10:37]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded.[2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch[2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded.[2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:10:38]: 150,818 lines of train loaded. [2024-03-19_03:10:38]: Initializing a new model from scratch [2024-03-19_03:11:05]: logging checkpoint here: /app/suno/checkpoints/2024-03-19_03-11-05 [2024-03-19_03:11:05]: loading data... [2024-03-19_03:11:05]: 1,524 lines of val loaded. [2024-03-19_03:11:07]: 150,818 lines of train loaded. [2024-03-19_03:11:07]: train data weights: 50.0% perference_0 50.0% perference_1 [2024-03-19_03:11:07]: Initializing a new model from scratch number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M number of parameters: 5869M [2024-03-19_03:12:54]: preloading checkpoint [2024-03-19_03:12:54]: not compiling model. number of parameters: 5869M number of parameters: 5869M [2024-03-19_03:12:55]: preloading checkpoint [2024-03-19_03:12:55]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:55]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:55]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:55]: preloading checkpoint [2024-03-19_03:12:55]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:55]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:55]: not compiling model. [2024-03-19_03:12:55]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:55]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:55]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:55]: preloading checkpoint [2024-03-19_03:12:55]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:55]: preloading checkpoint number of parameters: 5869M number of parameters: 5869M [2024-03-19_03:12:55]: preloading checkpoint [2024-03-19_03:12:55]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:55]: preloading checkpoint [2024-03-19_03:12:55]: not compiling model. [2024-03-19_03:12:55]: not compiling model. [2024-03-19_03:12:55]: not compiling model. [2024-03-19_03:12:55]: not compiling model. [2024-03-19_03:12:55]: not compiling model. [2024-03-19_03:12:55]: not compiling model. [2024-03-19_03:12:55]: not compiling model. [2024-03-19_03:12:55]: not compiling model. [2024-03-19_03:12:55]: not compiling model. [2024-03-19_03:12:55]: not compiling model. [2024-03-19_03:12:55]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:55]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:55]: preloading checkpoint [2024-03-19_03:12:55]: not compiling model. [2024-03-19_03:12:55]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:55]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:55]: preloading checkpoint [2024-03-19_03:12:56]: not compiling model. [2024-03-19_03:12:56]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:56]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:56]: preloading checkpoint [2024-03-19_03:12:56]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:56]: preloading checkpoint [2024-03-19_03:12:56]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:56]: preloading checkpoint [2024-03-19_03:12:56]: not compiling model. [2024-03-19_03:12:56]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:56]: preloading checkpoint [2024-03-19_03:12:56]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:57]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:57]: preloading checkpoint [2024-03-19_03:12:57]: not compiling model. [2024-03-19_03:12:57]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:57]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:57]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:57]: preloading checkpoint [2024-03-19_03:12:57]: not compiling model. [2024-03-19_03:12:57]: not compiling model. [2024-03-19_03:12:57]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:57]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:57]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:57]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:57]: preloading checkpoint [2024-03-19_03:12:57]: not compiling model. [2024-03-19_03:12:57]: not compiling model. [2024-03-19_03:12:57]: not compiling model. [2024-03-19_03:12:57]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:57]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:57]: preloading checkpoint [2024-03-19_03:12:58]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:58]: preloading checkpoint [2024-03-19_03:12:58]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:58]: not compiling model. [2024-03-19_03:12:58]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:58]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:58]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:58]: preloading checkpoint [2024-03-19_03:12:58]: not compiling model. [2024-03-19_03:12:58]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:58]: preloading checkpoint [2024-03-19_03:12:58]: not compiling model. [2024-03-19_03:12:58]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:58]: preloading checkpoint [2024-03-19_03:12:58]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:58]: preloading checkpoint [2024-03-19_03:12:58]: not compiling model. [2024-03-19_03:12:58]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:58]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:58]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:58]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:58]: preloading checkpoint [2024-03-19_03:12:58]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:59]: preloading checkpoint [2024-03-19_03:12:59]: not compiling model. [2024-03-19_03:12:59]: not compiling model. [2024-03-19_03:12:59]: not compiling model. [2024-03-19_03:12:59]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:59]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:59]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:59]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:59]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:12:59]: preloading checkpoint [2024-03-19_03:12:59]: not compiling model. [2024-03-19_03:12:59]: not compiling model. [2024-03-19_03:12:59]: not compiling model. number of parameters: 5869M [2024-03-19_03:12:59]: preloading checkpoint [2024-03-19_03:12:59]: not compiling model. [2024-03-19_03:12:59]: not compiling model. number of parameters: 5869M [2024-03-19_03:13:00]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:13:00]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:13:00]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:13:00]: preloading checkpoint [2024-03-19_03:13:00]: not compiling model. [2024-03-19_03:13:00]: not compiling model. [2024-03-19_03:13:00]: not compiling model. [2024-03-19_03:13:00]: not compiling model. [2024-03-19_03:13:00]: not compiling model. number of parameters: 5869M [2024-03-19_03:13:00]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:13:00]: preloading checkpoint [2024-03-19_03:13:00]: not compiling model. [2024-03-19_03:13:00]: not compiling model. number of parameters: 5869M [2024-03-19_03:13:00]: preloading checkpoint number of parameters: 5869M [2024-03-19_03:13:00]: preloading checkpoint [2024-03-19_03:13:01]: not compiling model. [2024-03-19_03:13:01]: not compiling model. number of parameters: 5869M [2024-03-19_03:13:12]: preloading checkpoint [2024-03-19_03:13:12]: not compiling model. number of parameters: 5869M [2024-03-19_03:13:25]: preloading checkpoint [2024-03-19_03:13:25]: not compiling model. [2024-03-19_03:13:25]: wrapping model in FSDP .... [2024-03-19_03:13:25]: loading state dict into model ... on rank 0 NCCL version 2.19.3+cuda12.3 applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing...applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing... applying fsdp activation checkpointing...applying fsdp activation checkpointing... applying fsdp activation checkpointing... num decayed parameter tensors: 220, with 93,733,120 parameters num non-decayed parameter tensors: 68, with 131,072 parameters using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,856,000 parameters num non-decayed parameter tensors: 68, with 8,192 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num decayed parameter tensors: 220, with 93,864,192 parametersnum non-decayed parameter tensors: 68, with 0 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters using fused Optimizer: False num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters using fused Optimizer: False using fused Optimizer: False num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False using fused Optimizer: Falsenum decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,860,096 parameters num non-decayed parameter tensors: 68, with 4,096 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parametersnum decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,860,096 parameters num non-decayed parameter tensors: 68, with 4,096 parameters using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num decayed parameter tensors: 220, with 93,864,192 parametersnum non-decayed parameter tensors: 68, with 0 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False num decayed parameter tensors: 220, with 93,733,120 parameters num non-decayed parameter tensors: 68, with 131,072 parameters using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters num decayed parameter tensors: 220, with 93,864,192 parameters num non-decayed parameter tensors: 68, with 0 parameters using fused Optimizer: False using fused Optimizer: False using fused Optimizer: False [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training...[2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training...[2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:13:57]: training... [2024-03-19_03:14:22]: loss estimation took 24.6 seconds. (100.0% of loop) [2024-03-19_03:14:22]: step 0: train loss 2.0787, val loss 1.9535 [2024-03-19_03:14:32]: iter 0: avg_loss 0.010, avg_acc 0.000, step_time 35044.0ms, mfu 0.0%, throughput 0k tok/s, total time 35s [2024-03-19_03:18:48]: iter 25: avg_loss 0.010, avg_acc 0.160, step_time 10513.2ms, mfu 21.9%, throughput 104k tok/s, total time 291s [2024-03-19_03:23:13]: iter 50: avg_loss 0.009, avg_acc 0.480, step_time 11976.2ms, mfu 19.3%, throughput 92k tok/s, total time 556s [2024-03-19_03:27:41]: iter 75: avg_loss 0.010, avg_acc 0.560, step_time 10104.5ms, mfu 22.8%, throughput 109k tok/s, total time 824s [2024-03-19_03:32:00]: iter 100: avg_loss 0.007, avg_acc 0.760, step_time 9577.5ms, mfu 24.1%, throughput 115k tok/s, total time 1083s [2024-03-19_03:36:18]: iter 125: avg_loss 0.011, avg_acc 0.320, step_time 9979.2ms, mfu 23.1%, throughput 110k tok/s, total time 1341s [2024-03-19_03:40:30]: iter 150: avg_loss 0.010, avg_acc 0.400, step_time 10573.5ms, mfu 21.8%, throughput 104k tok/s, total time 1593s [2024-03-19_03:44:53]: iter 175: avg_loss 0.009, avg_acc 0.560, step_time 10242.9ms, mfu 22.5%, throughput 107k tok/s, total time 1856s [2024-03-19_03:49:17]: iter 200: avg_loss 0.005, avg_acc 0.960, step_time 10856.2ms, mfu 21.2%, throughput 101k tok/s, total time 2120s [2024-03-19_03:53:32]: iter 225: avg_loss 0.008, avg_acc 0.600, step_time 9761.3ms, mfu 23.6%, throughput 112k tok/s, total time 2375s [2024-03-19_03:58:05]: iter 250: avg_loss 0.007, avg_acc 0.760, step_time 10963.5ms, mfu 21.0%, throughput 100k tok/s, total time 2647s [2024-03-19_04:02:25]: iter 275: avg_loss 0.008, avg_acc 0.640, step_time 10898.7ms, mfu 21.2%, throughput 101k tok/s, total time 2908s [2024-03-19_04:06:49]: iter 300: avg_loss 0.007, avg_acc 0.640, step_time 11941.2ms, mfu 19.3%, throughput 92k tok/s, total time 3172s [2024-03-19_04:11:05]: iter 325: avg_loss 0.008, avg_acc 0.680, step_time 10457.6ms, mfu 22.1%, throughput 105k tok/s, total time 3428s [2024-03-19_04:15:28]: iter 350: avg_loss 0.010, avg_acc 0.440, step_time 11091.5ms, mfu 20.8%, throughput 99k tok/s, total time 3690s [2024-03-19_04:19:52]: iter 375: avg_loss 0.007, avg_acc 0.640, step_time 9804.5ms, mfu 23.5%, throughput 112k tok/s, total time 3954s [2024-03-19_04:24:09]: iter 400: avg_loss 0.011, avg_acc 0.480, step_time 10332.8ms, mfu 22.3%, throughput 106k tok/s, total time 4211s [2024-03-19_04:28:39]: iter 425: avg_loss 0.008, avg_acc 0.720, step_time 10086.0ms, mfu 22.9%, throughput 109k tok/s, total time 4482s [2024-03-19_04:33:09]: iter 450: avg_loss 0.008, avg_acc 0.680, step_time 9546.1ms, mfu 24.2%, throughput 115k tok/s, total time 4752s [2024-03-19_04:37:29]: iter 475: avg_loss 0.009, avg_acc 0.640, step_time 9810.8ms, mfu 23.5%, throughput 112k tok/s, total time 5012s [2024-03-19_04:42:13]: loss estimation took 24.6 seconds. (0.5% of loop) [2024-03-19_04:42:13]: step 500: train loss 2.0677, val loss 1.9807 [2024-03-19_04:42:48]: saving checkpoint to /app/suno/checkpoints/2024-03-19_03-11-05 [2024-03-19_04:48:56]: saving took 367.9 seconds. [2024-03-19_04:49:04]: iter 500: avg_loss 0.007, avg_acc 0.600, step_time 436289.3ms, mfu 0.5%, throughput 3k tok/s, total time 5707s [2024-03-19_04:53:17]: iter 525: avg_loss 0.011, avg_acc 0.440, step_time 9590.0ms, mfu 24.1%, throughput 114k tok/s, total time 5960s [2024-03-19_04:57:30]: iter 550: avg_loss 0.008, avg_acc 0.680, step_time 10271.7ms, mfu 22.5%, throughput 107k tok/s, total time 6213s [2024-03-19_05:02:00]: iter 575: avg_loss 0.008, avg_acc 0.640, step_time 11409.0ms, mfu 20.2%, throughput 96k tok/s, total time 6483s [2024-03-19_05:06:19]: iter 600: avg_loss 0.008, avg_acc 0.640, step_time 9666.5ms, mfu 23.9%, throughput 114k tok/s, total time 6742s [2024-03-19_05:10:39]: iter 625: avg_loss 0.006, avg_acc 0.840, step_time 10250.2ms, mfu 22.5%, throughput 107k tok/s, total time 7002s [2024-03-19_05:14:59]: iter 650: avg_loss 0.009, avg_acc 0.560, step_time 10088.6ms, mfu 22.9%, throughput 109k tok/s, total time 7261s [2024-03-19_05:19:10]: iter 675: avg_loss 0.007, avg_acc 0.760, step_time 11102.5ms, mfu 20.8%, throughput 99k tok/s, total time 7513s [2024-03-19_05:23:31]: iter 700: avg_loss 0.008, avg_acc 0.560, step_time 11302.3ms, mfu 20.4%, throughput 97k tok/s, total time 7773s [2024-03-19_05:27:48]: iter 725: avg_loss 0.008, avg_acc 0.680, step_time 10500.1ms, mfu 22.0%, throughput 105k tok/s, total time 8031s [2024-03-19_05:32:08]: iter 750: avg_loss 0.005, avg_acc 0.880, step_time 11168.9ms, mfu 20.7%, throughput 98k tok/s, total time 8291s [2024-03-19_05:36:27]: iter 775: avg_loss 0.009, avg_acc 0.560, step_time 10356.2ms, mfu 22.3%, throughput 106k tok/s, total time 8550s [2024-03-19_05:40:45]: iter 800: avg_loss 0.006, avg_acc 0.880, step_time 9727.8ms, mfu 23.7%, throughput 113k tok/s, total time 8807s [2024-03-19_05:45:05]: iter 825: avg_loss 0.007, avg_acc 0.720, step_time 10818.2ms, mfu 21.3%, throughput 101k tok/s, total time 9068s [2024-03-19_05:49:31]: iter 850: avg_loss 0.006, avg_acc 0.800, step_time 9280.9ms, mfu 24.9%, throughput 118k tok/s, total time 9334s [2024-03-19_05:53:59]: iter 875: avg_loss 0.008, avg_acc 0.480, step_time 11310.0ms, mfu 20.4%, throughput 97k tok/s, total time 9601s [2024-03-19_05:58:17]: iter 900: avg_loss 0.006, avg_acc 0.800, step_time 9385.0ms, mfu 24.6%, throughput 117k tok/s, total time 9860s [2024-03-19_06:02:38]: iter 925: avg_loss 0.006, avg_acc 0.760, step_time 8890.5ms, mfu 25.9%, throughput 123k tok/s, total time 10121s [2024-03-19_06:06:48]: iter 950: avg_loss 0.007, avg_acc 0.640, step_time 10039.8ms, mfu 23.0%, throughput 109k tok/s, total time 10371s [2024-03-19_06:11:13]: iter 975: avg_loss 0.009, avg_acc 0.560, step_time 10215.9ms, mfu 22.6%, throughput 107k tok/s, total time 10635s [2024-03-19_06:15:49]: loss estimation took 25.4 seconds. (0.5% of loop) [2024-03-19_06:15:49]: step 1000: train loss 2.0739, val loss 2.0143 [2024-03-19_06:16:30]: saving checkpoint to /app/suno/checkpoints/2024-03-19_03-11-05 [2024-03-19_06:21:59]: saving took 332.9 seconds. [2024-03-19_06:22:06]: iter 1000: avg_loss 0.006, avg_acc 0.880, step_time 402400.1ms, mfu 0.6%, throughput 3k tok/s, total time 11289s [2024-03-19_06:26:27]: iter 1025: avg_loss 0.008, avg_acc 0.640, step_time 11491.6ms, mfu 20.1%, throughput 96k tok/s, total time 11550s [2024-03-19_06:30:50]: iter 1050: avg_loss 0.007, avg_acc 0.720, step_time 10233.6ms, mfu 22.5%, throughput 107k tok/s, total time 11813s [2024-03-19_06:35:16]: iter 1075: avg_loss 0.007, avg_acc 0.640, step_time 11163.1ms, mfu 20.7%, throughput 98k tok/s, total time 12078s [2024-03-19_06:39:35]: iter 1100: avg_loss 0.008, avg_acc 0.600, step_time 10973.8ms, mfu 21.0%, throughput 100k tok/s, total time 12337s [2024-03-19_06:44:04]: iter 1125: avg_loss 0.007, avg_acc 0.760, step_time 11132.0ms, mfu 20.7%, throughput 99k tok/s, total time 12607s [2024-03-19_06:48:29]: iter 1150: avg_loss 0.007, avg_acc 0.720, step_time 9700.6ms, mfu 23.8%, throughput 113k tok/s, total time 12872s [2024-03-19_06:52:58]: iter 1175: avg_loss 0.007, avg_acc 0.680, step_time 9645.8ms, mfu 23.9%, throughput 114k tok/s, total time 13141s [2024-03-19_06:57:30]: iter 1200: avg_loss 0.008, avg_acc 0.680, step_time 9857.9ms, mfu 23.4%, throughput 111k tok/s, total time 13412s [2024-03-19_07:01:59]: iter 1225: avg_loss 0.008, avg_acc 0.680, step_time 9812.1ms, mfu 23.5%, throughput 112k tok/s, total time 13682s [2024-03-19_07:06:30]: loss estimation took 24.9 seconds. (0.8% of loop) [2024-03-19_07:06:30]: step 1249: train loss 2.0973, val loss 2.0315 [2024-03-19_07:07:12]: saving checkpoint to /app/suno/checkpoints/2024-03-19_03-11-05 [2024-03-19_07:12:13]: saving took 306.6 seconds. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done.[2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: iter 1249: avg_loss 0.008, avg_acc 0.708, step_time 375729.0ms, mfu 0.6%, throughput 3k tok/s, total time 14304s [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done. [2024-03-19_07:12:21]: done.