#!/bin/bash
export CUDA_LAUNCH_BLOCKING=0
export NCCL_DEBUG=WARN
export TORCH_DISTRIBUTED_DEBUG=OFF
export TORCH_CPP_LOG_LEVEL=WARNING

export OMP_NUM_THREADS=1
export HOSTNAMES=`scontrol show hostnames "$SLURM_JOB_NODELIST"`
export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
export MASTER_PORT=12832
export COUNT_NODE=`scontrol show hostnames "$SLURM_JOB_NODELIST" | wc -l`

export NCCL_DEBUG=WARN
export NCCL_CROSS_NIC=2

export TRITON_CACHE_DIR=/mnt/localdisk/.triton_cache_minz

TRAIN_PATH=/home/minz/neon/ditto-training
echo working from $TRAIN_PATH
cd $TRAIN_PATH

torchrun \
    --standalone\
    --nproc_per_node=1 \
    train_nolightning.py