#!/bin/bash -l

# SLURM SUBMIT SCRIPT
#SBATCH --job-name=boost
#SBATCH --nodes=1             # This needs to match Trainer(num_nodes=...)
#SBATCH --gres=gpu:8
#SBATCH --ntasks-per-node=8   # This needs to match Trainer(devices=...)

# ACTIVATE ANACONDA
eval "$(conda shell.bash hook)"

# activate conda env
conda activate boost-env

# debugging flags (optional)
export NCCL_DEBUG=INFO
export PYTHONFAULTHANDLER=1

# on your cluster you might need these:
# set the network interface
# export NCCL_SOCKET_IFNAME=^docker0,lo

# might need the latest CUDA
#module load NCCL/2.4.7-1-cuda.10.0

# run script from above
srun python3 scripts/main.py fit \
-c cfg/trainer.yaml --trainer.num_nodes 1 --trainer.devices 8 \
-c cfg/models/model.yaml \
-c cfg/data/upsample_genius.yaml --data.batch_size 16