predict.sh

#!/bin/bash

CHECKPOINTS=(
    "runs/run_100/checkpoints/best-score-epoch=81-val_final_score=04756.ckpt"
    "runs/run_100/checkpoints/best-score-epoch=84-val_final_score=04753.ckpt"
    "runs/run_100/checkpoints/best-iou-build-epoch=54-val_iou_build=03743.ckpt"
)

for i in "${!CHECKPOINTS[@]}"; do
    ckpt="${CHECKPOINTS[$i]}"
    pred_dir="predictions/run_100_$i"
    out_file="predict_100_$i.out"
    err_file="predict_100_$i.err"

    # Submit each as an independent SLURM job
    #SBATCH --gres=gpu:1
    sbatch <<EOF
#!/bin/bash
#SBATCH -p rivulet,brook
#SBATCH --mem=120G
#SBATCH --time=0-2:00:00
#SBATCH -o $out_file
#SBATCH -e $err_file

source /opt/csg/anaconda/etc/profile.d/conda.sh
conda activate /csghome/sm330/emb_env

export WANDB_API_KEY=\$(cat /csghome/sm330/.netrc | grep password | awk '{print \$2}')
export PYTHONPATH="/csghome/sm330/embed2heights-challenge-submission:\$PYTHONPATH"

srun -u env PYTHONNOUSERSITE=1 \
    python src/predict.py \
    --data-root data/npy \
    --checkpoint "$ckpt" \
    --predictions-dir "$pred_dir" \
    --batch-size 32 \
    --num-workers 1
EOF
done