DEIMv2/run_ddp_train_dinov3_x_crowd_human_mot.sh at main · NikhilSandy/DEIMv2 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
#!/usr/bin/env bash
set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "${ROOT_DIR}"

CUDA_DEVICES="${CUDA_DEVICES:-2}"
MASTER_PORT="${MASTER_PORT:-17786}"
SEED="${SEED:-42}"
PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
NCCL_ASYNC_ERROR_HANDLING="${NCCL_ASYNC_ERROR_HANDLING:-1}"
NCCL_IB_DISABLE="${NCCL_IB_DISABLE:-1}"
NCCL_P2P_DISABLE="${NCCL_P2P_DISABLE:-0}"

CONFIG="${CONFIG:-configs/deimv2/deimv2_dinov3_x_crowd_human_mot.yml}"
OUTPUT_DIR="${OUTPUT_DIR:-outputs/deimv2_dinov3_x_crowd_human_mot}"
TRAIN_ANN="${TRAIN_ANN:-/home/nikhileswara/DEIMv2/data/crowd_human_mot_coco/annotations/instances_train_id01_crowdhuman_nega_minus1000neg_le100.json}"
VAL_ANN="${VAL_ANN:-/home/nikhileswara/DEIMv2/data/crowd_human_mot_coco/annotations/instances_val_id01_crowdhuman_nega_plus1000neg.json}"
TUNING_CKPT="${TUNING_CKPT:-/home/nikhileswara/DEIMv2/ckpts/deimv2_dinov3_x_coco.pth}"
AUTO_RESUME="${AUTO_RESUME:-0}"

IFS=',' read -r -a GPU_ARRAY <<< "${CUDA_DEVICES}"
NPROC_PER_NODE="${NPROC_PER_NODE:-${#GPU_ARRAY[@]}}"

for required in "${CONFIG}" "${TRAIN_ANN}" "${VAL_ANN}"; do
  if [[ ! -e "${required}" ]]; then
    echo "Missing required file: ${required}" >&2
    exit 1
  fi
done
if [[ ! -e "${TUNING_CKPT}" ]]; then
  echo "Missing TUNING_CKPT: ${TUNING_CKPT}" >&2
  exit 1
fi

mkdir -p "${OUTPUT_DIR}"
timestamp="$(date +%Y%m%d_%H%M%S)"
log_file="${OUTPUT_DIR}/train_${timestamp}.log"

RESUME_CKPT="${RESUME_CKPT:-${OUTPUT_DIR}/last.pth}"
mode_args=()
if [[ "${AUTO_RESUME}" == "1" && -f "${RESUME_CKPT}" ]]; then
  echo "Resuming from ${RESUME_CKPT}"
  mode_args=(-r "${RESUME_CKPT}")
else
  echo "Starting fresh from tuning checkpoint ${TUNING_CKPT}"
  mode_args=(-t "${TUNING_CKPT}")
fi

cmd=(
  torchrun
  --master_port="${MASTER_PORT}"
  --nproc_per_node="${NPROC_PER_NODE}"
  train.py
  -c "${CONFIG}"
  --seed "${SEED}"
  --output-dir "${OUTPUT_DIR}"
  -u "train_dataloader.dataset.ann_file=${TRAIN_ANN}"
  -u "val_dataloader.dataset.ann_file=${VAL_ANN}"
  "${mode_args[@]}"
)

if [[ "${USE_AMP:-0}" == "1" ]]; then
  cmd+=(--use-amp)
fi

if [[ "$#" -gt 0 ]]; then
  cmd+=("$@")
fi

echo "Launching command:"
echo "PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF} NCCL_ASYNC_ERROR_HANDLING=${NCCL_ASYNC_ERROR_HANDLING} NCCL_IB_DISABLE=${NCCL_IB_DISABLE} NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE} CUDA_VISIBLE_DEVICES=${CUDA_DEVICES} ${cmd[*]}"

if [[ "${DRY_RUN:-0}" == "1" ]]; then
  echo "DRY_RUN=1 set; exiting before training launch."
  exit 0
fi

PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF}" \
NCCL_ASYNC_ERROR_HANDLING="${NCCL_ASYNC_ERROR_HANDLING}" \
NCCL_IB_DISABLE="${NCCL_IB_DISABLE}" \
NCCL_P2P_DISABLE="${NCCL_P2P_DISABLE}" \
CUDA_VISIBLE_DEVICES="${CUDA_DEVICES}" \
"${cmd[@]}" 2>&1 | tee "${log_file}"