|
13 | 13 | # |
14 | 14 | # Full run, 36 GPUs on 9 hosts: |
15 | 15 | # sbatch --account=YOUR_ACCOUNT --partition=beta --qos=standard \ |
16 | | -# --requeue --signal=B:SIGTERM@900 \ |
17 | 16 | # --nodes=9 --gres=gpu:4 --time=12:00:00 \ |
18 | 17 | # scripts/empireai_nvl72_train.sbatch |
19 | 18 | # |
@@ -81,18 +80,6 @@ if [[ -z "${WANDB_MODE:-}" ]]; then |
81 | 80 | fi |
82 | 81 | fi |
83 | 82 |
|
84 | | -if [[ -n "${GHCR_USERNAME:-}" && -n "${GHCR_TOKEN:-}" ]]; then |
85 | | - export ENROOT_CONFIG_PATH="${ENROOT_CONFIG_PATH:-${HOME}/.config/enroot}" |
86 | | - mkdir -p "${ENROOT_CONFIG_PATH}" |
87 | | - chmod 700 "${ENROOT_CONFIG_PATH}" |
88 | | - credentials_file="${ENROOT_CONFIG_PATH}/.credentials" |
89 | | - if [[ ! -f "${credentials_file}" ]] || ! grep -q '^machine ghcr.io ' "${credentials_file}"; then |
90 | | - umask 077 |
91 | | - printf 'machine ghcr.io login %s password %s\n' \ |
92 | | - "${GHCR_USERNAME}" "${GHCR_TOKEN}" >> "${credentials_file}" |
93 | | - fi |
94 | | -fi |
95 | | - |
96 | 83 | gpus_on_node="${GPUS_PER_NODE:-${SLURM_GPUS_ON_NODE:-}}" |
97 | 84 | if [[ -z "${gpus_on_node}" ]]; then |
98 | 85 | gpus_on_node="$(scontrol show job "${SLURM_JOB_ID}" | sed -n 's/.*gres:gpu:\\([0-9][0-9]*\\).*/\\1/p' | head -n 1)" |
@@ -128,11 +115,11 @@ export MELLANOX_VISIBLE_DEVICES="${MELLANOX_VISIBLE_DEVICES:-all}" |
128 | 115 | export TORCH_NCCL_ASYNC_ERROR_HANDLING="${TORCH_NCCL_ASYNC_ERROR_HANDLING:-1}" |
129 | 116 |
|
130 | 117 | if [[ -z "${NAME:-}" ]]; then |
131 | | - run_kind="full" |
| 118 | + run_kind="" |
132 | 119 | if [[ "${SMOKE}" == "1" ]]; then |
133 | | - run_kind="smoke" |
| 120 | + run_kind="-smoke" |
134 | 121 | fi |
135 | | - NAME="$(date +%Y-%m-%d-%H%M%S)-empireai-halfdeg-${run_kind}" |
| 122 | + NAME="$(date +%Y-%m-%d-%H%M%S)-empireai${run_kind}" |
136 | 123 | fi |
137 | 124 | RUN_DIR="${OUTPUT_BASE}/${NAME}" |
138 | 125 | if [[ -e "${RUN_DIR}" ]]; then |
@@ -172,8 +159,6 @@ if [[ "${SMOKE}" == "1" ]]; then |
172 | 159 | "--val_time.start=2013-10-05" |
173 | 160 | "--val_time.end=2013-11-05" |
174 | 161 | ) |
175 | | -else |
176 | | - train_args+=("--preemptible=true") |
177 | 162 | fi |
178 | 163 |
|
179 | 164 | if [[ -n "${EXTRA_ARGS}" ]]; then |
|
0 commit comments