fix: always produce adapter.tar.gz placeholder so transfer doesn't hold the job

Author: Devesh-Maheshwari

chtc/train_sft.sh

@@ -59,18 +59,27 @@ if [ ! -f "${DATASET_PATH}" ]; then
 fi
 echo "==> dataset: ${DATASET_PATH} ($(wc -l < "${DATASET_PATH}") lines)"
 
+# Placeholder tarball so HTCondor's transfer_output_files never fails
+# (even if training crashes below). Overwritten on success.
+tar -czf "${INITIAL_PWD}/adapter.tar.gz" --files-from /dev/null
+
+# Run training WITHOUT set -e so a crash still reaches the tar step.
+set +e
 echo "==> python scripts/train_sft.py --config ${CONFIG}"
 python scripts/train_sft.py --config "${CONFIG}"
+TRAIN_EXIT=$?
+set -e
+
+if [ "${TRAIN_EXIT}" -ne 0 ]; then
+    echo "ERROR: training exited ${TRAIN_EXIT}. Placeholder tarball will be transferred."
+fi
 
-# Tar the LoRA adapter + tokenizer for transfer back. HTCondor's
-# transfer_output_files expects the file at the job cwd (initial_pwd).
-OUTPUT_DIR=$(python -c "import yaml; c = yaml.safe_load(open('${CONFIG}')); print(c['output_dir'])")
-if [ -d "${OUTPUT_DIR}" ]; then
+# Tar the LoRA adapter + tokenizer. Overwrites the placeholder on success.
+OUTPUT_DIR=$(python -c "import yaml; c = yaml.safe_load(open('${CONFIG}')); print(c['output_dir'])" 2>/dev/null || echo "")
+if [ -n "${OUTPUT_DIR}" ] && [ -d "${OUTPUT_DIR}" ]; then
     tar -czf "${INITIAL_PWD}/adapter.tar.gz" -C "$(dirname "${OUTPUT_DIR}")" "$(basename "${OUTPUT_DIR}")"
     echo "==> wrote adapter tarball: ${INITIAL_PWD}/adapter.tar.gz"
     ls -lh "${INITIAL_PWD}/adapter.tar.gz"
-else
-    echo "WARNING: output_dir ${OUTPUT_DIR} not found — training may have failed silently"
-    # Create an empty tarball so HTCondor's transfer_output_files doesn't error.
-    tar -czf "${INITIAL_PWD}/adapter.tar.gz" --files-from /dev/null
 fi
+
+exit "${TRAIN_EXIT}"