Skip to content

Commit a0177a3

Browse files
aparnabgaparnabg
committed
add pyskl
1 parent 0e3a8c5 commit a0177a3

5 files changed

Lines changed: 1097 additions & 0 deletions

File tree

‎jobs/action_model_testing/pyskl/test_job.sh‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
#!/bin/bash
2+
WORKSPACE=/home/aparnabg/orcd/pool/pyskl_workspace
3+
PYSKL_ROOT=${WORKSPACE}/pyskl
4+
LOG_DIR=${WORKSPACE}/train_logs
5+
ENV_PATH=${WORKSPACE}/envs/pyskl
6+
mkdir -p ${LOG_DIR}
7+
8+
submit_test() {
9+
local JOB_NAME=$1
10+
local CONFIG=$2
11+
local WORK_DIR=$3
12+
local SEED=$4
13+
14+
# Find best checkpoint
15+
CKPT=$(ls ${PYSKL_ROOT}/${WORK_DIR}/best_top1_acc_epoch_*.pth 2>/dev/null | head -1)
16+
if [ -z "$CKPT" ]; then
17+
echo "SKIPPING ${JOB_NAME}_s${SEED} — no checkpoint found in ${WORK_DIR}"
18+
return
19+
fi
20+
21+
echo "Submitting test: ${JOB_NAME}_s${SEED}, ckpt: $(basename $CKPT)"
22+
23+
sbatch << SBATCH
24+
#!/bin/bash
25+
#SBATCH --job-name=test_${JOB_NAME}_s${SEED}
26+
#SBATCH --partition=mit_normal_gpu
27+
#SBATCH --gres=gpu:l40s:1
28+
#SBATCH --cpus-per-task=8
29+
#SBATCH --mem=64G
30+
#SBATCH --time=01:30:00
31+
#SBATCH --output=${LOG_DIR}/test_${JOB_NAME}_s${SEED}_%j.out
32+
#SBATCH --error=${LOG_DIR}/test_${JOB_NAME}_s${SEED}_%j.err
33+
34+
module purge
35+
export PATH=/usr/bin:/bin:${ENV_PATH}/bin:\$PATH
36+
export CONDA_PREFIX=${ENV_PATH}
37+
export LD_PRELOAD=${ENV_PATH}/lib/libstdc++.so.6
38+
module load cuda/12.9.1
39+
40+
cd ${PYSKL_ROOT}
41+
42+
# Patch config to use test split
43+
TEST_CONFIG=/tmp/test_${JOB_NAME}_s${SEED}.py
44+
cp ${CONFIG} \${TEST_CONFIG}
45+
python - << PYEOF
46+
with open('\${TEST_CONFIG}', 'r') as f:
47+
content = f.read()
48+
# Change work_dir to seed-specific dir
49+
import re
50+
content = re.sub(r"work_dir = '(.*?)'", r"work_dir = '${WORK_DIR}'", content)
51+
with open('\${TEST_CONFIG}', 'w') as f:
52+
f.write(content)
53+
PYEOF
54+
55+
echo "Testing: ${JOB_NAME}_s${SEED}"
56+
echo "Config : \${TEST_CONFIG}"
57+
echo "Ckpt : ${CKPT}"
58+
59+
bash tools/dist_test.sh \${TEST_CONFIG} ${CKPT} 1 \
60+
--out ${PYSKL_ROOT}/${WORK_DIR}/test_pred.pkl \
61+
--eval top_k_accuracy mean_class_accuracy \
62+
2>&1 | tee ${PYSKL_ROOT}/${WORK_DIR}/test_results.txt
63+
64+
echo "Done testing ${JOB_NAME}_s${SEED}"
65+
SBATCH
66+
}
67+
68+
# ================================================================
69+
# All models x 3 seeds
70+
# ================================================================
71+
for SEED in 123 42 456; do
72+
# RMM
73+
submit_test "stgcnpp_rmm_j" "configs/custom/stgcnpp_rmm/j.py" "work_dirs/stgcnpp_rmm/j_s${SEED}" ${SEED}
74+
submit_test "stgcnpp_rmm_b" "configs/custom/stgcnpp_rmm/b.py" "work_dirs/stgcnpp_rmm/b_s${SEED}" ${SEED}
75+
submit_test "stgcnpp_rmm_jm" "configs/custom/stgcnpp_rmm/jm.py" "work_dirs/stgcnpp_rmm/jm_s${SEED}" ${SEED}
76+
submit_test "stgcnpp_rmm_bm" "configs/custom/stgcnpp_rmm/bm.py" "work_dirs/stgcnpp_rmm/bm_s${SEED}" ${SEED}
77+
submit_test "ctrgcn_rmm_j" "configs/custom/ctrgcn_rmm/j.py" "work_dirs/ctrgcn_rmm/j_s${SEED}" ${SEED}
78+
submit_test "ctrgcn_rmm_b" "configs/custom/ctrgcn_rmm/b.py" "work_dirs/ctrgcn_rmm/b_s${SEED}" ${SEED}
79+
submit_test "ctrgcn_rmm_jm" "configs/custom/ctrgcn_rmm/jm.py" "work_dirs/ctrgcn_rmm/jm_s${SEED}" ${SEED}
80+
submit_test "ctrgcn_rmm_bm" "configs/custom/ctrgcn_rmm/bm.py" "work_dirs/ctrgcn_rmm/bm_s${SEED}" ${SEED}
81+
submit_test "posec3d_rmm" "configs/custom/posec3d_rmm/joint.py" "work_dirs/posec3d_rmm/joint_s${SEED}" ${SEED}
82+
83+
# LOCO
84+
submit_test "stgcnpp_loco_j" "configs/custom/stgcnpp_loco/j.py" "work_dirs/stgcnpp_loco/j_s${SEED}" ${SEED}
85+
submit_test "stgcnpp_loco_b" "configs/custom/stgcnpp_loco/b.py" "work_dirs/stgcnpp_loco/b_s${SEED}" ${SEED}
86+
submit_test "stgcnpp_loco_jm" "configs/custom/stgcnpp_loco/jm.py" "work_dirs/stgcnpp_loco/jm_s${SEED}" ${SEED}
87+
submit_test "stgcnpp_loco_bm" "configs/custom/stgcnpp_loco/bm.py" "work_dirs/stgcnpp_loco/bm_s${SEED}" ${SEED}
88+
submit_test "ctrgcn_loco_j" "configs/custom/ctrgcn_loco/j.py" "work_dirs/ctrgcn_loco/j_s${SEED}" ${SEED}
89+
submit_test "ctrgcn_loco_b" "configs/custom/ctrgcn_loco/b.py" "work_dirs/ctrgcn_loco/b_s${SEED}" ${SEED}
90+
submit_test "ctrgcn_loco_jm" "configs/custom/ctrgcn_loco/jm.py" "work_dirs/ctrgcn_loco/jm_s${SEED}" ${SEED}
91+
submit_test "ctrgcn_loco_bm" "configs/custom/ctrgcn_loco/bm.py" "work_dirs/ctrgcn_loco/bm_s${SEED}" ${SEED}
92+
submit_test "posec3d_loco" "configs/custom/posec3d_loco/joint.py" "work_dirs/posec3d_loco/joint_s${SEED}" ${SEED}
93+
done
94+
95+
echo ""
96+
echo "All test jobs submitted. Monitor: squeue -u \$USER"

0 commit comments

Comments
 (0)