Skip to content

Commit 4581410

Browse files
aparnabgaparnabg
authored
Merge pull request #13 from sensein/vjepa
Vjepa
2 parents 00aeb7c + 089a33b commit 4581410

54 files changed

Lines changed: 7921 additions & 0 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ Included model families:
9191
- `pyskl/` - PySKL config/logit utilities.
9292
- `vlm_models/` - Qwen/Ovis vision-language model classifiers.
9393
- `feature_extraction/` - I3D/R(2+1)D and V-JEPA feature extraction.
94+
- `vjepa/` - vjepa model variants.
9495
- `dlc_action/` - DLC2Action data preparation workflow.
9596

9697
### Fusion Models

‎jobs/action_model_testing/vjepa/clips_fixed_length/submit_all_jobs.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
# Usage: bash submit_all.sh
3+
4+
SCRIPT_DIR="/home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjepa_crop"
5+
6+
echo "Submitting all 6 jobs (all reuse existing feature cache)..."
7+
for LABEL in loco rmm; do
8+
for SEED in 42 123 456; do
9+
JID=$(sbatch --parsable "$SCRIPT_DIR/vjepa_clip_level_ablation.sh" $LABEL $SEED)
10+
echo " $LABEL seed $SEED -> job $JID"
11+
done
12+
done
13+
echo ""
14+
echo "All 6 jobs submitted. Check with: squeue -u $USER"

‎jobs/action_model_testing/vjepa/clips_fixed_length/vjepa_clip_level_ablation.sh‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/bin/bash
2+
#SBATCH --job-name=vjepa2_ablation
3+
#SBATCH --partition=mit_normal_gpu
4+
#SBATCH --cpus-per-task=4
5+
#SBATCH --mem=120GB
6+
#SBATCH --gres=gpu:h200:1
7+
#SBATCH --time=04:00:00
8+
#SBATCH --output=/home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjepa_crop/logs/%x_%A.out
9+
#SBATCH --error=/home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjepa_crop/logs/%x_%A.err
10+
11+
mkdir -p /home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjepa_crop/logs
12+
13+
module load miniforge/24.3.0-0
14+
module load cuda
15+
module load cudnn
16+
source /home/aparnabg/orcd/scratch/miniconda3/etc/profile.d/conda.sh
17+
conda activate vjepa2-312
18+
cd /home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjepa_crop
19+
20+
LABEL=$1
21+
SEED=$2
22+
23+
if [ -z "$LABEL" ] || [ -z "$SEED" ]; then
24+
echo "Error: Usage: sbatch run.sh <label> <seed>"
25+
exit 1
26+
fi
27+
28+
echo "============================================"
29+
echo " Label : $LABEL"
30+
echo " Seed : $SEED"
31+
echo "============================================"
32+
33+
START_TIME=$(date +%s)
34+
echo "Start time: $(date)"
35+
36+
# Where old extracted features already live (from original run)
37+
OLD_CACHE_BASE="/orcd/data/satra/002/projects/SAILS/vjepa_features/action_model_outputs/clips_h5/vjepa_new_crop/clip_level_ablation"
38+
39+
# Where new seed outputs will go
40+
NEW_BASE="/orcd/data/satra/002/projects/SAILS/vjepa_features/models_output_seeds/clips_h5/vjepa_new_crop/clip_level_ablation"
41+
42+
CACHE_SRC="${OLD_CACHE_BASE}/${LABEL}/extracted_features.pt"
43+
SEED_OUT_DIR="${NEW_BASE}/${LABEL}/seed_${SEED}"
44+
CACHE_DST="${SEED_OUT_DIR}/extracted_features.pt"
45+
46+
# Verify old cache exists
47+
if [ ! -f "$CACHE_SRC" ]; then
48+
echo "ERROR: old feature cache not found at:"
49+
echo " $CACHE_SRC"
50+
exit 1
51+
fi
52+
53+
# Create output dir and symlink the cache
54+
mkdir -p "$SEED_OUT_DIR"
55+
ln -sf "$CACHE_SRC" "$CACHE_DST"
56+
echo "Symlinked cache: $CACHE_SRC -> $CACHE_DST"
57+
58+
echo "Running seed $SEED with --skip_extraction"
59+
python vjepa_clip_level_ablation.py \
60+
--label $LABEL \
61+
--head all \
62+
--seed $SEED \
63+
--skip_extraction
64+
65+
END_TIME=$(date +%s)
66+
DURATION=$((END_TIME - START_TIME))
67+
echo "End time: $(date)"
68+
echo "Total runtime: ${DURATION} seconds ($(($DURATION / 60)) minutes)"

‎jobs/action_model_testing/vjepa/clips_without_coi_crop/job_ab.sh‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/bash
2+
#SBATCH --job-name=vjepa_ablation_NWE
3+
#SBATCH --partition=pi_satra
4+
#SBATCH --cpus-per-task=8
5+
#SBATCH --mem=400G
6+
#SBATCH --gres=gpu:1
7+
#SBATCH --time=06:00:00
8+
#SBATCH --array=0-14
9+
#SBATCH --output=/home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjeap_full_video/logs/ablation_%A_%a.out
10+
#SBATCH --error=/home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjeap_full_video/logs/ablation_%A_%a.err
11+
12+
export HF_HOME=/home/aparnabg/.cache/huggingface
13+
export PYTHONUNBUFFERED=1
14+
15+
module load miniforge/24.3.0-0
16+
module load cudnn
17+
module load cuda
18+
conda deactivate
19+
source /home/aparnabg/orcd/scratch/miniconda3/etc/profile.d/conda.sh
20+
conda activate vjepa2-312
21+
22+
# -------------------------------------------------------
23+
# Map array index (0-14) -> (seed, head)
24+
# 0 = seed 42, linear 5 = seed 456, linear
25+
# 1 = seed 42, mlp_small 6 = seed 456, mlp_small
26+
# 2 = seed 42, mlp_large 7 = seed 456, mlp_large
27+
# 3 = seed 42, attentive 8 = seed 456, attentive
28+
# 4 = seed 42, transformer 9 = seed 456, transformer
29+
# 10 = seed 123, linear
30+
# 11 = seed 123, mlp_small
31+
# 12 = seed 123, mlp_large
32+
# 13 = seed 123, attentive
33+
# 14 = seed 123, transformer
34+
# -------------------------------------------------------
35+
36+
SEEDS=(42 456 123)
37+
HEADS=(linear mlp_small mlp_large attentive transformer)
38+
39+
SEED_IDX=$(( SLURM_ARRAY_TASK_ID / 5 ))
40+
HEAD_IDX=$(( SLURM_ARRAY_TASK_ID % 5 ))
41+
42+
SEED=${SEEDS[$SEED_IDX]}
43+
HEAD=${HEADS[$HEAD_IDX]}
44+
45+
echo "=============================================="
46+
echo "Job array id : ${SLURM_ARRAY_TASK_ID}"
47+
echo "Seed : ${SEED}"
48+
echo "Head : ${HEAD}"
49+
echo "Start : $(date)"
50+
echo "Node : ${SLURMD_NODENAME}"
51+
echo "GPU : $(nvidia-smi --query-gpu=name --format=csv,noheader)"
52+
echo "=============================================="
53+
54+
python /home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjeap_full_video/train_probe_ablation.py \
55+
--seed ${SEED} \
56+
--head ${HEAD}
57+
58+
echo "Done seed=${SEED} head=${HEAD}: $(date)"

‎jobs/action_model_testing/vjepa/clips_without_coi_crop/launch_all.sh‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
# ============================================================
3+
# Master launcher
4+
# Usage: bash launch_all.sh
5+
#
6+
# What it does:
7+
# 1. Submits extract job
8+
# 2. Submits seed array job with --dependency=afterok:<extract_job_id>
9+
# so seeds only start after features are successfully extracted
10+
# ============================================================
11+
12+
LOG_DIR="/orcd/data/satra/002/projects/SAILS/vjepa_features/models_output_seeds/clips/vjepa/logs"
13+
mkdir -p $LOG_DIR
14+
15+
CODE_DIR="/home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjeap_full_video"
16+
17+
# --- Step 1: Submit feature extraction ---
18+
EXTRACT_JOB=$(sbatch --parsable ${CODE_DIR}/submit_extract.sh)
19+
echo "Submitted feature extraction job: $EXTRACT_JOB"
20+
21+
# --- Step 2: Submit seed training jobs (depend on extraction finishing) ---
22+
SEED_JOB=$(sbatch --parsable \
23+
--dependency=afterok:${EXTRACT_JOB} \
24+
${CODE_DIR}/submit_seeds.sh)
25+
echo "Submitted seed training jobs (array): $SEED_JOB"
26+
echo " -> Seeds 42, 456, 123 will start after job $EXTRACT_JOB completes"
27+
28+
echo ""
29+
echo "Monitor with:"
30+
echo " squeue -u \$USER"
31+
echo " tail -f ${LOG_DIR}/extract_${EXTRACT_JOB}.out"
32+
echo " tail -f ${LOG_DIR}/probe_${SEED_JOB}_0.out # seed 42"
33+
echo " tail -f ${LOG_DIR}/probe_${SEED_JOB}_1.out # seed 456"
34+
echo " tail -f ${LOG_DIR}/probe_${SEED_JOB}_2.out # seed 123"

‎jobs/action_model_testing/vjepa/clips_without_coi_crop/rmm/job_ab.sh‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/bash
2+
#SBATCH --job-name=bash
3+
#SBATCH --partition=pi_satra
4+
#SBATCH --cpus-per-task=8
5+
#SBATCH --mem=200G
6+
#SBATCH --gres=gpu:1
7+
#SBATCH --time=06:00:00
8+
#SBATCH --array=0-14
9+
#SBATCH --output=/home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjeap_full_video/rmm/logs/ablation_%A_%a.out
10+
#SBATCH --error=/home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjeap_full_video/rmm/logs/ablation_%A_%a.err
11+
12+
export HF_HOME=/home/aparnabg/.cache/huggingface
13+
export PYTHONUNBUFFERED=1
14+
15+
module load miniforge/24.3.0-0
16+
module load cudnn
17+
module load cuda
18+
conda deactivate
19+
source /home/aparnabg/orcd/scratch/miniconda3/etc/profile.d/conda.sh
20+
conda activate vjepa2-312
21+
22+
# -------------------------------------------------------
23+
# Map array index (0-14) -> (seed, head)
24+
# 0 = seed 42, linear 5 = seed 456, linear
25+
# 1 = seed 42, mlp_small 6 = seed 456, mlp_small
26+
# 2 = seed 42, mlp_large 7 = seed 456, mlp_large
27+
# 3 = seed 42, attentive 8 = seed 456, attentive
28+
# 4 = seed 42, transformer 9 = seed 456, transformer
29+
# 10 = seed 123, linear
30+
# 11 = seed 123, mlp_small
31+
# 12 = seed 123, mlp_large
32+
# 13 = seed 123, attentive
33+
# 14 = seed 123, transformer
34+
# -------------------------------------------------------
35+
36+
SEEDS=(42 456 123)
37+
HEADS=(linear mlp_small mlp_large attentive transformer)
38+
39+
SEED_IDX=$(( SLURM_ARRAY_TASK_ID / 5 ))
40+
HEAD_IDX=$(( SLURM_ARRAY_TASK_ID % 5 ))
41+
42+
SEED=${SEEDS[$SEED_IDX]}
43+
HEAD=${HEADS[$HEAD_IDX]}
44+
45+
echo "=============================================="
46+
echo "Job array id : ${SLURM_ARRAY_TASK_ID}"
47+
echo "Seed : ${SEED}"
48+
echo "Head : ${HEAD}"
49+
echo "Start : $(date)"
50+
echo "Node : ${SLURMD_NODENAME}"
51+
echo "GPU : $(nvidia-smi --query-gpu=name --format=csv,noheader)"
52+
echo "=============================================="
53+
54+
python /home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjeap_full_video/rmm/train_probe_ablation.py \
55+
--seed ${SEED} \
56+
--head ${HEAD}
57+
58+
echo "Done seed=${SEED} head=${HEAD}: $(date)"

‎jobs/action_model_testing/vjepa/clips_without_coi_crop/rmm/launch_all.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
# ============================================================
3+
# RMM Master Launcher
4+
# Usage: bash launch_all.sh
5+
#
6+
# 1. Submits RMM feature extraction job
7+
# 2. Submits 3 seed probe jobs (array) with dependency on
8+
# extraction finishing successfully (afterok)
9+
# ============================================================
10+
11+
LOG_DIR="/orcd/data/satra/002/projects/SAILS/vjepa_features/models_output_seeds/clips/rmm/logs"
12+
mkdir -p $LOG_DIR
13+
14+
CODE_DIR="/home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjeap_full_video/rmm"
15+
16+
# --- Step 1: Submit feature extraction ---
17+
EXTRACT_JOB=$(sbatch --parsable ${CODE_DIR}/submit_extract.sh)
18+
echo "Submitted RMM feature extraction job: $EXTRACT_JOB"
19+
20+
# --- Step 2: Submit seed training jobs (depend on extraction finishing) ---
21+
SEED_JOB=$(sbatch --parsable \
22+
--dependency=afterok:${EXTRACT_JOB} \
23+
${CODE_DIR}/submit_seeds.sh)
24+
echo "Submitted RMM seed training jobs (array): $SEED_JOB"
25+
echo " -> Seeds 42, 456, 123 will start after job $EXTRACT_JOB completes"
26+
27+
echo ""
28+
echo "Monitor with:"
29+
echo " squeue -u \$USER"
30+
echo " tail -f ${LOG_DIR}/extract_${EXTRACT_JOB}.out"
31+
echo " tail -f ${LOG_DIR}/probe_${SEED_JOB}_0.out # seed 42"
32+
echo " tail -f ${LOG_DIR}/probe_${SEED_JOB}_1.out # seed 456"
33+
echo " tail -f ${LOG_DIR}/probe_${SEED_JOB}_2.out # seed 123"

‎jobs/action_model_testing/vjepa/clips_without_coi_crop/rmm/submit_extract.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
#SBATCH --job-name=rmm_extract
3+
#SBATCH --partition=pi_satra
4+
#SBATCH --cpus-per-task=8
5+
#SBATCH --mem=400G
6+
#SBATCH --gres=gpu:h100:1
7+
#SBATCH --time=10:00:00
8+
#SBATCH --output=/orcd/data/satra/002/projects/SAILS/vjepa_features/models_output_seeds/clips/rmm/logs/extract_%j.out
9+
#SBATCH --error=/orcd/data/satra/002/projects/SAILS/vjepa_features/models_output_seeds/clips/rmm/logs/extract_%j.err
10+
11+
export HF_HOME=/home/aparnabg/.cache/huggingface
12+
export PYTHONUNBUFFERED=1
13+
14+
module load miniforge/24.3.0-0
15+
module load cudnn
16+
module load cuda
17+
conda deactivate
18+
source /home/aparnabg/orcd/scratch/miniconda3/etc/profile.d/conda.sh
19+
conda activate vjepa2-312
20+
21+
mkdir -p /orcd/data/satra/002/projects/SAILS/vjepa_features/models_output_seeds/clips/rmm/logs
22+
23+
echo "Starting RMM feature extraction: $(date)"
24+
echo "Node: $SLURMD_NODENAME"
25+
echo "GPU: $(nvidia-smi --query-gpu=name --format=csv,noheader)"
26+
27+
python /home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjeap_full_video/rmm/extract_features.py
28+
29+
echo "RMM feature extraction done: $(date)"

‎jobs/action_model_testing/vjepa/clips_without_coi_crop/rmm/submit_seeds.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
#SBATCH --job-name=bash
3+
#SBATCH --partition=pi_satra
4+
#SBATCH --cpus-per-task=8
5+
#SBATCH --mem=400G
6+
#SBATCH --gres=gpu:1
7+
#SBATCH --time=06:00:00
8+
#SBATCH --array=0-2
9+
#SBATCH --output=/home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjeap_full_video/rmm/logs/probe_%A_%a.out
10+
#SBATCH --error=/home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjeap_full_video/rmm/logs/probe_%A_%a.err
11+
12+
export HF_HOME=/home/aparnabg/.cache/huggingface
13+
export PYTHONUNBUFFERED=1
14+
15+
module load miniforge/24.3.0-0
16+
module load cudnn
17+
module load cuda
18+
conda deactivate
19+
source /home/aparnabg/orcd/scratch/miniconda3/etc/profile.d/conda.sh
20+
conda activate vjepa2-312
21+
22+
# Map array index -> seed
23+
SEEDS=(42 456 123)
24+
SEED=${SEEDS[$SLURM_ARRAY_TASK_ID]}
25+
26+
echo "Starting RMM probe training seed=${SEED} array_id=${SLURM_ARRAY_TASK_ID}: $(date)"
27+
echo "Node: $SLURMD_NODENAME"
28+
echo "GPU: $(nvidia-smi --query-gpu=name --format=csv,noheader)"
29+
30+
python /home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjeap_full_video/rmm/train_probe.py \
31+
--seed ${SEED}
32+
33+
echo "RMM probe training done seed=${SEED}: $(date)"

‎jobs/action_model_testing/vjepa/clips_without_coi_crop/submit_extract.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
#SBATCH --job-name=vjepa_extract
3+
#SBATCH --partition=pi_satra
4+
#SBATCH --cpus-per-task=8
5+
#SBATCH --mem=400G
6+
#SBATCH --gres=gpu:h100:1
7+
#SBATCH --time=10:00:00
8+
#SBATCH --output=/orcd/data/satra/002/projects/SAILS/vjepa_features/models_output_seeds/clips/vjepa/logs/extract_%j.out
9+
#SBATCH --error=/orcd/data/satra/002/projects/SAILS/vjepa_features/models_output_seeds/clips/vjepa/logs/extract_%j.err
10+
11+
export HF_HOME=/home/aparnabg/.cache/huggingface
12+
export PYTHONUNBUFFERED=1
13+
14+
module load miniforge/24.3.0-0
15+
module load cudnn
16+
module load cuda
17+
conda deactivate
18+
source /home/aparnabg/orcd/scratch/miniconda3/etc/profile.d/conda.sh
19+
conda activate vjepa2-312
20+
21+
# Make log dir just in case
22+
mkdir -p /orcd/data/satra/002/projects/SAILS/vjepa_features/models_output_seeds/clips/vjepa/logs
23+
24+
echo "Starting feature extraction: $(date)"
25+
echo "Node: $SLURMD_NODENAME"
26+
echo "GPU: $(nvidia-smi --query-gpu=name --format=csv,noheader)"
27+
28+
python /home/aparnabg/orcd/scratch/all_project_files/action_sota_models/vjeap_full_video/extract_features.py
29+
30+
echo "Feature extraction done: $(date)"

0 commit comments

Comments
 (0)