Skip to content

Commit 9e686cb

Browse files
clane9clane9
committed
Add job files
1 parent f3994fb commit 9e686cb

11 files changed

Lines changed: 234 additions & 0 deletions

‎jobs/attn_comp_01/configs/05_untied_all_linmix.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
model: vision_transformer_tiny_patch16_128
2+
untied: "1,1,1"
3+
attn_bias: true
4+
attn_head_bias: true
5+
attn_mode: linmixing
6+
skip_attn: false
7+
lr: 1.0e-04
8+
desc: >
9+
all untied weights with attn (head) bias and linear mixing attention (w/o softmax) and
10+
no attention skip connection
11+
12+
dataset: imagenet-100
13+
crop_min_scale: 0.33334
14+
hflip: 0.5
15+
color_jitter: 0.4
16+
epochs: 50
17+
amp: true
18+
wandb: true
19+
figure_interval: 5

‎jobs/attn_comp_01/configs/06_untied_all_linmix_dw.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
model: vision_transformer_tiny_patch16_128
2+
untied: "1,1,1"
3+
attn_bias: true
4+
attn_head_bias: true
5+
attn_mode: linmixing
6+
skip_attn: false
7+
num_heads: 384
8+
lr: 1.0e-04
9+
desc: >
10+
all untied weights with attn (head) bias and linear mixing attention (w/o softmax) and
11+
no attention skip connection and depthwise mixing (num_heads = dim)
12+
13+
dataset: imagenet-100
14+
crop_min_scale: 0.33334
15+
hflip: 0.5
16+
color_jitter: 0.4
17+
epochs: 50
18+
amp: true
19+
wandb: true
20+
figure_interval: 5

‎jobs/moe_01/base_config.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
model: vision_transformer_tiny_patch16_128
2+
mlp_mode: "moe"
3+
mlp_ratio: 4.0
4+
desc: "ViT with varying number of MLP experts"
5+
6+
dataset: imagenet-100
7+
crop_min_scale: 0.33334
8+
hflip: 0.5
9+
color_jitter: 0.4
10+
epochs: 50
11+
amp: true
12+
wandb: true
13+
figure_interval: 5

‎jobs/moe_01/batch_array.sh‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/bin/bash
2+
3+
#SBATCH --job-name=moe_01
4+
#SBATCH --partition=GPU-shared
5+
#SBATCH -N 1
6+
#SBATCH --ntasks=5
7+
#SBATCH --gpus=v100-32:1
8+
#SBATCH --time=03:00:00
9+
# #SBATCH --array=0-5
10+
# #SBATCH --array=6-7
11+
# #SBATCH --array=8-9
12+
# #SBATCH --array=10-11
13+
#SBATCH --array=12-13
14+
#SBATCH --account=med230001p
15+
16+
# Set some environment variables
17+
ROOT="/ocean/projects/med230001p/clane2/code/columnformers"
18+
cd $ROOT
19+
20+
# Set up python environment
21+
source .venv/bin/activate
22+
23+
# Setup wandb
24+
source .env
25+
wandb login
26+
27+
JOB="moe_01"
28+
CONFIG="jobs/${JOB}/base_config.yaml"
29+
OPTS=(
30+
"--moe_experts 1"
31+
"--moe_experts 2"
32+
"--moe_experts 4"
33+
"--moe_experts 1,1,1,2,2,2"
34+
"--moe_experts 1,1,1,4,4,4"
35+
"--moe_experts 1,1,2,2,4,4"
36+
"--moe_experts 2,2,2,1,1,1"
37+
"--moe_experts 4,4,4,1,1,1"
38+
"--moe_experts 1 --mlp_ratio 4,4,4,2,2,2"
39+
"--moe_experts 1 --mlp_ratio 4,4,4,1,1,1"
40+
"--moe_experts 1 --mlp_ratio 2"
41+
"--moe_experts 1 --mlp_ratio 1"
42+
"--moe_experts 1,1,1,2,2,2 --no_moe_conserve"
43+
"--moe_experts 1,1,1,4,4,4 --no_moe_conserve"
44+
)
45+
NAMES=(
46+
"01_E-1"
47+
"02_E-2"
48+
"03_E-4"
49+
"04_E-1-2"
50+
"05_E-1-4"
51+
"06_E-1-2-4"
52+
"07_E-2-1"
53+
"08_E-4-1"
54+
"09_E-1_R-4-2"
55+
"10_E-1_R-4-1"
56+
"11_E-1_R-2"
57+
"12_E-1_R-1"
58+
"13_E-1-2_NC"
59+
"14_E-1-4_NC"
60+
)
61+
62+
OPT="${OPTS[SLURM_ARRAY_TASK_ID]}"
63+
NAME="${NAMES[SLURM_ARRAY_TASK_ID]}"
64+
FULL_NAME="${JOB}/${NAME}"
65+
66+
python columnformers/train.py $CONFIG --name $FULL_NAME $OPT

‎jobs/sumoe_01/batch_array.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/bash
2+
3+
#SBATCH --job-name=sumoe_01
4+
#SBATCH --partition=GPU-shared
5+
#SBATCH -N 1
6+
#SBATCH --ntasks=5
7+
#SBATCH --gpus=v100-32:1
8+
#SBATCH --time=04:00:00
9+
# #SBATCH --array=0
10+
#SBATCH --array=1-5
11+
#SBATCH --account=med230001p
12+
13+
# Set some environment variables
14+
ROOT="/ocean/projects/med230001p/clane2/code/columnformers"
15+
cd $ROOT
16+
17+
# Set up python environment
18+
source .venv/bin/activate
19+
20+
# Setup wandb
21+
source .env
22+
wandb login
23+
24+
JOB="sumoe_01"
25+
NAMES=(
26+
"01_baseline"
27+
"02_ratio-2"
28+
"03_ratio-1"
29+
"04_ratio-2_rank-2"
30+
"05_ratio-1_rank-2"
31+
"06_ratio-1_rank-4"
32+
)
33+
34+
NAME="${NAMES[SLURM_ARRAY_TASK_ID]}"
35+
CONFIG="jobs/${JOB}/configs/${NAME}.yaml"
36+
FULL_NAME="${JOB}/${NAME}"
37+
38+
python columnformers/train.py $CONFIG --name $FULL_NAME

‎jobs/sumoe_01/configs/01_baseline.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
model: vision_transformer_tiny_patch16_128
2+
mlp_ratio: 4.0
3+
mlp_rank: null
4+
desc: "baseline ViT"
5+
6+
dataset: imagenet-100
7+
crop_min_scale: 0.33334
8+
hflip: 0.5
9+
color_jitter: 0.4
10+
epochs: 50
11+
amp: true
12+
wandb: true
13+
figure_interval: 5

‎jobs/sumoe_01/configs/02_ratio-2.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
model: vision_transformer_tiny_patch16_128
2+
mlp_ratio: 2.0
3+
mlp_rank: null
4+
desc: "ViT MLP ratio 2"
5+
6+
dataset: imagenet-100
7+
crop_min_scale: 0.33334
8+
hflip: 0.5
9+
color_jitter: 0.4
10+
epochs: 50
11+
amp: true
12+
wandb: true
13+
figure_interval: 5

‎jobs/sumoe_01/configs/03_ratio-1.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
model: vision_transformer_tiny_patch16_128
2+
mlp_ratio: 1.0
3+
mlp_rank: null
4+
desc: "ViT MLP ratio 1"
5+
6+
dataset: imagenet-100
7+
crop_min_scale: 0.33334
8+
hflip: 0.5
9+
color_jitter: 0.4
10+
epochs: 50
11+
amp: true
12+
wandb: true
13+
figure_interval: 5

‎jobs/sumoe_01/configs/04_ratio-2_rank-2.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
model: vision_transformer_tiny_patch16_128
2+
mlp_ratio: 2.0
3+
mlp_rank: "2"
4+
desc: "ViT MLP ratio 2, SuMoE rank 2"
5+
6+
dataset: imagenet-100
7+
crop_min_scale: 0.33334
8+
hflip: 0.5
9+
color_jitter: 0.4
10+
epochs: 50
11+
amp: true
12+
wandb: true
13+
figure_interval: 5

‎jobs/sumoe_01/configs/05_ratio-1_rank-2.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
model: vision_transformer_tiny_patch16_128
2+
mlp_ratio: 1.0
3+
mlp_rank: "2"
4+
desc: "ViT MLP ratio 1, SuMoE rank 2"
5+
6+
dataset: imagenet-100
7+
crop_min_scale: 0.33334
8+
hflip: 0.5
9+
color_jitter: 0.4
10+
epochs: 50
11+
amp: true
12+
wandb: true
13+
figure_interval: 5

0 commit comments

Comments
 (0)