hparams_step_2.yaml

att_fn: xmea
aug_flip: false
batch_size: 400
d_heads: 48
ffn_bias: false
ffn_multi: 4
fold: 0
grad_accum_sched: {}
grad_clip: 5
layer_bpp: !!python/tuple
- 1
- 1
- 1
- 1
- 1
- 1
- 0
- 0
- 0
- 0
- 0
- 0
layer_gru: !!python/tuple
- 1
lr: 0.003
lr_scale: 5
lr_warmup: 0.005
n_epochs: 200
n_folds: 5
n_heads: 6
n_layers: 12
noise_bpp: 0.025
noise_react: 0
norm_rms: true
note: PL
p_dropout: 0.1
pos_bias_params: !!python/tuple
- 32
- 128
pretrained: false
qkv_bias: false
seed: 420
sn_bias_sched: {}
tta_flip: false
val_flip: false
wt_decay: 0.1