train_BPE_5000.yaml

# ############################################################################
# Model: E2E ASR with attention-based ASR
# Encoder: CRDNN model
# Decoder: GRU + beamsearch + Transformer
# Tokens: BPE with unigram
# losses: CTC+ NLL
# Training: Librispeech 960h
# Authors:  Ju-Chieh Chou, Mirco Ravanelli, Abdel Heba, Peter Plantinga,
# Samuele Cornell, Sung-Lin Yeh, Titouan Parcollet 2021
# ############################################################################

# Seed needs to be set at top of yaml, before objects with parameters
# are instantiated
seed: 1986
__set_seed: !apply:speechbrain.utils.seed_everything [!ref <seed>]
output_folder: !ref results/CRDNN_BPE_960h_5k_LM/<seed>
output_wer_folder: !ref <output_folder>/
save_folder: !ref <output_folder>/save
train_log: !ref <output_folder>/train_log.txt

# Language model (LM) pretraining
# NB: To avoid mismatch, the speech recognizer must be trained with the same
# tokenizer used for LM training. Here, we download everything from the
# speechbrain HuggingFace repository. However, a local path pointing to a
# directory containing the lm.ckpt and tokenizer.ckpt may also be specified
# instead. E.g if you want to use your own LM / tokenizer.
pretrained_lm_tokenizer_path: speechbrain/asr-crdnn-transformerlm-librispeech

# Data files
data_folder: !PLACEHOLDER # e,g./path/to/LibriSpeech

train_splits: ["train-clean-100", "train-clean-360", "train-other-500"]
dev_splits: ["dev-clean"]
test_splits: ["test-clean", "test-other"]
skip_prep: False
ckpt_interval_minutes: 25 # save checkpoint every N min
train_csv: !ref <save_folder>/train.csv
valid_csv: !ref <save_folder>/dev-clean.csv
test_csv:
   - !ref <save_folder>/test-clean.csv
   - !ref <save_folder>/test-other.csv

# Data for augmentation
data_folder_noise: !ref <data_folder>/noise # The noisy sequences for data augmentation will automatically be downloaded here.
NOISE_DATASET_URL: https://www.dropbox.com/scl/fi/a09pj97s5ifan81dqhi4n/noises.zip?rlkey=j8b0n9kdjdr32o1f06t0cw5b7&dl=1
noise_annotation: !ref <save_folder>/noise.csv #The data manifest files are created by the data preparation script

####################### Training Parameters ####################################

number_of_epochs: 25
number_of_ctc_epochs: 25
batch_size: 8
lr: 1.0
ctc_weight: 0.5
sorting: ascending
dynamic_batching: False
precision: fp32 # bf16, fp16 or fp32

# dynamic batching parameters, if used
feats_hop_size: 0.01
max_batch_length: 20000 # in terms of frames
shuffle: True
batch_ordering: random
num_buckets: 20
dynamic_batch_sampler:
   max_batch_length: !ref <max_batch_length>
   shuffle: !ref <shuffle>
   batch_ordering: !ref <batch_ordering>
   num_buckets: !ref <num_buckets>

# Feature parameters
sample_rate: 16000
n_fft: 400
n_mels: 40

opt_class: !name:torch.optim.Adadelta
   lr: !ref <lr>
   rho: 0.95
   eps: 1.e-8

# Dataloader options
num_workers: 4
train_dataloader_opts:
   num_workers: !ref <num_workers>
   batch_size: !ref <batch_size>

valid_dataloader_opts:
   num_workers: !ref <num_workers>
   batch_size: !ref <batch_size>

test_dataloader_opts:
   batch_size: 1

####################### Model Parameters #######################################

activation: !name:torch.nn.LeakyReLU
dropout: 0.15
cnn_blocks: 2
cnn_channels: (64, 128)
inter_layer_pooling_size: (2, 2)
cnn_kernelsize: (3, 3)
time_pooling_size: 4
rnn_class: !name:speechbrain.nnet.RNN.LSTM
rnn_layers: 4
rnn_neurons: 1024
rnn_bidirectional: True
dnn_blocks: 1
dnn_neurons: 1024
emb_size: 1024
dec_neurons: 1024
output_neurons: 5000  # Number of tokens (same as LM)
blank_index: 0
bos_index: 1
eos_index: 2

# Decoding parameters
min_decode_ratio: 0.0
max_decode_ratio: 1.0
valid_beam_size: 20
test_beam_size: 40
using_eos_threshold: True
eos_threshold: 1.5
using_max_attn_shift: True
max_attn_shift: 300
lm_weight: 0.8
temperature: 1.0
ctc_window_size: 200

# Scoring parameters
ctc_weight_decode: 0.40
coverage_penalty: 1.5


epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter
   limit: !ref <number_of_epochs>

normalize: !new:speechbrain.processing.features.InputNormalization
   norm_type: global

compute_features: !new:speechbrain.lobes.features.Fbank
   sample_rate: !ref <sample_rate>
   n_fft: !ref <n_fft>
   n_mels: !ref <n_mels>


enc: !new:speechbrain.lobes.models.CRDNN.CRDNN
   input_shape: [null, null, !ref <n_mels>]
   activation: !ref <activation>
   dropout: !ref <dropout>
   cnn_blocks: !ref <cnn_blocks>
   cnn_channels: !ref <cnn_channels>
   cnn_kernelsize: !ref <cnn_kernelsize>
   inter_layer_pooling_size: !ref <inter_layer_pooling_size>
   time_pooling: True
   using_2d_pooling: False
   time_pooling_size: !ref <time_pooling_size>
   rnn_class: !ref <rnn_class>
   rnn_layers: !ref <rnn_layers>
   rnn_neurons: !ref <rnn_neurons>
   rnn_bidirectional: !ref <rnn_bidirectional>
   rnn_re_init: True
   dnn_blocks: !ref <dnn_blocks>
   dnn_neurons: !ref <dnn_neurons>
   use_rnnp: True

emb: !new:speechbrain.nnet.embedding.Embedding
   num_embeddings: !ref <output_neurons>
   embedding_dim: !ref <emb_size>

dec: !new:speechbrain.nnet.RNN.AttentionalRNNDecoder
   enc_dim: !ref <dnn_neurons>
   input_size: !ref <emb_size>
   rnn_type: gru
   attn_type: location
   hidden_size: !ref <dec_neurons>
   attn_dim: 1024
   num_layers: 1
   scaling: 1.0
   channels: 10
   kernel_size: 100
   re_init: True
   dropout: !ref <dropout>

ctc_lin: !new:speechbrain.nnet.linear.Linear
   input_size: !ref <dnn_neurons>
   n_neurons: !ref <output_neurons>

seq_lin: !new:speechbrain.nnet.linear.Linear
   input_size: !ref <dec_neurons>
   n_neurons: !ref <output_neurons>

log_softmax: !new:speechbrain.nnet.activations.Softmax
   apply_log: True

ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
   blank_index: !ref <blank_index>

seq_cost: !name:speechbrain.nnet.losses.nll_loss
   label_smoothing: 0.1

# This is the TransformerLM that is used according to the Huggingface repository
# Visit the HuggingFace model corresponding to the pretrained_lm_tokenizer_path
# For more details about the model!
# NB: It has to match the pre-trained TransformerLM!!
lm_model: !new:speechbrain.lobes.models.transformer.TransformerLM.TransformerLM
   vocab: 5000
   d_model: 768
   nhead: 12
   num_encoder_layers: 12
   num_decoder_layers: 0
   d_ffn: 3072
   dropout: 0.0
   activation: !name:torch.nn.GELU
   normalize_before: False

tokenizer: !new:sentencepiece.SentencePieceProcessor

modules:
   enc: !ref <enc>
   emb: !ref <emb>
   dec: !ref <dec>
   ctc_lin: !ref <ctc_lin>
   seq_lin: !ref <seq_lin>
   normalize: !ref <normalize>
   lm_model: !ref <lm_model>

model: !new:torch.nn.ModuleList
   - [!ref <enc>, !ref <emb>, !ref <dec>, !ref <ctc_lin>, !ref <seq_lin>]

############################## Decoding & optimiser ############################

ctc_scorer: !new:speechbrain.decoders.scorer.CTCScorer
   eos_index: !ref <eos_index>
   blank_index: !ref <blank_index>
   ctc_fc: !ref <ctc_lin>
   ctc_window_size: !ref <ctc_window_size>

coverage_scorer: !new:speechbrain.decoders.scorer.CoverageScorer
   vocab_size: !ref <output_neurons>

transformerlm_scorer: !new:speechbrain.decoders.scorer.TransformerLMScorer
   language_model: !ref <lm_model>

valid_scorer: !new:speechbrain.decoders.scorer.ScorerBuilder
   full_scorers: [!ref <coverage_scorer>]
   weights:
      coverage: !ref <coverage_penalty>

test_scorer: !new:speechbrain.decoders.scorer.ScorerBuilder
   full_scorers: [
      !ref <transformerlm_scorer>,
      !ref <coverage_scorer>]
   partial_scorers: [!ref <ctc_scorer>]
   weights:
      transformerlm: !ref <lm_weight>
      coverage: !ref <coverage_penalty>
      ctc: !ref <ctc_weight_decode>

# Search
valid_search: !new:speechbrain.decoders.S2SRNNBeamSearcher
   embedding: !ref <emb>
   decoder: !ref <dec>
   linear: !ref <seq_lin>
   bos_index: !ref <bos_index>
   eos_index: !ref <eos_index>
   min_decode_ratio: !ref <min_decode_ratio>
   max_decode_ratio: !ref <max_decode_ratio>
   beam_size: !ref <valid_beam_size>
   eos_threshold: !ref <eos_threshold>
   using_max_attn_shift: !ref <using_max_attn_shift>
   max_attn_shift: !ref <max_attn_shift>
   temperature: !ref <temperature>
   scorer: !ref <valid_scorer>

test_search: !new:speechbrain.decoders.S2SRNNBeamSearcher
   embedding: !ref <emb>
   decoder: !ref <dec>
   linear: !ref <seq_lin>
   bos_index: !ref <bos_index>
   eos_index: !ref <eos_index>
   min_decode_ratio: !ref <min_decode_ratio>
   max_decode_ratio: !ref <max_decode_ratio>
   beam_size: !ref <test_beam_size>
   eos_threshold: !ref <eos_threshold>
   using_max_attn_shift: !ref <using_max_attn_shift>
   max_attn_shift: !ref <max_attn_shift>
   using_eos_threshold: !ref <using_eos_threshold>
   temperature: !ref <temperature>
   scorer: !ref <test_scorer>

lr_annealing: !new:speechbrain.nnet.schedulers.NewBobScheduler
   initial_value: !ref <lr>
   improvement_threshold: 0.0025
   annealing_factor: 0.8
   patient: 0

############################## Augmentations ###################################

prepare_noise_data: !name:speechbrain.augment.preparation.prepare_dataset_from_URL
   URL: !ref <NOISE_DATASET_URL>
   dest_folder: !ref <data_folder_noise>
   ext: wav
   csv_file: !ref <noise_annotation>

# Add noise to input signal
add_noise: !new:speechbrain.augment.time_domain.AddNoise
   csv_file: !ref <noise_annotation>
   snr_low: 0
   snr_high: 15
   noise_sample_rate: !ref <sample_rate>
   clean_sample_rate: !ref <sample_rate>
   num_workers: !ref <num_workers>

# Speed perturbation
speed_perturb: !new:speechbrain.augment.time_domain.SpeedPerturb
   orig_freq: !ref <sample_rate>
   speeds: [95, 100, 105]

# Frequency drop: randomly drops a number of frequency bands to zero.
drop_freq: !new:speechbrain.augment.time_domain.DropFreq
   drop_freq_low: 0
   drop_freq_high: 1
   drop_freq_count_low: 1
   drop_freq_count_high: 3
   drop_freq_width: 0.05

# Time drop: randomly drops a number of temporal chunks.
drop_chunk: !new:speechbrain.augment.time_domain.DropChunk
   drop_length_low: 1000
   drop_length_high: 2000
   drop_count_low: 1
   drop_count_high: 5

# Augmenter: Combines previously defined augmentations to perform data augmentation
wav_augment: !new:speechbrain.augment.augmenter.Augmenter
   concat_original: True
   min_augmentations: 4
   max_augmentations: 4
   augment_prob: 1.0
   augmentations: [
      !ref <add_noise>,
      !ref <speed_perturb>,
      !ref <drop_freq>,
      !ref <drop_chunk>]

############################## Logging and Pretrainer ##########################

checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
   checkpoints_dir: !ref <save_folder>
   recoverables:
      model: !ref <model>
      scheduler: !ref <lr_annealing>
      normalizer: !ref <normalize>
      counter: !ref <epoch_counter>

train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
   save_file: !ref <train_log>

error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats

cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
   split_tokens: True

# The pretrainer allows a mapping between pretrained files and instances that
# are declared in the yaml. E.g here, we will download the file lm.ckpt
# and it will be loaded into "lm" which is pointing to the <lm_model> defined
# before.
pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer
   collect_in: !ref <save_folder>
   loadables:
      lm: !ref <lm_model>
      tokenizer: !ref <tokenizer>
   paths:
      lm: !ref <pretrained_lm_tokenizer_path>/lm.ckpt
      tokenizer: !ref <pretrained_lm_tokenizer_path>/tokenizer.ckpt