Skip to content

Commit 8d5c51a

Browse files
author
mattsetzler
committed
Update CTC model config for BPE tokenizer.
1 parent 737eb70 commit 8d5c51a

1 file changed

Lines changed: 26 additions & 0 deletions

File tree

src/fairseq2/models/wav2vec2/asr/_config.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -402,3 +402,29 @@ def v3_tokenizer_7b() -> Wav2Vec2AsrConfig:
402402
config = bib1143_7b()
403403
config.vocab_info.size = 9656
404404
return config
405+
406+
@wav2vec2_asr_arch("7b_v4_exp_tokenizer")
407+
def v4_exp_tokenizer_7b() -> Wav2Vec2AsrConfig:
408+
config = bib1143_7b()
409+
config.vocab_info = VocabularyInfo(
410+
size=9722,
411+
unk_idx=3,
412+
bos_idx=0,
413+
eos_idx=2,
414+
pad_idx=1,
415+
)
416+
return config
417+
418+
@wav2vec2_asr_arch("7b_bpe_tokenizer")
419+
def bpe_tokenizer_7b() -> Wav2Vec2AsrConfig:
420+
config = bib1143_7b()
421+
config.vocab_info = VocabularyInfo(
422+
size=5042,
423+
unk_idx=3,
424+
bos_idx=4786,
425+
eos_idx=4787,
426+
pad_idx=4790,
427+
boh_idx=4792,
428+
eoh_idx=4793,
429+
)
430+
return config

0 commit comments

Comments
 (0)