There was an error while loading. Please reload this page.
1 parent 737eb70 commit 8d5c51aCopy full SHA for 8d5c51a
1 file changed
src/fairseq2/models/wav2vec2/asr/_config.py
@@ -402,3 +402,29 @@ def v3_tokenizer_7b() -> Wav2Vec2AsrConfig:
402
config = bib1143_7b()
403
config.vocab_info.size = 9656
404
return config
405
+
406
+ @wav2vec2_asr_arch("7b_v4_exp_tokenizer")
407
+ def v4_exp_tokenizer_7b() -> Wav2Vec2AsrConfig:
408
+ config = bib1143_7b()
409
+ config.vocab_info = VocabularyInfo(
410
+ size=9722,
411
+ unk_idx=3,
412
+ bos_idx=0,
413
+ eos_idx=2,
414
+ pad_idx=1,
415
+ )
416
+ return config
417
418
+ @wav2vec2_asr_arch("7b_bpe_tokenizer")
419
+ def bpe_tokenizer_7b() -> Wav2Vec2AsrConfig:
420
421
422
+ size=5042,
423
424
+ bos_idx=4786,
425
+ eos_idx=4787,
426
+ pad_idx=4790,
427
+ boh_idx=4792,
428
+ eoh_idx=4793,
429
430
0 commit comments