early exit

Author: Artyom Kozhevnikov

src/fairseq2/datasets/asr.py

@@ -8,19 +8,17 @@
 
 from abc import ABC, abstractmethod
 from functools import partial
-from typing import Any, Dict, Final, Tuple, cast
-
-from typing_extensions import override
+from typing import Any, cast, Dict, Final, Tuple
 
 from fairseq2.data import (
     CollateOptionsOverride,
     Collater,
     DataPipeline,
     DataPipelineBuilder,
-    SequenceData,
     read_sequence,
+    SequenceData,
 )
-from fairseq2.data.text import StrSplitter, read_text
+from fairseq2.data.text import read_text, StrSplitter
 from fairseq2.data.text.tokenizers import TextTokenizer
 from fairseq2.datasets import (
     DataPipelineReader,
@@ -38,6 +36,8 @@
 from fairseq2.nn.padding import get_seqs_and_padding_mask
 from fairseq2.typing import Device
 
+from typing_extensions import override
+
 
 class AsrDataset(ABC):
     """Represents an automatic speech recognition dataset."""
@@ -167,6 +167,10 @@ def add_tokenization_pipeline(
     ) -> DataPipelineBuilder:
         # Tokenize target text.
         text_encoder = tokenizer.create_encoder()
+
+        # to avoid to tokenize empty text, we filter out them out first
+        builder = builder.filter(lambda x: bool(len(x["text"]) > 0))
+
         builder.map(text_encoder, selector="text")
 
         unk_idx = tokenizer.vocab_info.unk_idx