Add log

timothyngo · timothyngo · commit 329d39f53fcd · 2025-12-18T15:22:53.000-05:00
diff --git a/src/tatm/tokenizer/engine.py b/src/tatm/tokenizer/engine.py
@@ -74,6 +74,7 @@ def get_example(self):
             return ExampleMessage(data=example, content_field=content_field)
         except StopIteration:
             self.dataset_iters.pop(dataset_idx)
+            LOGGER.info("A dataset has been exhausted and removed from rotation.")
             return self.get_example()
 
     def run(self):
