Special tokens included in the parameters

Author: WojciechMat

midi_tokenizers/no_loss_tokenizer.py

@@ -54,7 +54,11 @@ def __rich_repr__(self):
 
     @property
     def parameters(self):
-        return {"min_time_unit": self.min_time_unit, "n_velocity_bins": self.n_velocity_bins}
+        return {
+            "min_time_unit": self.min_time_unit,
+            "n_velocity_bins": self.n_velocity_bins,
+            "special_tokens": self.special_tokens,
+        }
 
     @property
     def vocab_size(self) -> int:

scripts/test_tokenizer.py

@@ -5,7 +5,6 @@
 from fortepyan import MidiPiece
 from datasets import load_dataset
 
-from midi_trainable_tokenizers import AwesomeMidiTokenizer
 from midi_tokenizers.no_loss_tokenizer import ExponentialTimeTokenizer
 
 
@@ -91,6 +90,8 @@ def main():
     print(f"MidiPieces loaded in {loading_time:.2f} seconds")
 
     tokenizer = ExponentialTimeTokenizer(min_time_unit=0.01, n_velocity_bins=32)
+    tokenizer_desc = tokenizer.to_dict()
+    tokenizer = ExponentialTimeTokenizer.from_dict(tokenizer_desc=tokenizer_desc)
     # tokenizer = AwesomeMidiTokenizer.from_file("dumps/awesome_tokenizers/awesome-tokenizer-test-2024-06-11_17-11-44.json")
 
     print("\nRunning speed and accuracy test for ExponentialTimeTokenizer on validation split")