v2.0 (#5)

v.2.0!

---------

Co-authored-by: Tomek Roszczynialski <roszcz@users.noreply.github.com>

Author: WojciechMat

README.md

@@ -104,11 +104,11 @@ For example, for `min_time_unit=0.01`, time token values are:
 {
     "1T": "10ms",
     "2T": "20ms",
-    "3T": "40ms",
-    "4T": "80ms",
-    "5T": "160ms",
-    "6T": "320ms",
-    "7T": "640ms",
+    "4T": "40ms",
+    "8T": "80ms",
+    "16T": "160ms",
+    "32T": "320ms",
+    "64T": "640ms",
 }
 ```
 
@@ -167,6 +167,7 @@ Let's illustrate the tokenization process with a simple example. Given a DataFra
 
 ```python
 import pandas as pd
+from midi_tokenizers import ExponentialTimeTokenizer
 
 # Sample MIDI data
 data = pd.DataFrame({
@@ -177,7 +178,6 @@ data = pd.DataFrame({
 })
 
 # Initialize the Exponential Time Tokenizer
-from midi_tokenizers import ExponentialTimeTokenizer
 exp_time_tokenizer = ExponentialTimeTokenizer()
 
 # Tokenize the sample data
@@ -190,7 +190,7 @@ print(tokens)
 The output tokens might look like this:
 
 ```
-['VELOCITY_94', 'NOTE_ON_59', '4T', 'VELOCITY_94', 'NOTE_OFF_59', 'VELOCITY_77', 'NOTE_ON_48', '2T', 'VELOCITY_95', 'NOTE_ON_60', '3T', '2T', 'VELOCITY_79', 'NOTE_ON_47', '2T', 'VELOCITY_77', 'NOTE_OFF_48', 'VELOCITY_97', 'NOTE_ON_59', '3T']
+['VELOCITY_94', 'NOTE_ON_59', '4T', '2T', '1T', 'NOTE_OFF_59', '1T', 'VELOCITY_77', 'NOTE_ON_48', '2T', 'VELOCITY_95', 'NOTE_ON_60', '4T', '2T', 'VELOCITY_79', 'NOTE_ON_47', '2T', 'NOTE_OFF_48', '4T', '1T', 'NOTE_OFF_60', '4T', '1T', 'NOTE_OFF_47']
 ```
 
 In this example, the tokens represent the time intervals (`1T`, `2T`), velocities (`VELOCITY_92`, `VELOCITY_110`, etc.), and the note events (`NOTE_ON_74`, `NOTE_OFF_74`, etc.).
@@ -256,8 +256,7 @@ When applying BPE to MIDI data, the process involves several steps to convert th
 Here is an example demonstrating the process:
 
 ```python
-from midi_trainable_tokenizers import AwesomeMidiTokenizer
-from midi_tokenizers import ExponentialTimeTokenizer
+from midi_tokenizers import ExponentialTimeTokenizer, AwesomeMidiTokenizer
 from datasets import load_dataset
 import pandas as pd
 
@@ -288,7 +287,7 @@ print(tokens)
 ```
 Output:
 ```plaintext
-['Ŵ±', 'ƘŴ', '²ţ', '\x9b', 'Ɩŵ', '³', 'ƗƖť', '\x99', 'Ɩţ', '\x9c', 'ƗƖŵ', '´Ɨť', '\x9a']
+['Ŵ', '²ƘƗƖ³Ɩ', 'ţ\x9cƗ', 'ŵ', '´ƘƗ', 'ť\x9aƗ', '\x9dƘƖ', 'µƘƖ', '\x9b']
 ```
 
 This example demonstrates how to use the `AwesomeMidiTokenizer` to tokenize a sample MIDI data. The tokenizer first needs to be trained on a dataset before it can be used to tokenize new data. The training process uses the `ExponentialTimeTokenizer` as a base tokenizer and trains the BPE tokenizer on the specified dataset. After training, the tokenizer can convert new MIDI data into a sequence of tokens.
@@ -299,7 +298,7 @@ This process ensures efficient encoding of MIDI data with minimal loss of inform
 ### BPE MIDI Tokenizer
 Like Awesome Tokenizer, but without converting to unicode and only merges time tokens.
 ```python
-from midi_trainable_tokenizers import BpeMidiTokenizer
+from midi_tokenizers import BpeMidiTokenizer
 
 # Initialize the base tokenizer
 base_tokenizer = oneTimeTokenizer()

__init__.py

@@ -9,8 +9,8 @@
 from datasets import Dataset, load_dataset
 from tokenizers.pre_tokenizers import PreTokenizer
 
-from midi_trainable_tokenizers.awesome_midi_tokenzier import AwesomeMidiTokenizer
-from midi_tokenizers_generation.base_tokenizer_generator import (
+from midi_tokenizers.midi_trainable_tokenizers.awesome_midi_tokenzier import AwesomeMidiTokenizer
+from midi_tokenizers.midi_tokenizers_generation.base_tokenizer_generator import (
     tokenizer_info,
     generate_tokenizer,
     name_to_base_factory_map,

dashboards/awesome_tokenizer_review.py

@@ -9,8 +9,8 @@
 from datasets import Dataset, load_dataset
 from tokenizers.pre_tokenizers import PreTokenizer
 
-from midi_trainable_tokenizers.bpe_tokenizer import BpeMidiTokenizer
-from midi_tokenizers_generation.base_tokenizer_generator import (
+from midi_tokenizers.midi_trainable_tokenizers.bpe_tokenizer import BpeMidiTokenizer
+from midi_tokenizers.midi_tokenizers_generation.base_tokenizer_generator import (
     tokenizer_info,
     generate_tokenizer,
     name_to_base_factory_map,

dashboards/bpe_review.py

@@ -4,7 +4,7 @@
 from fortepyan import MidiPiece
 from datasets import Dataset, load_dataset
 
-from midi_tokenizers_generation.quantizer_generator import (
+from midi_tokenizers.midi_tokenizers_generation.quantizer_generator import (
     quantization_info,
     name_to_quantizer_factory_map,
     generate_quantizer_with_streamlit,

dashboards/quantizer_review.py

@@ -4,7 +4,11 @@
 from fortepyan import MidiPiece
 from datasets import Dataset, load_dataset
 
-from midi_tokenizers_generation.tokenizer_generator import tokenizer_info, name_to_factory_map, generate_tokenizer_with_streamlit
+from midi_tokenizers.midi_tokenizers_generation.tokenizer_generator import (
+    tokenizer_info,
+    name_to_factory_map,
+    generate_tokenizer_with_streamlit,
+)
 
 
 @st.cache_data

dashboards/tokenizer_review.py

@@ -1,36 +1,17 @@
-"""
-The midi_tokenizers package provides utilities to tokenize and process MIDI files
-for various tasks, including music generation and analysis.
-
-This package includes the following modules and classes:
-
-- MidiTokenizer: Base class for all MIDI tokenizers.
-- OneTimeTokenizer: Tokenizer that uses a single time token.
-- ExponentialTimeTokenizer: Tokenizer that uses multiple time tokens, rising exponentially.
-- QuantizedMidiTokenizer: Tokenizer that uses quantization to first bin the data and then
-treats all possible combinations as separate tokens.
-
-Example usage:
-    from midi_tokenizers import OneTimeTokenizer
-
-    # Initialize a tokenizer
-    tokenizer = OneTimeTokenizer(min_time_unit=0.01, n_velocity_bins=128)
-
-    # Tokenize MIDI notes
-    tokens = tokenizer.tokenize(notes)
-
-    # Untokenize to get back MIDI notes
-    notes = tokenizer.untokenize(tokens)
-"""
-
-from .midi_tokenizer import MidiTokenizer
-from .one_time_tokenizer import OneTimeTokenizer
-from .no_loss_tokenizer import ExponentialTimeTokenizer
-from .quantized_midi_tokenizer import QuantizedMidiTokenizer
+from .base_tokenizers.midi_tokenizer import MidiTokenizer
+from .base_tokenizers.one_time_tokenizer import OneTimeTokenizer
+from .midi_trainable_tokenizers.bpe_tokenizer import BpeMidiTokenizer
+from .base_tokenizers.quantized_midi_tokenizer import QuantizedMidiTokenizer
+from .base_tokenizers.exponential_time_tokenizer import ExponentialTimeTokenizer
+from .midi_trainable_tokenizers.trainable_tokenizer import MidiTrainableTokenizer
+from .midi_trainable_tokenizers.awesome_midi_tokenzier import AwesomeMidiTokenizer
 
 __all__ = [
     "MidiTokenizer",
     "ExponentialTimeTokenizer",
     "OneTimeTokenizer",
     "QuantizedMidiTokenizer",
+    "MidiTrainableTokenizer",
+    "BpeMidiTokenizer",
+    "AwesomeMidiTokenizer",
 ]

midi_tokenizers/__init__.py

@@ -0,0 +1,36 @@
+"""
+The midi_tokenizers package provides utilities to tokenize and process MIDI files
+for various tasks, including music generation and analysis.
+
+This package includes the following modules and classes:
+
+- MidiTokenizer: Base class for all MIDI tokenizers.
+- OneTimeTokenizer: Tokenizer that uses a single time token.
+- ExponentialTimeTokenizer: Tokenizer that uses multiple time tokens, rising exponentially.
+- QuantizedMidiTokenizer: Tokenizer that uses quantization to first bin the data and then
+treats all possible combinations as separate tokens.
+
+Example usage:
+    from midi_tokenizers import OneTimeTokenizer
+
+    # Initialize a tokenizer
+    tokenizer = OneTimeTokenizer(min_time_unit=0.01, n_velocity_bins=128)
+
+    # Tokenize MIDI notes
+    tokens = tokenizer.tokenize(notes)
+
+    # Untokenize to get back MIDI notes
+    notes = tokenizer.untokenize(tokens)
+"""
+
+from .midi_tokenizer import MidiTokenizer
+from .one_time_tokenizer import OneTimeTokenizer
+from .quantized_midi_tokenizer import QuantizedMidiTokenizer
+from .exponential_time_tokenizer import ExponentialTimeTokenizer
+
+__all__ = [
+    "MidiTokenizer",
+    "ExponentialTimeTokenizer",
+    "OneTimeTokenizer",
+    "QuantizedMidiTokenizer",
+]

midi_tokenizers/base_tokenizers/__init__.py

@@ -1,7 +1,7 @@
 import numpy as np
 import pandas as pd
 
-from midi_tokenizers.midi_tokenizer import MidiTokenizer
+from midi_tokenizers.base_tokenizers.midi_tokenizer import MidiTokenizer
 
 
 class ExponentialTimeTokenizer(MidiTokenizer):
@@ -126,7 +126,7 @@ def _time_vocab(self) -> tuple[dict, dict, dict]:
             dt_to_token |= {dt: time_token}
             token_to_dt |= {time_token: dt}
             dt *= 2
-            dt_it += 1
+            dt_it *= 2
         return time_vocab, token_to_dt, dt_to_token
 
     def quantize_frame(self, df: pd.DataFrame):
@@ -143,6 +143,8 @@ def quantize_frame(self, df: pd.DataFrame):
         df["velocity_bin"] = np.digitize(df["velocity"], self.velocity_bin_edges) - 1
         df["start"] = np.round(df["start"] / self.min_time_unit) * self.min_time_unit
         df["end"] = np.round(df["end"] / self.min_time_unit) * self.min_time_unit
+        # We have to manually prevent notes with 0.0 duration after rounding
+        df.loc[df["start"] == df["end"], "end"] += self.min_time_unit
         df["duration"] = df["end"] - df["start"]
         return df
 
@@ -216,8 +218,8 @@ def tokenize(self, notes: pd.DataFrame) -> list[str]:
         for event in events:
             dt = event["time"] - previous_time
             tokens.extend(self.tokenize_time_distance(dt))
-            tokens.append(self.velocity_bin_to_token[event["velocity_bin"]])
             if event["event"] == "NOTE_ON":
+                tokens.append(self.velocity_bin_to_token[event["velocity_bin"]])
                 tokens.append(self.pitch_to_on_token[event["pitch"]])
             else:
                 tokens.append(self.pitch_to_off_token[event["pitch"]])