diff --git a/borb/io/read/tokenize/low_level_tokenizer.py b/borb/io/read/tokenize/low_level_tokenizer.py index d085ab85..a64eaae1 100644 --- a/borb/io/read/tokenize/low_level_tokenizer.py +++ b/borb/io/read/tokenize/low_level_tokenizer.py @@ -236,7 +236,7 @@ def next_token(self) -> typing.Optional[Token]: while True: ch = self._next_byte() if len(ch) == 0: - break + return None # escape char if ch == b"\\": ch = self._next_byte()