stricter check

Signed-off-by: alessiodevoto <devoto.alessio@gmail.com>

Author: alessiodevoto

logits_processor_zoo/utils.py

@@ -24,7 +24,7 @@ def text_to_token(tokenizer: PreTrainedTokenizer, text: str, last: bool):
     tokens = tokenizer.encode(text, add_special_tokens=False)
 
     # We allow 2 tokens to account for the BOS token
-    max_token_count = 2 if tokenizer.bos_token_id else 1
+    max_token_count = 2 if tokenizer.bos_token and tokenizer.bos_token_id in tokens else 1
 
     if not last and len(tokens) > max_token_count:
         raise Exception(f"Can't convert {text} to token. It has {len(tokens)} tokens.")