fix: custom tokenizer truncates inputs to model max input length (kubernetes-sigs#266)

Author: changminbark

inference_perf/utils/custom_tokenizer.py

@@ -24,7 +24,7 @@ def __init__(self, config: CustomTokenizerConfig) -> None:
     def count_tokens(self, text: str) -> int:
         if text == "":
             return 0
-        return len(self.tokenizer(text).input_ids)
+        return len(self.tokenizer(text, truncation=True, max_length=self.tokenizer.model_max_length).input_ids)
 
     def get_tokenizer(self) -> PreTrainedTokenizerBase:
         return self.tokenizer