diff --git a/operators/tokenizer/tokenizer_jsconfig.hpp b/operators/tokenizer/tokenizer_jsconfig.hpp index 4346fbacc..e185b951f 100644 --- a/operators/tokenizer/tokenizer_jsconfig.hpp +++ b/operators/tokenizer/tokenizer_jsconfig.hpp @@ -21,6 +21,7 @@ constexpr std::pair kTokenizerDict[] = { {"CLIPTokenizer", TokenType::kBPE}, {"WhisperTokenizer", TokenType::kBPE}, {"GemmaTokenizer", TokenType::kBPE}, + {"InternLM2Tokenizer", TokenType::kBPE}, // InternLM2 uses BPE (same as Llama) {"LlamaTokenizer", TokenType::kBPE}, {"Phi3Tokenizer", TokenType::kBPE}, {"CodeLlamaTokenizer", TokenType::kBPE},