We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 31ee9fe commit f13ff6aCopy full SHA for f13ff6a
operators/tokenizer/tokenizer_jsconfig.hpp
@@ -15,7 +15,7 @@ enum class TokenType {
15
};
16
17
constexpr std::pair<const char*, TokenType> kTokenizerDict[] = {
18
- {"PreTrainedTokenizerFast", TokenType::kBPE},
+ {"PreTrainedTokenizer", TokenType::kBPE},
19
{"CLIPTokenizer", TokenType::kBPE},
20
{"WhisperTokenizer", TokenType::kBPE},
21
{"GemmaTokenizer", TokenType::kBPE},
0 commit comments