We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 2e66538 commit 33f1bb9Copy full SHA for 33f1bb9
1 file changed
src/llama-vocab.cpp
@@ -341,11 +341,6 @@ struct llm_tokenizer_bpe : llm_tokenizer {
341
};
342
break;
343
case LLAMA_VOCAB_PRE_TYPE_EXAONE4:
344
- regex_exprs = {
345
- "\\p{N}",
346
- "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+",
347
- };
348
- break;
349
case LLAMA_VOCAB_PRE_TYPE_GPT2:
350
case LLAMA_VOCAB_PRE_TYPE_MPT:
351
case LLAMA_VOCAB_PRE_TYPE_OLMO:
0 commit comments