Skip to content

Commit 33f1bb9

Browse files
committed
Update src/llama-vocab.cpp
1 parent 2e66538 commit 33f1bb9

1 file changed

Lines changed: 0 additions & 5 deletions

File tree

src/llama-vocab.cpp

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -341,11 +341,6 @@ struct llm_tokenizer_bpe : llm_tokenizer {
341341
};
342342
break;
343343
case LLAMA_VOCAB_PRE_TYPE_EXAONE4:
344-
regex_exprs = {
345-
"\\p{N}",
346-
"'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+",
347-
};
348-
break;
349344
case LLAMA_VOCAB_PRE_TYPE_GPT2:
350345
case LLAMA_VOCAB_PRE_TYPE_MPT:
351346
case LLAMA_VOCAB_PRE_TYPE_OLMO:

0 commit comments

Comments
 (0)