unicode : include '~' in collapsed symbol class (#26972)
The collapsed \p{S} class was missing '~', which split " ~" into
separate pre-tokens and prevented the Ġ~ BPE merge used by DeepSeek V4.
This caused re-tokenized prompts to diverge from sampled tokens and
broke KV cache reuse.
Assisted-by: Codex
This commit is contained in:
@@ -116,6 +116,8 @@ function(llama_build_and_test source)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY LABELS ${LLAMA_TEST_LABEL})
|
||||
endfunction()
|
||||
|
||||
llama_build_and_test(test-unicode.cpp)
|
||||
|
||||
# build test-tokenizer-0 target once and add many tests
|
||||
llama_build(test-tokenizer-0.cpp)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user