gguf-py : validate every tokenizer merge pair, not only the first (#31631)
* gguf-py : validate every tokenizer merge pair, not only the first The transformers 4.45 pair format was detected by checking merges[0], then every element was converted unchecked: a non-pair element raised a bare TypeError, and pairs with the wrong arity or non-string parts were silently mangled into invalid merges. Validate each element (2-string list) up front and raise the existing "Unknown tokenizer merges format" error for anything else. Assisted-by: Devin * gguf-py : remove test_vocab.py per review
This commit is contained in:
1 parent
52e53a76df
commit
034e2535a9
1 file changed
+3
-1
@@ -221,10 +221,12 @@ class SpecialVocab:
|
||||
if isinstance(merges, list) and merges:
|
||||
if isinstance(merges[0], str):
|
||||
self.merges = merges
|
||||
elif isinstance(merges[0], list) and len(merges[0]) == 2 and isinstance(merges[0][0], str):
|
||||
elif isinstance(merges[0], list):
|
||||
# New format since transformers 4.45 to support spaces in merges
|
||||
# ref: https://github.com/ggml-org/llama.cpp/issues/9692
|
||||
# TODO: internally store as the new format instead of converting to old
|
||||
if not all(isinstance(pair, list) and len(pair) == 2 and all(isinstance(s, str) for s in pair) for pair in merges):
|
||||
raise ValueError("Unknown tokenizer merges format")
|
||||
if any(' ' in s for pair in merges for s in pair):
|
||||
logger.warning(f'Spaces in merges detected, encoding as {chr(ord(" ") + 256)!r}')
|
||||
self.merges = [
|
||||
|
||||
Reference in new issue
Block a user