From 034e2535a90718762d3bda169ff1a06bcf3e5d1e Mon Sep 17 00:00:00 2001 From: Aaron Thomas <84493696+arxhr007@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:12:39 +0530 Subject: [PATCH] gguf-py : validate every tokenizer merge pair, not only the first (#31631) * gguf-py : validate every tokenizer merge pair, not only the first The transformers 4.45 pair format was detected by checking merges[0], then every element was converted unchecked: a non-pair element raised a bare TypeError, and pairs with the wrong arity or non-string parts were silently mangled into invalid merges. Validate each element (2-string list) up front and raise the existing "Unknown tokenizer merges format" error for anything else. Assisted-by: Devin * gguf-py : remove test_vocab.py per review --- gguf-py/gguf/vocab.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index 3b82c3961..d299cfd00 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -221,10 +221,12 @@ class SpecialVocab: if isinstance(merges, list) and merges: if isinstance(merges[0], str): self.merges = merges - elif isinstance(merges[0], list) and len(merges[0]) == 2 and isinstance(merges[0][0], str): + elif isinstance(merges[0], list): # New format since transformers 4.45 to support spaces in merges # ref: https://github.com/ggml-org/llama.cpp/issues/9692 # TODO: internally store as the new format instead of converting to old + if not all(isinstance(pair, list) and len(pair) == 2 and all(isinstance(s, str) for s in pair) for pair in merges): + raise ValueError("Unknown tokenizer merges format") if any(' ' in s for pair in merges for s in pair): logger.warning(f'Spaces in merges detected, encoding as {chr(ord(" ") + 256)!r}') self.merges = [