diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index 3b82c3961..d299cfd00 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -221,10 +221,12 @@ class SpecialVocab: if isinstance(merges, list) and merges: if isinstance(merges[0], str): self.merges = merges - elif isinstance(merges[0], list) and len(merges[0]) == 2 and isinstance(merges[0][0], str): + elif isinstance(merges[0], list): # New format since transformers 4.45 to support spaces in merges # ref: https://github.com/ggml-org/llama.cpp/issues/9692 # TODO: internally store as the new format instead of converting to old + if not all(isinstance(pair, list) and len(pair) == 2 and all(isinstance(s, str) for s in pair) for pair in merges): + raise ValueError("Unknown tokenizer merges format") if any(' ' in s for pair in merges for s in pair): logger.warning(f'Spaces in merges detected, encoding as {chr(ord(" ") + 256)!r}') self.merges = [