Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

convert : import bytes_to_unicode from convert_slow_tokenizer (#26217)

Browse Source 
bytes_to_unicode was removed from transformers.models.gpt2.tokenization_gpt2
in huggingface/transformers#40936, but it had already been copied into
transformers.convert_slow_tokenizer in huggingface/transformers#30334
(transformers 4.54.1), so import it directly from there.

Applies the same fix to chatglm.py.
This commit is contained in:
Caleb DeLeeuw
2026-08-04 10:34:30 +03:00
committed by GitHub
parent 1269cb1ff1
commit b06fbc968b
2 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/chatglm.py
+1 -1
View File
@@ -81,7 +81,7 @@ class ChatGLMModel(TextModel):
@staticmethod @staticmethod
def token_bytes_to_string(b): def token_bytes_to_string(b):
from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode # ty: ignore[unresolved-import] from transformers.convert_slow_tokenizer import bytes_to_unicode
byte_encoder = bytes_to_unicode() byte_encoder = bytes_to_unicode()
return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')]) return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])
conversion/qwen.py
+1 -1
View File
@@ -18,7 +18,7 @@ class QwenModel(TextModel):
@staticmethod @staticmethod
def token_bytes_to_string(b): def token_bytes_to_string(b):
from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode # ty: ignore[unresolved-import] from transformers.convert_slow_tokenizer import bytes_to_unicode
byte_encoder = bytes_to_unicode() byte_encoder = bytes_to_unicode()
return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')]) return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])