Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

vocab : validate plamo2 byte tokens (#26511)

Browse Source 
* validate plamo2 byte tokens

* --typo
This commit is contained in:
Sigbjørn Skjæret
2026-08-04 11:40:02 +03:00
committed by GitHub
parent b06fbc968b
commit cf06ad7dfe
1 changed files with 12 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-vocab.cpp
+6 -1
View File
@@ -1373,9 +1373,11 @@ struct llm_tokenizer_plamo2 : llm_tokenizer {
if (vocab.is_byte(token_id)) {
if (entry.text.length() == 6 && entry.text.substr(0, 3) == "<0x" && entry.text.back() == '>') {
std::string hex_str = entry.text.substr(3, 2);
if (std::isxdigit(static_cast<unsigned char>(hex_str[0])) && std::isxdigit(static_cast<unsigned char>(hex_str[1]))) {
int byte_val = std::stoi(hex_str, nullptr, 16);
bytes_[byte_val] = static_cast<llama_token>(token_id);
}
}
continue;
}
@@ -3625,7 +3627,9 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t
if (vocab.is_byte(token)) {
// Handle byte tokens like <0xXX>
if (token_text.length() == 6 && token_text.substr(0, 3) == "<0x" && token_text.back() == '>') {
int hex_val = std::stoi(token_text.substr(3, 2), nullptr, 16);
std::string hex_str = token_text.substr(3, 2);
if (std::isxdigit(static_cast<unsigned char>(hex_str[0])) && std::isxdigit(static_cast<unsigned char>(hex_str[1]))) {
int hex_val = std::stoi(hex_str, nullptr, 16);
if (length < 1) {
return -1;
}
@@ -3633,6 +3637,7 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t
return 1;
}
}
}
// Normal token - just copy the text
std::string result = token_text;