Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

vocab : validate default special token ids (#26506)

Browse Source
This commit is contained in:
Sigbjørn Skjæret
2026-08-03 17:40:53 +02:00
committed by GitHub
parent dbadb68eec
commit e8e06f78e2
1 changed files with 6 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-vocab.cpp
+6
View File
@@ -2532,6 +2532,12 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
const std::string & key = kv(std::get<0>(it)); const std::string & key = kv(std::get<0>(it));
int32_t & id = std::get<1>(it); int32_t & id = std::get<1>(it);
if (id >= 0 && static_cast<size_t>(id) >= id_to_token.size()) {
LLAMA_LOG_WARN("%s: default special token '%s' = %d out of vocab range, disabling\n",
__func__, key.c_str(), id);
id = LLAMA_TOKEN_NULL;
}
uint32_t new_id; uint32_t new_id;
if (!ml.get_key(std::get<0>(it), new_id, false)) { if (!ml.get_key(std::get<0>(it), new_id, false)) {
continue; continue;