Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

vocab : support integer tokenizer scores (#27260)

Browse Source
This commit is contained in:
Sigbjørn Skjæret
2026-08-17 20:13:15 +02:00
committed by GitHub
parent b75ecd1971
commit d8df12ebc4
1 changed files with 17 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-vocab.cpp
+16 -3
View File
@@ -2428,18 +2428,25 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
const uint32_t n_tokens = gguf_get_arr_n(ctx, token_idx); const uint32_t n_tokens = gguf_get_arr_n(ctx, token_idx);
const float * scores = nullptr; const float * scores = nullptr;
const int * iscores = nullptr;
const int score_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SCORES).c_str()); const int score_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SCORES).c_str());
if (score_idx != -1) { if (score_idx != -1) {
if (gguf_get_kv_type(ctx, score_idx) != GGUF_TYPE_ARRAY || const gguf_type kv_type = gguf_get_kv_type(ctx, score_idx);
gguf_get_arr_type(ctx, score_idx) != GGUF_TYPE_FLOAT32) { const gguf_type arr_type = kv_type == GGUF_TYPE_ARRAY ? gguf_get_arr_type(ctx, score_idx) : GGUF_TYPE_COUNT;
if (arr_type != GGUF_TYPE_INT32 &&
arr_type != GGUF_TYPE_FLOAT32) {
throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_SCORES).c_str())); throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_SCORES).c_str()));
} }
const uint32_t n_scores = gguf_get_arr_n(ctx, score_idx); const uint32_t n_scores = gguf_get_arr_n(ctx, score_idx);
if (n_scores < n_tokens) { if (n_scores < n_tokens) {
throw std::runtime_error("Index out of array bounds for scores (" + std::to_string(n_scores) + " < " + std::to_string(n_tokens) + ")\n"); throw std::runtime_error("Index out of array bounds for scores (" + std::to_string(n_scores) + " < " + std::to_string(n_tokens) + ")\n");
} }
if (arr_type == GGUF_TYPE_INT32) {
iscores = (const int *) gguf_get_arr_data(ctx, score_idx);
} else {
scores = (const float * ) gguf_get_arr_data(ctx, score_idx); scores = (const float * ) gguf_get_arr_data(ctx, score_idx);
} }
}
const int * toktypes = nullptr; const int * toktypes = nullptr;
const int toktype_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_TOKEN_TYPE).c_str()); const int toktype_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_TOKEN_TYPE).c_str());
@@ -2469,7 +2476,13 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
auto & token_data = id_to_token[i]; auto & token_data = id_to_token[i];
token_data.text = std::move(word); token_data.text = std::move(word);
token_data.score = scores ? scores[i] : 0.0f; if (scores) {
token_data.score = scores[i];
} else if (iscores) {
token_data.score = static_cast<float>(iscores[i]);
} else {
token_data.score = 0.0f;
}
token_data.attr = LLAMA_TOKEN_ATTR_NORMAL; token_data.attr = LLAMA_TOKEN_ATTR_NORMAL;
if (toktypes) { //TODO: remove, required until per token attributes are available from GGUF file if (toktypes) { //TODO: remove, required until per token attributes are available from GGUF file