Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: check gguf array type before reading (#27075)

Browse Source 
* fix: check gguf array type before reading

* update skill
This commit is contained in:
Xuan-Son Nguyen
2026-08-15 11:45:30 +02:00
committed by GitHub
parent 9b0a2ce859
commit 27df9199d1
6 changed files with 86 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp
+22 -8
View File
@@ -549,20 +549,34 @@ static void load_vocab(const char * filename, const Config * config, struct my_l
const int token_idx = gguf_find_key(ctx, KV_TOKENIZER_LIST);
GGML_ASSERT(token_idx >= 0);
const int score_idx = gguf_find_key(ctx, KV_TOKENIZER_SCORES);
GGML_ASSERT(score_idx >= 0);
const float * scores = (const float * ) gguf_get_arr_data(ctx, score_idx);
const int toktype_idx = gguf_find_key(ctx, KV_TOKENIZER_TOKEN_TYPE);
GGML_ASSERT(toktype_idx >= 0);
const int * toktypes = (const int * ) gguf_get_arr_data(ctx, toktype_idx);
if (gguf_get_kv_type(ctx, token_idx) != GGUF_TYPE_ARRAY ||
gguf_get_arr_type(ctx, token_idx) != GGUF_TYPE_STRING) {
die_fmt("invalid gguf type for %s", KV_TOKENIZER_LIST);
}
const uint32_t n_vocab = gguf_get_arr_n(ctx, token_idx);
if (n_vocab != static_cast<uint32_t>(config->vocab_size)) {
die_fmt("vocab size mismatch: (gguf) %u != (llama2c) %d", n_vocab, config->vocab_size);
}
const int score_idx = gguf_find_key(ctx, KV_TOKENIZER_SCORES);
GGML_ASSERT(score_idx >= 0);
if (gguf_get_kv_type(ctx, score_idx) != GGUF_TYPE_ARRAY ||
gguf_get_arr_type(ctx, score_idx) != GGUF_TYPE_FLOAT32 ||
gguf_get_arr_n(ctx, score_idx) < n_vocab) {
die_fmt("invalid gguf type or size for %s", KV_TOKENIZER_SCORES);
}
const float * scores = (const float * ) gguf_get_arr_data(ctx, score_idx);
const int toktype_idx = gguf_find_key(ctx, KV_TOKENIZER_TOKEN_TYPE);
GGML_ASSERT(toktype_idx >= 0);
if (gguf_get_kv_type(ctx, toktype_idx) != GGUF_TYPE_ARRAY ||
gguf_get_arr_type(ctx, toktype_idx) != GGUF_TYPE_INT32 ||
gguf_get_arr_n(ctx, toktype_idx) < n_vocab) {
die_fmt("invalid gguf type or size for %s", KV_TOKENIZER_TOKEN_TYPE);
}
const int * toktypes = (const int * ) gguf_get_arr_data(ctx, toktype_idx);
vocab->id_to_token.resize(n_vocab);
for (uint32_t i = 0; i < n_vocab; i++) {