Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: add mtmd_tokenize_from_parts() (#28250)

Browse Source 
* add mtmd_tokenize_from_parts

* use it in mtmd-cli

* move add_special to call level
This commit is contained in:
Xuan-Son Nguyen
2026-09-02 21:20:10 +02:00
committed by GitHub
parent 9cc33944f9
commit 7339054744
7 changed files with 154 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/mtmd-cli.cpp
+43 -15
View File
@@ -109,16 +109,15 @@ struct mtmd_cli_context {
mtmd_cli_context(common_params & params) : llama_init(common_init_from_params(params)) {
model = llama_init->model();
lctx = llama_init->context();
if (!model || !lctx) {
exit(1);
}
vocab = llama_model_get_vocab(model);
smpl = common_sampler_init(model, params.sampling);
n_threads = params.cpuparams.n_threads;
batch = llama_batch_init(1, 0, 1); // batch for next token generation
n_batch = params.n_batch;
if (!model || !lctx) {
exit(1);
}
init_vision_context(params);
if (!mtmd_helper_model_can_chat(lctx, ctx_vision.get())) {
@@ -265,21 +264,50 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) {
auto formatted_chat = chat_add_and_format(ctx, msg);
LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.c_str());
mtmd_input_text text;
text.text = formatted_chat.data();
text.text_len = formatted_chat.size();
text.add_special = add_bos;
text.parse_special = true;
if (g_is_interrupted) return 0;
mtmd::input_chunks chunks(mtmd_input_chunks_init());
// note: we replace the marker here instead of letting mtmd_tokenize() to do that
// because we want to demonstrate how to use mtmd_tokenize_from_parts()
// split the formatted chat on the media marker to get text segments
const std::string marker = mtmd_default_marker();
std::vector<std::string> segments;
size_t start = 0;
size_t pos;
while ((pos = formatted_chat.find(marker, start)) != std::string::npos) {
segments.push_back(formatted_chat.substr(start, pos - start));
start = pos + marker.size();
}
segments.push_back(formatted_chat.substr(start));
auto bitmaps_c_ptr = ctx.bitmaps.c_ptr();
int32_t res = mtmd_tokenize(ctx.ctx_vision.get(),
if (segments.size() - 1 != bitmaps_c_ptr.size()) {
LOG_ERR("Number of media markers (%zu) does not match number of loaded media (%zu)\n",
segments.size() - 1, bitmaps_c_ptr.size());
return 1;
}
// interleave text and media parts
std::vector<mtmd_input_text> texts(segments.size());
std::vector<mtmd_input_part> parts;
for (size_t i = 0; i < segments.size(); i++) {
texts[i] = {segments[i].data(), segments[i].size(), /* add_special */ false, /* parse_special */ true};
parts.push_back({&texts[i], nullptr});
if (i < bitmaps_c_ptr.size()) {
parts.push_back({nullptr, bitmaps_c_ptr[i]});
}
}
std::vector<const mtmd_input_part *> parts_ptr;
for (const auto & p : parts) {
parts_ptr.push_back(&p);
}
mtmd::input_chunks chunks(mtmd_input_chunks_init());
int32_t res = mtmd_tokenize_from_parts(ctx.ctx_vision.get(),
chunks.ptr.get(), // output
&text, // text
bitmaps_c_ptr.data(),
bitmaps_c_ptr.size());
parts_ptr.data(),
parts_ptr.size(),
add_bos);
if (res != 0) {
LOG_ERR("Unable to tokenize prompt, res = %d\n", res);
return 1;