mtmd: cap max_image to ubatch for non_causal models (#29773)

This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-10-01 11:55:11 +02:00
1 parent 5503b04b05
commit 552f18f912
7 files changed
+74 -12

No files matched your search

+11
View File
@@ -162,6 +162,17 @@ struct mtmd_cli_context {
mparams.warmup = params.warmup;
mparams.image_min_tokens = params.image_min_tokens;
mparams.image_max_tokens = params.image_max_tokens;
{
// non-causal models need the whole image in one ubatch
const int n_ubatch = llama_n_ubatch(lctx);
auto mem = mtmd_get_memory_usage(clip_path, mparams);
if (mem.use_non_causal && mem.image_max_tokens > n_ubatch) {
LOG_WRN("%s: cap image_max_tokens (original=%d) to n_ubatch (%d) because model needs non-causal attention on image\n", __func__, mem.image_max_tokens, n_ubatch);
LOG_WRN("%s: increase n_ubatch (-ub) to increase vision token budget\n", __func__);
mparams.image_max_tokens = n_ubatch;
mparams.image_min_tokens = std::min(mparams.image_min_tokens, n_ubatch);
}
}
if (std::getenv("MTMD_DEBUG_GRAPH") != nullptr) {
mparams.cb_eval_user_data = &cb_data;
mparams.cb_eval = common_debug_cb_eval;