mtmd: cap max_image to ubatch for non_causal models (#29773)
This commit is contained in:
1 parent
5503b04b05
commit
552f18f912
7 files changed
+74
-12
No files matched your search
@@ -162,6 +162,17 @@ struct mtmd_cli_context {
|
||||
mparams.warmup = params.warmup;
|
||||
mparams.image_min_tokens = params.image_min_tokens;
|
||||
mparams.image_max_tokens = params.image_max_tokens;
|
||||
{
|
||||
// non-causal models need the whole image in one ubatch
|
||||
const int n_ubatch = llama_n_ubatch(lctx);
|
||||
auto mem = mtmd_get_memory_usage(clip_path, mparams);
|
||||
if (mem.use_non_causal && mem.image_max_tokens > n_ubatch) {
|
||||
LOG_WRN("%s: cap image_max_tokens (original=%d) to n_ubatch (%d) because model needs non-causal attention on image\n", __func__, mem.image_max_tokens, n_ubatch);
|
||||
LOG_WRN("%s: increase n_ubatch (-ub) to increase vision token budget\n", __func__);
|
||||
mparams.image_max_tokens = n_ubatch;
|
||||
mparams.image_min_tokens = std::min(mparams.image_min_tokens, n_ubatch);
|
||||
}
|
||||
}
|
||||
if (std::getenv("MTMD_DEBUG_GRAPH") != nullptr) {
|
||||
mparams.cb_eval_user_data = &cb_data;
|
||||
mparams.cb_eval = common_debug_cb_eval;
|
||||
|
||||
Reference in new issue
Block a user