diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp index faabd4840..72e8ae657 100644 --- a/tools/mtmd/mtmd-cli.cpp +++ b/tools/mtmd/mtmd-cli.cpp @@ -91,8 +91,6 @@ struct mtmd_cli_context { mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default(); std::string video_ffmpeg_bin_dir; - mtmd::batch_ptr mbatch; - // chat template common_chat_templates_ptr tmpls; std::vector chat_history; @@ -408,6 +406,8 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) { } mtmd::input_chunks chunks(mtmd_input_chunks_init()); + // mbatch must not outlive the chunks it references + mtmd::batch_ptr mbatch; int32_t res = mtmd_tokenize_from_parts(ctx.ctx_vision.get(), chunks.ptr.get(), // output parts_ptr.data(), @@ -446,8 +446,8 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) { } else { // media chunk: try to get embd from existing batch, or create a new batch float * embd = nullptr; - if (ctx.mbatch) { - embd = mtmd_batch_get_output_embd(ctx.mbatch.get(), chunk); + if (mbatch) { + embd = mtmd_batch_get_output_embd(mbatch.get(), chunk); if (embd) { LOG_DBG("found embd for media chunk %zu in existing batch\n", i); @@ -458,8 +458,8 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) { if (!embd) { // create and encode a new batch with as many media chunks as possible - ctx.mbatch.reset(mtmd_batch_init(ctx.ctx_vision.get())); - res = mtmd_batch_add_chunk(ctx.mbatch.get(), chunk); + mbatch.reset(mtmd_batch_init(ctx.ctx_vision.get())); + res = mtmd_batch_add_chunk(mbatch.get(), chunk); GGML_ASSERT(res == 0); // first chunk must always succeed int n_added = 1; @@ -470,7 +470,7 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) { if (next_type == MTMD_INPUT_CHUNK_TYPE_TEXT) { break; // text chunk splits the batch } - res = mtmd_batch_add_chunk(ctx.mbatch.get(), next_chunk); + res = mtmd_batch_add_chunk(mbatch.get(), next_chunk); if (res != 0) { break; // batch full or incompatible } @@ -479,14 +479,14 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) { int64_t time_start = ggml_time_ms(); LOG_INF("encoding mtmd batch, n_chunks = %d (done = %zu, total = %zu)\n", n_added, i, n_chunks); - res = mtmd_batch_encode(ctx.mbatch.get()); + res = mtmd_batch_encode(mbatch.get()); if (res != 0) { LOG_ERR("Failed to encode mtmd batch, res = %d\n", res); return 1; } LOG_INF("mtmd batch encoding done in %d ms\n", (int)(ggml_time_ms() - time_start)); - embd = mtmd_batch_get_output_embd(ctx.mbatch.get(), chunk); + embd = mtmd_batch_get_output_embd(mbatch.get(), chunk); } GGML_ASSERT(embd != nullptr);