batch: migrate speculative, mtmd and server to batch_ext (#29385)

* adapt common

* add common_batch

* wip

* wip: spec

* cont

* common_speculative_process

* server_batch to use common_batch

* rm some stale calls

Assisted-by: Claude Fable 5.1

* migrate mtmd

* handle imrope, handle return val of add()/add_embd()

* add spec zeros vector

* add warning on zero fill path
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-09-28 19:52:45 +02:00
1 parent 6c7a87f7e5
commit f1ea206218
11 files changed
+544 -456

No files matched your search

+5 -6
View File
@@ -81,7 +81,7 @@ struct mtmd_cli_context {
llama_context * lctx;
const llama_vocab * vocab;
common_sampler * smpl;
llama_batch batch;
common_batch batch;
int n_batch;
mtmd::bitmaps bitmaps;
@@ -115,7 +115,7 @@ struct mtmd_cli_context {
vocab = llama_model_get_vocab(model);
smpl = common_sampler_init(model, params.sampling);
n_threads = params.cpuparams.n_threads;
batch = llama_batch_init(1, 0, 1); // batch for next token generation
batch = common_batch(lctx); // batch for next token generation
n_batch = params.n_batch;
init_vision_context(params);
@@ -148,7 +148,6 @@ struct mtmd_cli_context {
}
~mtmd_cli_context() {
llama_batch_free(batch);
common_sampler_free(smpl);
}
@@ -230,9 +229,9 @@ static int generate_response(mtmd_cli_context & ctx, int n_predict) {
}
// eval the token
common_batch_clear(ctx.batch);
common_batch_add(ctx.batch, token_id, ctx.n_past++, {0}, true);
if (llama_decode(ctx.lctx, ctx.batch)) {
ctx.batch.clear();
ctx.batch.add(token_id, ctx.n_past++, 0, true);
if (llama_process(ctx.lctx, LLAMA_PROCESS_TYPE_DECODE, ctx.batch.get())) {
LOG_ERR("failed to decode token\n");
return 1;
}