batch: migrate speculative, mtmd and server to batch_ext (#29385)

* adapt common

* add common_batch

* wip

* wip: spec

* cont

* common_speculative_process

* server_batch to use common_batch

* rm some stale calls

Assisted-by: Claude Fable 5.1

* migrate mtmd

* handle imrope, handle return val of add()/add_embd()

* add spec zeros vector

* add warning on zero fill path
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-09-28 19:52:45 +02:00
1 parent 6c7a87f7e5
commit f1ea206218
11 files changed
+544 -456

No files matched your search

+137 -51
View File
@@ -613,34 +613,6 @@ std::string string_from(const struct llama_context * ctx, const std::vector<llam
return buf.str();
}
std::string string_from(const struct llama_context * ctx, const struct llama_batch & batch) {
std::stringstream buf;
buf << "[ ";
bool first = true;
for (int i = 0; i < batch.n_tokens; ++i) {
if (!first) {
buf << ", ";
} else {
first = false;
}
auto detokenized = common_token_to_piece(ctx, batch.token[i]);
buf << "\n" << std::to_string(i)
<< ", token '" << detokenized << "'"
<< ", pos " << std::to_string(batch.pos[i])
<< ", n_seq_id " << std::to_string(batch.n_seq_id[i])
<< ", seq_id " << std::to_string(batch.seq_id[i][0])
<< ", logits " << std::to_string(batch.logits[i]);
}
buf << " ]";
return buf.str();
}
void string_process_escapes(std::string & input) {
std::size_t input_len = input.length();
std::size_t output_idx = 0;
@@ -1491,7 +1463,8 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
}
if (llama_model_has_encoder(model)) {
llama_encode(lctx, llama_batch_get_one(tmp.data(), tmp.size()));
common_batch batch = common_batch_get_one(lctx, tmp);
llama_process(lctx, LLAMA_PROCESS_TYPE_ENCODE, batch.get());
llama_token decoder_start_token_id = llama_model_decoder_start_token(model);
if (decoder_start_token_id == LLAMA_TOKEN_NULL) {
decoder_start_token_id = bos;
@@ -1500,7 +1473,9 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
tmp.push_back(decoder_start_token_id);
}
if (llama_model_has_decoder(model)) {
llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch)));
tmp.resize(std::min(tmp.size(), (size_t) params.n_batch));
common_batch batch = common_batch_get_one(lctx, tmp);
llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
}
llama_memory_clear(llama_get_memory(lctx), true);
llama_synchronize(lctx);
@@ -1564,9 +1539,13 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) {
tmp.push_back(0);
tmp.push_back(0);
int ret = llama_decode(ctx, llama_batch_get_one(tmp.data(), tmp.size()));
int ret;
{
common_batch batch = common_batch_get_one(ctx, tmp);
ret = llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
}
if (ret != 0) {
COM_ERR("llama_decode() failed: %d\n", ret);
COM_ERR("llama_process() failed: %d\n", ret);
res = COMMON_CONTEXT_SEQ_RM_TYPE_NO;
goto done;
}
@@ -2153,29 +2132,138 @@ float lr_opt::get_lr(float epoch) const {
}
bool common_replay_last_token(struct llama_context * ctx, llama_token last_token, int32_t pos) {
llama_batch batch = llama_batch_get_one(&last_token, 1);
batch.pos = &pos;
if (llama_decode(ctx, batch)) {
common_batch batch(ctx);
batch.add(last_token, pos, 0, true);
if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
LOG_ERR("%s: failed to replay last token\n", __func__);
return false;
}
return true;
}
llama_batch_ext_ptr common_batch_ext_get_one(llama_context * ctx, const llama_tokens & tokens) {
llama_batch_ext_ptr batch(llama_batch_ext_init(ctx));
common_batch::common_batch(llama_context * ctx) : batch(llama_batch_ext_init(ctx)) {
const auto rope_type = llama_model_rope_type(llama_get_model(ctx));
n_pos = rope_type == LLAMA_ROPE_TYPE_MROPE || rope_type == LLAMA_ROPE_TYPE_IMROPE ? GGML_MROPE_SECTIONS : 1;
}
auto mem = llama_get_memory(ctx);
llama_pos pos = mem ? llama_memory_seq_pos_max(mem, 0) + 1 : 0;
void common_batch::clear() {
tokens.clear();
llama_batch_ext_clear(batch.get());
}
for (size_t i = 0; i < tokens.size(); ++i) {
const int32_t idx = llama_batch_ext_add_token(batch.get(), 0, tokens[i]);
llama_batch_ext_set_pos(batch.get(), idx, &pos);
pos++;
int32_t common_batch::add(llama_token id, llama_pos pos, llama_seq_id seq_id, bool output) {
const int32_t idx = llama_batch_ext_add_token(batch.get(), seq_id, id);
if (idx < 0) {
GGML_ABORT("%s: failed to add token %d to the batch (error %d, n_tokens = %d)\n", __func__, id, idx, size());
}
llama_batch_ext_set_pos(batch.get(), idx, &pos);
if (output) {
llama_batch_ext_set_output_logits(batch.get(), idx, true);
}
tokens.push_back({ id, { pos, 0, 0, 0 }, seq_id, output, { nullptr, 0, 0 } });
return idx;
}
bool common_batch::set_output(int32_t idx, bool value) {
if (idx < 0 || idx >= (int32_t) tokens.size()) {
return false;
}
tokens[idx].output = value;
return llama_batch_ext_set_output_logits(batch.get(), idx, value);
}
bool common_batch::set_embd(int32_t idx, llama_embd embd) {
if (idx < 0 || idx >= (int32_t) tokens.size()) {
return false;
}
if (!llama_batch_ext_set_embd_token(batch.get(), idx, embd)) {
return false;
}
tokens[idx].embd = embd;
return true;
}
int32_t common_batch::add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output) {
const int32_t idx = llama_batch_ext_add_embd(batch.get(), seq_id, embd);
if (idx < 0) {
GGML_ABORT("%s: failed to add embedding to the batch (error %d, n_tokens = %d)\n", __func__, idx, size());
}
llama_batch_ext_set_pos(batch.get(), idx, pos);
if (output) {
llama_batch_ext_set_output_logits(batch.get(), idx, true);
}
token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, embd };
for (int32_t j = 0; j < n_pos; ++j) {
t.pos[j] = pos[j];
}
tokens.push_back(t);
return idx;
}
common_batch common_batch_from_llama_batch(llama_context * ctx, const llama_batch & batch) {
common_batch res(ctx);
const bool has_token = batch.token != nullptr;
const bool has_embd = batch.embd != nullptr;
const size_t n_embd = llama_model_n_embd_inp(llama_get_model(ctx));
// positions continue from the memory when none are given
auto * mem = llama_get_memory(ctx);
std::vector<llama_pos> pos_next(llama_n_seq_max(ctx));
for (llama_seq_id s = 0; s < (llama_seq_id) pos_next.size(); ++s) {
pos_next[s] = llama_memory_seq_pos_max(mem, s) + 1;
}
if (!tokens.empty()) {
llama_batch_ext_set_output_logits(batch.get(), (int32_t) tokens.size() - 1, true);
for (int32_t i = 0; i < batch.n_tokens; ++i) {
const int32_t n_sid = batch.n_seq_id ? batch.n_seq_id[i] : 1;
const llama_seq_id seq_id = batch.seq_id ? batch.seq_id[i][0] : 0;
llama_pos pos[GGML_MROPE_SECTIONS] = { 0, 0, 0, 0 };
if (!batch.pos) {
pos[0] = pos_next[seq_id]++;
} else if (has_token) {
pos[0] = batch.pos[i];
} else {
// embedding batch: section-major layout pos[j*n_tokens + i]
for (int32_t j = 0; j < res.n_pos; ++j) {
pos[j] = batch.pos[j * batch.n_tokens + i];
}
}
const bool output = batch.logits ? batch.logits[i] != 0 : i == batch.n_tokens - 1;
const llama_embd embd = { has_embd ? batch.embd + (size_t) i * n_embd : nullptr, 1, n_embd };
int32_t idx;
if (has_token) {
idx = res.add(batch.token[i], pos[0], seq_id, output);
if (has_embd) {
res.set_embd(idx, embd);
}
} else {
idx = res.add_embd(embd, pos, seq_id, output);
}
for (int32_t s = 1; s < n_sid; ++s) {
llama_batch_ext_add_seq(res.get(), idx, batch.seq_id[i][s]);
}
}
return res;
}
common_batch common_batch_get_one(llama_context * ctx, const llama_tokens & tokens) {
common_batch batch(ctx);
auto mem = llama_get_memory(ctx);
llama_pos pos = llama_memory_seq_pos_max(mem, 0) + 1; // -1 + 1 == 0 when the memory is empty
for (size_t i = 0; i < tokens.size(); ++i) {
const bool output = i == tokens.size() - 1;
batch.add(tokens[i], pos, 0, output);
pos++;
}
return batch;
@@ -2205,7 +2293,7 @@ bool common_prompt_batch_decode(
// memory, so we can't just remove the last token from the memory and replay the last token which
// is the reason for this logic.
llama_tokens prefix_tokens(all_tokens.begin() + offset, all_tokens.begin() + offset + n_tokens_before_last);
llama_batch_ext_ptr batch_prefix = common_batch_ext_get_one(ctx, prefix_tokens);
common_batch batch_prefix = common_batch_get_one(ctx, prefix_tokens);
if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch_prefix.get())) {
COM_ERR("%s", "failed to eval\n");
return false;
@@ -2215,10 +2303,8 @@ bool common_prompt_batch_decode(
llama_state_save_file(ctx, state_path.data(), all_tokens.data(), all_tokens.size());
COM_INF("saved session before last token to %s, n_new = %zu\n", state_path.data(), all_tokens.size());
llama_token last_token = all_tokens.back();
llama_batch_ext_ptr batch_last = common_batch_ext_get_one(ctx, { last_token });
llama_pos pos = n_past;
llama_batch_ext_set_pos(batch_last.get(), 0, &pos);
common_batch batch_last(ctx);
batch_last.add(all_tokens.back(), n_past, 0, true);
if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch_last.get())) {
COM_ERR("%s", "failed to eval last token\n");
@@ -2227,7 +2313,7 @@ bool common_prompt_batch_decode(
n_past++;
} else {
llama_tokens new_tokens(all_tokens.begin() + offset, all_tokens.begin() + offset + n_new);
llama_batch_ext_ptr batch = common_batch_ext_get_one(ctx, new_tokens);
common_batch batch = common_batch_get_one(ctx, new_tokens);
if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
COM_ERR("%s", "failed to eval\n");
return false;