spec: properly handle mtmd input for mtp (#30257)

* spec: properly handle mtmd input for mtp

* nits
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-10-10 11:22:38 +02:00
1 parent 0fd868cbca
commit 781dbc5ac9
27 files changed
+248 -99

No files matched your search

+83 -15
View File
@@ -31,9 +31,10 @@ bool llama_batch_allocr::init(
bool output_all) {
clear();
this->vocab = &vocab;
this->n_embd = batch_inp.n_embd > 0 ? batch_inp.n_embd : batch_inp.n_embd_inp;
this->n_seq_max = batch_inp.n_seq_max;
this->vocab = &vocab;
this->n_embd = batch_inp.n_embd > 0 ? batch_inp.n_embd : batch_inp.n_embd_inp;
this->n_embd_state = batch_inp.n_embd_state;
this->n_seq_max = batch_inp.n_seq_max;
const int32_t n_tok = (int32_t) batch_inp.tokens.size();
@@ -48,14 +49,17 @@ bool llama_batch_allocr::init(
//
// determine the content types of the batch
// an entry can carry a token id, a token embedding, or both (e.g. MTP hook batches)
// an entry can carry a token id, a token embedding, or both
// all entries must carry the same combination, or be a mix of token and embd entries
// a state embedding (e.g. MTP hook batches) is set on all entries or on none
//
int32_t n_tok_only = 0;
int32_t n_embd_only = 0;
int32_t n_both = 0;
const bool has_state = batch_inp.tokens[0].has_state;
for (int32_t i = 0; i < n_tok; ++i) {
const bool is_tok = batch_inp.tokens[i].id != LLAMA_TOKEN_NULL;
const bool is_emb = batch_inp.tokens[i].has_embd;
@@ -65,6 +69,11 @@ bool llama_batch_allocr::init(
return false;
}
if (batch_inp.tokens[i].has_state != has_state) {
LLAMA_LOG_ERROR("%s: all entries in the batch must have the same state embedding presence\n", __func__);
return false;
}
n_tok_only += is_tok && !is_emb;
n_embd_only += is_emb && !is_tok;
n_both += is_tok && is_emb;
@@ -124,6 +133,10 @@ bool llama_batch_allocr::init(
embd_vec = batch_inp.embd;
}
if (has_state) {
state_vec = batch_inp.state;
}
//
// build flat pos array, section-major: pos[j*n_tok + i] = section j of entry i
// token entry: [p, p, p, 0] (M-RoPE text position)
@@ -292,6 +305,7 @@ bool llama_batch_allocr::init(
/*.n_pos =*/ n_pos_per_embd,
/*.token =*/ batch.token,
/*.embd =*/ batch.embd,
/*.embd_state =*/ state_vec.empty() ? nullptr : state_vec.data(),
/*.pos =*/ batch.pos,
/*.n_seq_id =*/ batch.n_seq_id,
/*.seq_id =*/ batch.seq_id,
@@ -493,6 +507,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
udata->token .resize(n_tokens);
udata->embd .clear();
udata->embd_state.clear();
udata->pos .resize(n_pos_all);
udata->n_seq_id .resize(n_tokens);
udata->seq_id .resize(n_tokens);
@@ -515,6 +530,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
/*.token =*/ udata->token.data(),
/*.embd =*/ nullptr,
/*.embd_state =*/ nullptr,
/*.pos =*/ udata->pos.data(),
/*.n_seq_id =*/ udata->n_seq_id.data(),
/*.seq_id =*/ udata->seq_id.data(),
@@ -821,6 +837,7 @@ void llama_batch_allocr::clear() {
token_vec .clear();
embd_vec .clear();
is_embd_vec .clear();
state_vec .clear();
seq_id_data .clear();
pos .clear();
n_seq_id .clear();
@@ -863,12 +880,15 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
const bool mixed = mixed_batch && n_embd_rows > 0 && n_embd_rows < n_tokens;
const bool use_token = batch.token && !(mixed_batch && n_embd_rows == n_tokens);
const bool use_embd = batch.embd && !(mixed_batch && n_embd_rows == 0);
const bool has_state = !state_vec.empty();
const int64_t n_embd_all = use_embd ? (int64_t) n_tokens*n_embd : 0;
const int64_t n_pos_all = (int64_t) n_tokens*n_pos_per_embd;
const int64_t n_embd_all = use_embd ? (int64_t) n_tokens*n_embd : 0;
const int64_t n_state_all = has_state ? (int64_t) n_tokens*n_embd_state : 0;
const int64_t n_pos_all = (int64_t) n_tokens*n_pos_per_embd;
udata->token .resize(n_tokens);
udata->embd .resize(n_embd_all);
udata->embd_state.resize(n_state_all);
udata->pos .resize(n_pos_all);
udata->n_seq_id .resize(n_tokens);
udata->seq_id .resize(n_tokens);
@@ -896,6 +916,10 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
udata->type[i] = is_embd_vec[idxs[i]];
}
if (has_state) {
memcpy(udata->embd_state.data() + i*n_embd_state, state_vec.data() + (int64_t) idxs[i]*n_embd_state, n_embd_state*sizeof(float));
}
for (size_t j = 0; j < (size_t)n_pos_per_embd; ++j) {
udata->pos[j*n_tokens + i] = batch.pos[j*batch.n_tokens + idxs[i]];
}
@@ -942,6 +966,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
/*.token =*/ use_token ? udata->token.data() : nullptr,
/*.embd =*/ use_embd ? udata->embd.data() : nullptr,
/*.embd_state =*/ has_state ? udata->embd_state.data() : nullptr,
/*.pos =*/ udata->pos.data(),
/*.n_seq_id =*/ udata->n_seq_id.data(),
/*.seq_id =*/ udata->seq_id.data(),
@@ -993,6 +1018,7 @@ void llama_batch_allocr::ubatch_print(const llama_ubatch & ubatch, int debug) {
LLAMA_LOG_DEBUG("%s: token = %p\n", __func__, (void *) ubatch.token);
LLAMA_LOG_DEBUG("%s: embd = %p\n", __func__, (void *) ubatch.embd);
LLAMA_LOG_DEBUG("%s: embd_state = %p\n", __func__, (void *) ubatch.embd_state);
LLAMA_LOG_DEBUG("%s: pos = %p\n", __func__, (void *) ubatch.pos);
LLAMA_LOG_DEBUG("%s: n_seq_id = %p\n", __func__, (void *) ubatch.n_seq_id);
LLAMA_LOG_DEBUG("%s: seq_id = %p\n", __func__, (void *) ubatch.seq_id);
@@ -1110,19 +1136,25 @@ void llama_batch_free(struct llama_batch batch) {
// llama_batch_ext
size_t llama_batch_ext_select_n_embd_inp(llama_context_type ctx_type, llm_arch arch, const llama_hparams & hparams) {
if (ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
return hparams.n_embd_out();
}
GGML_UNUSED(ctx_type);
if (arch == LLM_ARCH_DFLASH) {
return hparams.n_embd_inp_enc();
}
return hparams.n_embd_inp();
}
size_t llama_batch_ext_select_n_embd_state(llama_context_type ctx_type, const llama_hparams & hparams) {
if (ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
return hparams.n_embd_out();
}
return 0;
}
llama_batch_ext::llama_batch_ext(llama_context * ctx) :
n_tokens_max(llama_n_batch(ctx)),
n_embd_inp(llama_batch_ext_select_n_embd_inp(ctx->get_cparams().ctx_type, llama_get_model(ctx)->arch, llama_get_model(ctx)->hparams)),
n_embd_inp_enc(llama_get_model(ctx)->hparams.n_embd_inp_enc()),
n_embd_state(llama_batch_ext_select_n_embd_state(ctx->get_cparams().ctx_type, llama_get_model(ctx)->hparams)),
n_seq_max(llama_n_seq_max(ctx)),
mem(llama_get_memory(ctx)),
n_vocab(llama_vocab_n_tokens(llama_model_get_vocab(llama_get_model(ctx)))),
@@ -1141,6 +1173,7 @@ llama_batch_ext::llama_batch_ext(
n_tokens_max(n_tokens_max),
n_embd_inp(n_embd_inp),
n_embd_inp_enc(n_embd_inp_enc),
n_embd_state(0),
n_seq_max(n_seq_max),
mem(mem),
n_vocab(n_vocab),
@@ -1151,6 +1184,7 @@ llama_batch_ext::llama_batch_ext(
void llama_batch_ext::clear() {
tokens.clear();
embd .clear();
state .clear();
n_embd = 0;
}
@@ -1233,6 +1267,38 @@ bool llama_batch_ext::set_token_embd(int32_t idx, llama_embd embd_in) {
return true;
}
bool llama_batch_ext::set_token_state(int32_t idx, llama_embd state_in) {
if (idx < 0 || idx >= (int32_t) tokens.size()) {
return false;
}
if (!state_in.data) {
return false;
}
if (n_embd_state == 0) {
return false; // this context does not take state embeddings
}
const size_t n_total = state_in.n_rows * state_in.n_embd;
if (n_total != n_embd_state) {
LLAMA_LOG_ERROR("%s: state size mismatch, got %zu rows x %zu = %zu, expected %zu\n",
__func__, state_in.n_rows, state_in.n_embd, n_total, n_embd_state);
return false;
}
token & t = tokens[idx];
if (t.has_state) {
LLAMA_LOG_ERROR("%s: state for token %d is already set\n", __func__, idx);
return false;
}
t.has_state = true;
t.state_off = state.size();
state.insert(state.end(), state_in.data, state_in.data + n_total);
return true;
}
bool llama_batch_ext::set_token_pos(int32_t idx, const llama_pos * pos_in) {
if (idx < 0 || idx >= (int32_t) tokens.size()) {
return false;
@@ -1320,11 +1386,7 @@ bool llama_batch_ext_set_embd_token(llama_batch_ext * batch, int32_t idx, llama_
}
bool llama_batch_ext_set_embd_state(llama_batch_ext * batch, int32_t idx, llama_embd embd) {
// TODO
GGML_UNUSED(batch);
GGML_UNUSED(idx);
GGML_UNUSED(embd);
return false;
return batch->set_token_state(idx, embd);
}
bool llama_batch_ext_set_output_embd(llama_batch_ext * batch, int32_t idx, bool value) {
@@ -1393,7 +1455,13 @@ void llama_batch_compat::init(llama_batch_ext & dst, const llama_batch & batch_i
t.id = batch_inp.token[i];
}
if (has_embd) {
// legacy MTP hook batches carry the hidden state next to the token ids
if (has_embd && has_token && batch_ext->n_embd_state > 0) {
t.has_state = true;
t.state_off = batch_ext->state.size();
const float * src = batch_inp.embd + (size_t) i * batch_ext->n_embd_state;
batch_ext->state.insert(batch_ext->state.end(), src, src + batch_ext->n_embd_state);
} else if (has_embd) {
t.has_embd = true;
t.embd_off = batch_ext->embd.size();
const float * src = batch_inp.embd + (size_t) i * n_embd_row;