spec: properly handle mtmd input for mtp (#30257)
* spec: properly handle mtmd input for mtp * nits
This commit is contained in:
1 parent
0fd868cbca
commit
781dbc5ac9
27 files changed
+248
-99
No files matched your search
+83
-15
@@ -31,9 +31,10 @@ bool llama_batch_allocr::init(
|
||||
bool output_all) {
|
||||
clear();
|
||||
|
||||
this->vocab = &vocab;
|
||||
this->n_embd = batch_inp.n_embd > 0 ? batch_inp.n_embd : batch_inp.n_embd_inp;
|
||||
this->n_seq_max = batch_inp.n_seq_max;
|
||||
this->vocab = &vocab;
|
||||
this->n_embd = batch_inp.n_embd > 0 ? batch_inp.n_embd : batch_inp.n_embd_inp;
|
||||
this->n_embd_state = batch_inp.n_embd_state;
|
||||
this->n_seq_max = batch_inp.n_seq_max;
|
||||
|
||||
const int32_t n_tok = (int32_t) batch_inp.tokens.size();
|
||||
|
||||
@@ -48,14 +49,17 @@ bool llama_batch_allocr::init(
|
||||
|
||||
//
|
||||
// determine the content types of the batch
|
||||
// an entry can carry a token id, a token embedding, or both (e.g. MTP hook batches)
|
||||
// an entry can carry a token id, a token embedding, or both
|
||||
// all entries must carry the same combination, or be a mix of token and embd entries
|
||||
// a state embedding (e.g. MTP hook batches) is set on all entries or on none
|
||||
//
|
||||
|
||||
int32_t n_tok_only = 0;
|
||||
int32_t n_embd_only = 0;
|
||||
int32_t n_both = 0;
|
||||
|
||||
const bool has_state = batch_inp.tokens[0].has_state;
|
||||
|
||||
for (int32_t i = 0; i < n_tok; ++i) {
|
||||
const bool is_tok = batch_inp.tokens[i].id != LLAMA_TOKEN_NULL;
|
||||
const bool is_emb = batch_inp.tokens[i].has_embd;
|
||||
@@ -65,6 +69,11 @@ bool llama_batch_allocr::init(
|
||||
return false;
|
||||
}
|
||||
|
||||
if (batch_inp.tokens[i].has_state != has_state) {
|
||||
LLAMA_LOG_ERROR("%s: all entries in the batch must have the same state embedding presence\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
n_tok_only += is_tok && !is_emb;
|
||||
n_embd_only += is_emb && !is_tok;
|
||||
n_both += is_tok && is_emb;
|
||||
@@ -124,6 +133,10 @@ bool llama_batch_allocr::init(
|
||||
embd_vec = batch_inp.embd;
|
||||
}
|
||||
|
||||
if (has_state) {
|
||||
state_vec = batch_inp.state;
|
||||
}
|
||||
|
||||
//
|
||||
// build flat pos array, section-major: pos[j*n_tok + i] = section j of entry i
|
||||
// token entry: [p, p, p, 0] (M-RoPE text position)
|
||||
@@ -292,6 +305,7 @@ bool llama_batch_allocr::init(
|
||||
/*.n_pos =*/ n_pos_per_embd,
|
||||
/*.token =*/ batch.token,
|
||||
/*.embd =*/ batch.embd,
|
||||
/*.embd_state =*/ state_vec.empty() ? nullptr : state_vec.data(),
|
||||
/*.pos =*/ batch.pos,
|
||||
/*.n_seq_id =*/ batch.n_seq_id,
|
||||
/*.seq_id =*/ batch.seq_id,
|
||||
@@ -493,6 +507,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
|
||||
|
||||
udata->token .resize(n_tokens);
|
||||
udata->embd .clear();
|
||||
udata->embd_state.clear();
|
||||
udata->pos .resize(n_pos_all);
|
||||
udata->n_seq_id .resize(n_tokens);
|
||||
udata->seq_id .resize(n_tokens);
|
||||
@@ -515,6 +530,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
|
||||
|
||||
/*.token =*/ udata->token.data(),
|
||||
/*.embd =*/ nullptr,
|
||||
/*.embd_state =*/ nullptr,
|
||||
/*.pos =*/ udata->pos.data(),
|
||||
/*.n_seq_id =*/ udata->n_seq_id.data(),
|
||||
/*.seq_id =*/ udata->seq_id.data(),
|
||||
@@ -821,6 +837,7 @@ void llama_batch_allocr::clear() {
|
||||
token_vec .clear();
|
||||
embd_vec .clear();
|
||||
is_embd_vec .clear();
|
||||
state_vec .clear();
|
||||
seq_id_data .clear();
|
||||
pos .clear();
|
||||
n_seq_id .clear();
|
||||
@@ -863,12 +880,15 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
const bool mixed = mixed_batch && n_embd_rows > 0 && n_embd_rows < n_tokens;
|
||||
const bool use_token = batch.token && !(mixed_batch && n_embd_rows == n_tokens);
|
||||
const bool use_embd = batch.embd && !(mixed_batch && n_embd_rows == 0);
|
||||
const bool has_state = !state_vec.empty();
|
||||
|
||||
const int64_t n_embd_all = use_embd ? (int64_t) n_tokens*n_embd : 0;
|
||||
const int64_t n_pos_all = (int64_t) n_tokens*n_pos_per_embd;
|
||||
const int64_t n_embd_all = use_embd ? (int64_t) n_tokens*n_embd : 0;
|
||||
const int64_t n_state_all = has_state ? (int64_t) n_tokens*n_embd_state : 0;
|
||||
const int64_t n_pos_all = (int64_t) n_tokens*n_pos_per_embd;
|
||||
|
||||
udata->token .resize(n_tokens);
|
||||
udata->embd .resize(n_embd_all);
|
||||
udata->embd_state.resize(n_state_all);
|
||||
udata->pos .resize(n_pos_all);
|
||||
udata->n_seq_id .resize(n_tokens);
|
||||
udata->seq_id .resize(n_tokens);
|
||||
@@ -896,6 +916,10 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
udata->type[i] = is_embd_vec[idxs[i]];
|
||||
}
|
||||
|
||||
if (has_state) {
|
||||
memcpy(udata->embd_state.data() + i*n_embd_state, state_vec.data() + (int64_t) idxs[i]*n_embd_state, n_embd_state*sizeof(float));
|
||||
}
|
||||
|
||||
for (size_t j = 0; j < (size_t)n_pos_per_embd; ++j) {
|
||||
udata->pos[j*n_tokens + i] = batch.pos[j*batch.n_tokens + idxs[i]];
|
||||
}
|
||||
@@ -942,6 +966,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
|
||||
/*.token =*/ use_token ? udata->token.data() : nullptr,
|
||||
/*.embd =*/ use_embd ? udata->embd.data() : nullptr,
|
||||
/*.embd_state =*/ has_state ? udata->embd_state.data() : nullptr,
|
||||
/*.pos =*/ udata->pos.data(),
|
||||
/*.n_seq_id =*/ udata->n_seq_id.data(),
|
||||
/*.seq_id =*/ udata->seq_id.data(),
|
||||
@@ -993,6 +1018,7 @@ void llama_batch_allocr::ubatch_print(const llama_ubatch & ubatch, int debug) {
|
||||
|
||||
LLAMA_LOG_DEBUG("%s: token = %p\n", __func__, (void *) ubatch.token);
|
||||
LLAMA_LOG_DEBUG("%s: embd = %p\n", __func__, (void *) ubatch.embd);
|
||||
LLAMA_LOG_DEBUG("%s: embd_state = %p\n", __func__, (void *) ubatch.embd_state);
|
||||
LLAMA_LOG_DEBUG("%s: pos = %p\n", __func__, (void *) ubatch.pos);
|
||||
LLAMA_LOG_DEBUG("%s: n_seq_id = %p\n", __func__, (void *) ubatch.n_seq_id);
|
||||
LLAMA_LOG_DEBUG("%s: seq_id = %p\n", __func__, (void *) ubatch.seq_id);
|
||||
@@ -1110,19 +1136,25 @@ void llama_batch_free(struct llama_batch batch) {
|
||||
// llama_batch_ext
|
||||
|
||||
size_t llama_batch_ext_select_n_embd_inp(llama_context_type ctx_type, llm_arch arch, const llama_hparams & hparams) {
|
||||
if (ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
|
||||
return hparams.n_embd_out();
|
||||
}
|
||||
GGML_UNUSED(ctx_type);
|
||||
if (arch == LLM_ARCH_DFLASH) {
|
||||
return hparams.n_embd_inp_enc();
|
||||
}
|
||||
return hparams.n_embd_inp();
|
||||
}
|
||||
|
||||
size_t llama_batch_ext_select_n_embd_state(llama_context_type ctx_type, const llama_hparams & hparams) {
|
||||
if (ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
|
||||
return hparams.n_embd_out();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
llama_batch_ext::llama_batch_ext(llama_context * ctx) :
|
||||
n_tokens_max(llama_n_batch(ctx)),
|
||||
n_embd_inp(llama_batch_ext_select_n_embd_inp(ctx->get_cparams().ctx_type, llama_get_model(ctx)->arch, llama_get_model(ctx)->hparams)),
|
||||
n_embd_inp_enc(llama_get_model(ctx)->hparams.n_embd_inp_enc()),
|
||||
n_embd_state(llama_batch_ext_select_n_embd_state(ctx->get_cparams().ctx_type, llama_get_model(ctx)->hparams)),
|
||||
n_seq_max(llama_n_seq_max(ctx)),
|
||||
mem(llama_get_memory(ctx)),
|
||||
n_vocab(llama_vocab_n_tokens(llama_model_get_vocab(llama_get_model(ctx)))),
|
||||
@@ -1141,6 +1173,7 @@ llama_batch_ext::llama_batch_ext(
|
||||
n_tokens_max(n_tokens_max),
|
||||
n_embd_inp(n_embd_inp),
|
||||
n_embd_inp_enc(n_embd_inp_enc),
|
||||
n_embd_state(0),
|
||||
n_seq_max(n_seq_max),
|
||||
mem(mem),
|
||||
n_vocab(n_vocab),
|
||||
@@ -1151,6 +1184,7 @@ llama_batch_ext::llama_batch_ext(
|
||||
void llama_batch_ext::clear() {
|
||||
tokens.clear();
|
||||
embd .clear();
|
||||
state .clear();
|
||||
n_embd = 0;
|
||||
}
|
||||
|
||||
@@ -1233,6 +1267,38 @@ bool llama_batch_ext::set_token_embd(int32_t idx, llama_embd embd_in) {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool llama_batch_ext::set_token_state(int32_t idx, llama_embd state_in) {
|
||||
if (idx < 0 || idx >= (int32_t) tokens.size()) {
|
||||
return false;
|
||||
}
|
||||
if (!state_in.data) {
|
||||
return false;
|
||||
}
|
||||
if (n_embd_state == 0) {
|
||||
return false; // this context does not take state embeddings
|
||||
}
|
||||
|
||||
const size_t n_total = state_in.n_rows * state_in.n_embd;
|
||||
if (n_total != n_embd_state) {
|
||||
LLAMA_LOG_ERROR("%s: state size mismatch, got %zu rows x %zu = %zu, expected %zu\n",
|
||||
__func__, state_in.n_rows, state_in.n_embd, n_total, n_embd_state);
|
||||
return false;
|
||||
}
|
||||
|
||||
token & t = tokens[idx];
|
||||
|
||||
if (t.has_state) {
|
||||
LLAMA_LOG_ERROR("%s: state for token %d is already set\n", __func__, idx);
|
||||
return false;
|
||||
}
|
||||
|
||||
t.has_state = true;
|
||||
t.state_off = state.size();
|
||||
state.insert(state.end(), state_in.data, state_in.data + n_total);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool llama_batch_ext::set_token_pos(int32_t idx, const llama_pos * pos_in) {
|
||||
if (idx < 0 || idx >= (int32_t) tokens.size()) {
|
||||
return false;
|
||||
@@ -1320,11 +1386,7 @@ bool llama_batch_ext_set_embd_token(llama_batch_ext * batch, int32_t idx, llama_
|
||||
}
|
||||
|
||||
bool llama_batch_ext_set_embd_state(llama_batch_ext * batch, int32_t idx, llama_embd embd) {
|
||||
// TODO
|
||||
GGML_UNUSED(batch);
|
||||
GGML_UNUSED(idx);
|
||||
GGML_UNUSED(embd);
|
||||
return false;
|
||||
return batch->set_token_state(idx, embd);
|
||||
}
|
||||
|
||||
bool llama_batch_ext_set_output_embd(llama_batch_ext * batch, int32_t idx, bool value) {
|
||||
@@ -1393,7 +1455,13 @@ void llama_batch_compat::init(llama_batch_ext & dst, const llama_batch & batch_i
|
||||
t.id = batch_inp.token[i];
|
||||
}
|
||||
|
||||
if (has_embd) {
|
||||
// legacy MTP hook batches carry the hidden state next to the token ids
|
||||
if (has_embd && has_token && batch_ext->n_embd_state > 0) {
|
||||
t.has_state = true;
|
||||
t.state_off = batch_ext->state.size();
|
||||
const float * src = batch_inp.embd + (size_t) i * batch_ext->n_embd_state;
|
||||
batch_ext->state.insert(batch_ext->state.end(), src, src + batch_ext->n_embd_state);
|
||||
} else if (has_embd) {
|
||||
t.has_embd = true;
|
||||
t.embd_off = batch_ext->embd.size();
|
||||
const float * src = batch_inp.embd + (size_t) i * n_embd_row;
|
||||
|
||||
Reference in new issue
Block a user