llama: support both embd + raw tokens in batch (#29622)

* llama: support both embd + raw tokens in batch

* add to test-llama-archs

* also check case llm_arch_supports_mixed_batch = false

* constant graph topology

* have dedicated input for mixed case

* rm set_tensor_backend

* is_embd --> type

* consolidate m-rope pos handling into one place

* nits
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-10-05 01:35:49 +02:00
1 parent 2ca15f5404
commit 0bb496dbd3
29 files changed
+506 -99

No files matched your search

+105 -34
View File
@@ -73,13 +73,55 @@ void llm_graph_input_embd::set_input(const llama_ubatch * ubatch) {
ggml_backend_tensor_set(tokens, ubatch->token, 0, n_tokens*ggml_element_size(tokens));
}
if (ubatch->embd) {
if (ubatch->embd && embd && !ubatch->is_mixed()) {
GGML_ASSERT(n_embd == embd->ne[0]);
const int64_t n_tokens = ubatch->n_tokens;
ggml_backend_tensor_set(embd, ubatch->embd, 0, n_tokens*n_embd*ggml_element_size(embd));
}
if (ubatch->is_mixed() && embd) {
GGML_ASSERT(mixed_tokens && mixed_slots && mixed_embd && "mixed token/embd ubatch is not supported here");
std::vector<int32_t> ids;
std::vector<int64_t> slots;
for (uint32_t i = 0; i < ubatch->n_tokens; ++i) {
if (!ubatch->type[i]) {
ids.push_back(ubatch->token[i]);
slots.push_back(i);
}
}
GGML_ASSERT((int64_t) ids.size() == mixed_tokens->ne[0]);
GGML_ASSERT(n_embd == mixed_embd->ne[0]);
ggml_backend_tensor_set(mixed_tokens, ids.data(), 0, ggml_nbytes(mixed_tokens));
ggml_backend_tensor_set(mixed_slots, slots.data(), 0, ggml_nbytes(mixed_slots));
ggml_backend_tensor_set(mixed_embd, ubatch->embd, 0, ggml_nbytes(mixed_embd));
}
if (scale_rows) {
const int64_t n_tokens = ubatch->n_tokens;
std::vector<float> data(n_tokens);
for (int64_t i = 0; i < n_tokens; ++i) {
const bool is_embd = !ubatch->token || (ubatch->is_mixed() && ubatch->type[i]);
data[i] = is_embd ? 1.0f : scale_tok;
}
ggml_backend_tensor_set(scale_rows, data.data(), 0, ggml_nbytes(scale_rows));
}
}
// number of token rows of the mixed path, a non-mixed ubatch is sized for the worst case
static int64_t llm_graph_n_tok_rows(const llama_ubatch & ubatch) {
if (!ubatch.is_mixed()) {
return ubatch.n_tokens;
}
int64_t n = 0;
for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {
n += !ubatch.type[i];
}
return n;
}
bool llm_graph_input_embd::can_reuse(const llm_graph_params & params) {
@@ -87,11 +129,16 @@ bool llm_graph_input_embd::can_reuse(const llm_graph_params & params) {
res &= (!params.ubatch.token) || (tokens && tokens->ne[0] == params.ubatch.n_tokens);
res &= (!params.ubatch.embd) || (embd && embd->ne[1] == params.ubatch.n_tokens);
res &= (!mixed_tokens) || mixed_tokens->ne[0] == llm_graph_n_tok_rows(params.ubatch);
res &= (!mixed_embd) || mixed_embd->ne[1] == params.ubatch.n_tokens;
res &= (!scale_rows) || scale_rows->ne[1] == params.ubatch.n_tokens;
return res;
}
void llm_graph_input_embd_h::set_input(const llama_ubatch * ubatch) {
ASSERT_EMBD_OR_TOKEN(*ubatch);
const int64_t n_tokens = ubatch->n_tokens;
if (ubatch->token) {
@@ -128,21 +175,7 @@ void llm_graph_input_pos::set_input(const llama_ubatch * ubatch) {
if (ubatch->pos && pos) {
const int64_t n_tokens = ubatch->n_tokens;
if (ubatch->token && n_pos_per_embd == 4) {
// in case we're using M-RoPE with text tokens, convert the 1D positions to 4D
// the 3 first dims are the same, and 4th dim is all 0
std::vector<llama_pos> pos_data(n_tokens*n_pos_per_embd);
// copy the first dimension
for (int i = 0; i < n_tokens; ++i) {
pos_data[ i] = ubatch->pos[i];
pos_data[ n_tokens + i] = ubatch->pos[i];
pos_data[2 * n_tokens + i] = ubatch->pos[i];
pos_data[3 * n_tokens + i] = 0; // 4th dim is 0
}
ggml_backend_tensor_set(pos, pos_data.data(), 0, pos_data.size()*ggml_element_size(pos));
} else {
ggml_backend_tensor_set(pos, ubatch->pos, 0, n_tokens*n_pos_per_embd*ggml_element_size(pos));
}
ggml_backend_tensor_set(pos, ubatch->pos, 0, n_tokens*n_pos_per_embd*ggml_element_size(pos));
}
}
@@ -2377,7 +2410,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
}
// input embeddings with optional lora
ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const {
ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float tok_scale) const {
const int64_t n_embd_inp = hparams.n_embd_inp();
const int64_t n_embd = hparams.n_embd;
@@ -2394,15 +2427,9 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const {
cb(inp->embd, "inp_embd", -1);
ggml_set_input(inp->embd);
// select one of the 2 inputs, based on the batch contents
// ref: https://github.com/ggml-org/llama.cpp/pull/18550
std::array<ggml_tensor *, 2> inps;
// token embeddings path (ubatch.token != nullptr)
{
auto & cur = inps[0];
cur = ggml_get_rows(ctx0, tok_embd, inp->tokens);
// token embeddings with lora and padding
auto build_tok = [&](ggml_tensor * ids) {
ggml_tensor * cur = ggml_get_rows(ctx0, tok_embd, ids);
// apply lora for embedding tokens if needed
for (const auto & lora : *loras) {
@@ -2416,7 +2443,7 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const {
ggml_tensor * inpL_delta = ggml_scale(ctx0, ggml_mul_mat(
ctx0, lw->b, // non-transposed lora_b
ggml_get_rows(ctx0, lw->a, inp->tokens)
ggml_get_rows(ctx0, lw->a, ids)
), scale);
cur = ggml_add(ctx0, cur, inpL_delta);
@@ -2425,19 +2452,48 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const {
if (n_embd_inp != n_embd) {
cur = ggml_pad(ctx0, cur, hparams.n_embd_inp() - n_embd, 0, 0, 0);
}
}
return cur;
};
// select one of the 3 inputs, based on the batch contents
// ref: https://github.com/ggml-org/llama.cpp/pull/18550
std::array<ggml_tensor *, 3> inps = {};
// token embeddings path (ubatch.token != nullptr)
inps[0] = build_tok(inp->tokens);
// vector embeddings path (ubatch.embd != nullptr)
{
auto & cur = inps[1];
inps[1] = inp->embd;
cur = inp->embd;
// mixed path (ubatch.is_mixed()): set_rows the token rows into a copy of the embd rows, with its own inputs as select branches must not share tensors
// TODO: use inp->tokens and inp->embd once ggml_build_forward_select allows it
const bool has_mixed = llm_arch_supports_mixed_batch(arch) && cparams.ctx_type == LLAMA_CONTEXT_TYPE_DEFAULT;
if (has_mixed) {
const int64_t n_tok_rows = llm_graph_n_tok_rows(ubatch);
inp->mixed_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tok_rows);
cb(inp->mixed_tokens, "inp_mixed_tokens", -1);
ggml_set_input(inp->mixed_tokens);
inp->mixed_slots = ggml_new_tensor_1d(ctx0, GGML_TYPE_I64, n_tok_rows);
cb(inp->mixed_slots, "inp_mixed_slots", -1);
ggml_set_input(inp->mixed_slots);
inp->mixed_embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, ubatch.n_tokens);
cb(inp->mixed_embd, "inp_mixed_embd", -1);
ggml_set_input(inp->mixed_embd);
// note: set_rows writes into its destination, so it gets a copy of the input
inps[2] = ggml_set_rows(ctx0, ggml_dup(ctx0, inp->mixed_embd), build_tok(inp->mixed_tokens), inp->mixed_slots);
}
assert(ggml_are_same_shape (inps[0], inps[1]));
assert(ggml_are_same_stride(inps[0], inps[1]));
ggml_tensor * cur = ggml_build_forward_select(gf, inps.data(), inps.size(), ubatch.token ? 0 : 1);
const int idx = ubatch.is_mixed() ? 2 : ubatch.token ? 0 : 1;
ggml_tensor * cur = ggml_build_forward_select(gf, inps.data(), has_mixed ? 3 : 2, idx);
if (n_embd_inp != n_embd) {
cur = ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0);
@@ -2445,16 +2501,31 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const {
res->t_inp_embd = cur;
// For Granite architecture
// NOTE: For deepstack models, only apply scale to token inputs (ie text-only input).
// Raw embeddings are assumed to be multimodal inputs that should not be scaled.
if (hparams.f_embedding_scale != 0.0f && (ubatch.token || hparams.n_deepstack_layers == 0)) {
const bool scale_tok_only = hparams.f_embedding_scale != 0.0f && hparams.n_deepstack_layers > 0;
// For Granite architecture
if (hparams.f_embedding_scale != 0.0f && !scale_tok_only) {
if (!ggml_is_contiguous(cur)) {
cur = ggml_cont(ctx0, cur);
}
cur = ggml_scale(ctx0, cur, hparams.f_embedding_scale);
}
// scale the token rows only, applied after the select so that the graph is the same for any batch contents
inp->scale_tok = tok_scale*(scale_tok_only ? hparams.f_embedding_scale : 1.0f);
if (inp->scale_tok != 1.0f) {
inp->scale_rows = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, ubatch.n_tokens);
cb(inp->scale_rows, "inp_scale_rows", -1);
ggml_set_input(inp->scale_rows);
if (!ggml_is_contiguous(cur)) {
cur = ggml_cont(ctx0, cur);
}
cur = ggml_mul(ctx0, cur, inp->scale_rows);
}
cb(cur, "embd", -1);
res->add_input(std::move(inp));