llama: support both embd + raw tokens in batch (#29622)
* llama: support both embd + raw tokens in batch * add to test-llama-archs * also check case llm_arch_supports_mixed_batch = false * constant graph topology * have dedicated input for mixed case * rm set_tensor_backend * is_embd --> type * consolidate m-rope pos handling into one place * nits
This commit is contained in:
1 parent
2ca15f5404
commit
0bb496dbd3
29 files changed
+506
-99
No files matched your search
+105
-34
@@ -73,13 +73,55 @@ void llm_graph_input_embd::set_input(const llama_ubatch * ubatch) {
|
||||
ggml_backend_tensor_set(tokens, ubatch->token, 0, n_tokens*ggml_element_size(tokens));
|
||||
}
|
||||
|
||||
if (ubatch->embd) {
|
||||
if (ubatch->embd && embd && !ubatch->is_mixed()) {
|
||||
GGML_ASSERT(n_embd == embd->ne[0]);
|
||||
|
||||
const int64_t n_tokens = ubatch->n_tokens;
|
||||
|
||||
ggml_backend_tensor_set(embd, ubatch->embd, 0, n_tokens*n_embd*ggml_element_size(embd));
|
||||
}
|
||||
|
||||
if (ubatch->is_mixed() && embd) {
|
||||
GGML_ASSERT(mixed_tokens && mixed_slots && mixed_embd && "mixed token/embd ubatch is not supported here");
|
||||
|
||||
std::vector<int32_t> ids;
|
||||
std::vector<int64_t> slots;
|
||||
for (uint32_t i = 0; i < ubatch->n_tokens; ++i) {
|
||||
if (!ubatch->type[i]) {
|
||||
ids.push_back(ubatch->token[i]);
|
||||
slots.push_back(i);
|
||||
}
|
||||
}
|
||||
GGML_ASSERT((int64_t) ids.size() == mixed_tokens->ne[0]);
|
||||
GGML_ASSERT(n_embd == mixed_embd->ne[0]);
|
||||
|
||||
ggml_backend_tensor_set(mixed_tokens, ids.data(), 0, ggml_nbytes(mixed_tokens));
|
||||
ggml_backend_tensor_set(mixed_slots, slots.data(), 0, ggml_nbytes(mixed_slots));
|
||||
ggml_backend_tensor_set(mixed_embd, ubatch->embd, 0, ggml_nbytes(mixed_embd));
|
||||
}
|
||||
|
||||
if (scale_rows) {
|
||||
const int64_t n_tokens = ubatch->n_tokens;
|
||||
|
||||
std::vector<float> data(n_tokens);
|
||||
for (int64_t i = 0; i < n_tokens; ++i) {
|
||||
const bool is_embd = !ubatch->token || (ubatch->is_mixed() && ubatch->type[i]);
|
||||
data[i] = is_embd ? 1.0f : scale_tok;
|
||||
}
|
||||
ggml_backend_tensor_set(scale_rows, data.data(), 0, ggml_nbytes(scale_rows));
|
||||
}
|
||||
}
|
||||
|
||||
// number of token rows of the mixed path, a non-mixed ubatch is sized for the worst case
|
||||
static int64_t llm_graph_n_tok_rows(const llama_ubatch & ubatch) {
|
||||
if (!ubatch.is_mixed()) {
|
||||
return ubatch.n_tokens;
|
||||
}
|
||||
int64_t n = 0;
|
||||
for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {
|
||||
n += !ubatch.type[i];
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
bool llm_graph_input_embd::can_reuse(const llm_graph_params & params) {
|
||||
@@ -87,11 +129,16 @@ bool llm_graph_input_embd::can_reuse(const llm_graph_params & params) {
|
||||
|
||||
res &= (!params.ubatch.token) || (tokens && tokens->ne[0] == params.ubatch.n_tokens);
|
||||
res &= (!params.ubatch.embd) || (embd && embd->ne[1] == params.ubatch.n_tokens);
|
||||
res &= (!mixed_tokens) || mixed_tokens->ne[0] == llm_graph_n_tok_rows(params.ubatch);
|
||||
res &= (!mixed_embd) || mixed_embd->ne[1] == params.ubatch.n_tokens;
|
||||
res &= (!scale_rows) || scale_rows->ne[1] == params.ubatch.n_tokens;
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
void llm_graph_input_embd_h::set_input(const llama_ubatch * ubatch) {
|
||||
ASSERT_EMBD_OR_TOKEN(*ubatch);
|
||||
|
||||
const int64_t n_tokens = ubatch->n_tokens;
|
||||
|
||||
if (ubatch->token) {
|
||||
@@ -128,21 +175,7 @@ void llm_graph_input_pos::set_input(const llama_ubatch * ubatch) {
|
||||
if (ubatch->pos && pos) {
|
||||
const int64_t n_tokens = ubatch->n_tokens;
|
||||
|
||||
if (ubatch->token && n_pos_per_embd == 4) {
|
||||
// in case we're using M-RoPE with text tokens, convert the 1D positions to 4D
|
||||
// the 3 first dims are the same, and 4th dim is all 0
|
||||
std::vector<llama_pos> pos_data(n_tokens*n_pos_per_embd);
|
||||
// copy the first dimension
|
||||
for (int i = 0; i < n_tokens; ++i) {
|
||||
pos_data[ i] = ubatch->pos[i];
|
||||
pos_data[ n_tokens + i] = ubatch->pos[i];
|
||||
pos_data[2 * n_tokens + i] = ubatch->pos[i];
|
||||
pos_data[3 * n_tokens + i] = 0; // 4th dim is 0
|
||||
}
|
||||
ggml_backend_tensor_set(pos, pos_data.data(), 0, pos_data.size()*ggml_element_size(pos));
|
||||
} else {
|
||||
ggml_backend_tensor_set(pos, ubatch->pos, 0, n_tokens*n_pos_per_embd*ggml_element_size(pos));
|
||||
}
|
||||
ggml_backend_tensor_set(pos, ubatch->pos, 0, n_tokens*n_pos_per_embd*ggml_element_size(pos));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2377,7 +2410,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
}
|
||||
|
||||
// input embeddings with optional lora
|
||||
ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const {
|
||||
ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float tok_scale) const {
|
||||
const int64_t n_embd_inp = hparams.n_embd_inp();
|
||||
const int64_t n_embd = hparams.n_embd;
|
||||
|
||||
@@ -2394,15 +2427,9 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const {
|
||||
cb(inp->embd, "inp_embd", -1);
|
||||
ggml_set_input(inp->embd);
|
||||
|
||||
// select one of the 2 inputs, based on the batch contents
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/18550
|
||||
std::array<ggml_tensor *, 2> inps;
|
||||
|
||||
// token embeddings path (ubatch.token != nullptr)
|
||||
{
|
||||
auto & cur = inps[0];
|
||||
|
||||
cur = ggml_get_rows(ctx0, tok_embd, inp->tokens);
|
||||
// token embeddings with lora and padding
|
||||
auto build_tok = [&](ggml_tensor * ids) {
|
||||
ggml_tensor * cur = ggml_get_rows(ctx0, tok_embd, ids);
|
||||
|
||||
// apply lora for embedding tokens if needed
|
||||
for (const auto & lora : *loras) {
|
||||
@@ -2416,7 +2443,7 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const {
|
||||
|
||||
ggml_tensor * inpL_delta = ggml_scale(ctx0, ggml_mul_mat(
|
||||
ctx0, lw->b, // non-transposed lora_b
|
||||
ggml_get_rows(ctx0, lw->a, inp->tokens)
|
||||
ggml_get_rows(ctx0, lw->a, ids)
|
||||
), scale);
|
||||
|
||||
cur = ggml_add(ctx0, cur, inpL_delta);
|
||||
@@ -2425,19 +2452,48 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const {
|
||||
if (n_embd_inp != n_embd) {
|
||||
cur = ggml_pad(ctx0, cur, hparams.n_embd_inp() - n_embd, 0, 0, 0);
|
||||
}
|
||||
}
|
||||
|
||||
return cur;
|
||||
};
|
||||
|
||||
// select one of the 3 inputs, based on the batch contents
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/18550
|
||||
std::array<ggml_tensor *, 3> inps = {};
|
||||
|
||||
// token embeddings path (ubatch.token != nullptr)
|
||||
inps[0] = build_tok(inp->tokens);
|
||||
|
||||
// vector embeddings path (ubatch.embd != nullptr)
|
||||
{
|
||||
auto & cur = inps[1];
|
||||
inps[1] = inp->embd;
|
||||
|
||||
cur = inp->embd;
|
||||
// mixed path (ubatch.is_mixed()): set_rows the token rows into a copy of the embd rows, with its own inputs as select branches must not share tensors
|
||||
// TODO: use inp->tokens and inp->embd once ggml_build_forward_select allows it
|
||||
const bool has_mixed = llm_arch_supports_mixed_batch(arch) && cparams.ctx_type == LLAMA_CONTEXT_TYPE_DEFAULT;
|
||||
if (has_mixed) {
|
||||
const int64_t n_tok_rows = llm_graph_n_tok_rows(ubatch);
|
||||
|
||||
inp->mixed_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tok_rows);
|
||||
cb(inp->mixed_tokens, "inp_mixed_tokens", -1);
|
||||
ggml_set_input(inp->mixed_tokens);
|
||||
|
||||
inp->mixed_slots = ggml_new_tensor_1d(ctx0, GGML_TYPE_I64, n_tok_rows);
|
||||
cb(inp->mixed_slots, "inp_mixed_slots", -1);
|
||||
ggml_set_input(inp->mixed_slots);
|
||||
|
||||
inp->mixed_embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, ubatch.n_tokens);
|
||||
cb(inp->mixed_embd, "inp_mixed_embd", -1);
|
||||
ggml_set_input(inp->mixed_embd);
|
||||
|
||||
// note: set_rows writes into its destination, so it gets a copy of the input
|
||||
inps[2] = ggml_set_rows(ctx0, ggml_dup(ctx0, inp->mixed_embd), build_tok(inp->mixed_tokens), inp->mixed_slots);
|
||||
}
|
||||
|
||||
assert(ggml_are_same_shape (inps[0], inps[1]));
|
||||
assert(ggml_are_same_stride(inps[0], inps[1]));
|
||||
|
||||
ggml_tensor * cur = ggml_build_forward_select(gf, inps.data(), inps.size(), ubatch.token ? 0 : 1);
|
||||
const int idx = ubatch.is_mixed() ? 2 : ubatch.token ? 0 : 1;
|
||||
|
||||
ggml_tensor * cur = ggml_build_forward_select(gf, inps.data(), has_mixed ? 3 : 2, idx);
|
||||
|
||||
if (n_embd_inp != n_embd) {
|
||||
cur = ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0);
|
||||
@@ -2445,16 +2501,31 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const {
|
||||
|
||||
res->t_inp_embd = cur;
|
||||
|
||||
// For Granite architecture
|
||||
// NOTE: For deepstack models, only apply scale to token inputs (ie text-only input).
|
||||
// Raw embeddings are assumed to be multimodal inputs that should not be scaled.
|
||||
if (hparams.f_embedding_scale != 0.0f && (ubatch.token || hparams.n_deepstack_layers == 0)) {
|
||||
const bool scale_tok_only = hparams.f_embedding_scale != 0.0f && hparams.n_deepstack_layers > 0;
|
||||
|
||||
// For Granite architecture
|
||||
if (hparams.f_embedding_scale != 0.0f && !scale_tok_only) {
|
||||
if (!ggml_is_contiguous(cur)) {
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
}
|
||||
cur = ggml_scale(ctx0, cur, hparams.f_embedding_scale);
|
||||
}
|
||||
|
||||
// scale the token rows only, applied after the select so that the graph is the same for any batch contents
|
||||
inp->scale_tok = tok_scale*(scale_tok_only ? hparams.f_embedding_scale : 1.0f);
|
||||
if (inp->scale_tok != 1.0f) {
|
||||
inp->scale_rows = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, ubatch.n_tokens);
|
||||
cb(inp->scale_rows, "inp_scale_rows", -1);
|
||||
ggml_set_input(inp->scale_rows);
|
||||
|
||||
if (!ggml_is_contiguous(cur)) {
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
}
|
||||
cur = ggml_mul(ctx0, cur, inp->scale_rows);
|
||||
}
|
||||
|
||||
cb(cur, "embd", -1);
|
||||
|
||||
res->add_input(std::move(inp));
|
||||
|
||||
Reference in new issue
Block a user