model : add LiquidAI/d1-omni-600M decision model (#30114)

* model : add LiquidAI/d1-omni-600M decision model

Assisted-by: Claude Opus 5.5

* mtmd : keep conformer GLU sigmoid on CUDA

Assisted-by: Claude Opus 5.5

* server : take d1omni audio through images and input_audio, scope memory-less lfm2 to non-causal

Assisted-by: Claude Opus 5.5

* common : rename decision type d1omni to lfm2-d1-omni, server : make images an alias of files

Assisted-by: Claude Opus 5.5
This commit is contained in:
Tarek Dakhran authored and GitHub committed 2026-10-08 02:07:49 +02:00
1 parent aa5e0092fd
commit a657f7e981
21 files changed
+833 -35

No files matched your search

+291 -4
View File
@@ -4,6 +4,9 @@
#include <algorithm>
// question types of a decision model: choice, score, noul
static const uint32_t N_DECISION_TYPES = 3;
void llama_model_lfm2::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
@@ -23,6 +26,15 @@ void llama_model_lfm2::load_arch_hparams(llama_model_loader & ml) {
default: type = LLM_TYPE_UNKNOWN;
}
ml.get_key(LLM_KV_DECISION_BLOCK_COUNT, hparams.n_layer_decision, false);
if (hparams.n_layer_decision > 0) {
if (hparams.n_layer_decision >= hparams.n_layer() || hparams.causal_attn) {
throw std::runtime_error("invalid decision head");
}
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
hparams.n_embd_out_impl = N_DECISION_TYPES;
}
if (const auto is_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); is_swa && hparams.n_swa > 0) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
@@ -37,13 +49,49 @@ void llama_model_lfm2::load_arch_tensors(llama_model_loader &) {
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM_LFM2, "weight"), {n_embd}, 0);
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
if (output == NULL) {
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
if (hparams.n_layer_decision > 0) {
// decision head: plain pre-norm blocks with biases
for (int i = n_layer - (int) hparams.n_layer_decision; i < n_layer; ++i) {
auto & layer = layers[i];
const int64_t n_ff_head = hparams.n_ff(i);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0);
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, 3 * n_embd}, 0);
layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {3 * n_embd}, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, 0);
layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff_head}, 0);
layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff_head}, 0);
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff_head, n_embd}, 0);
layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, 0);
}
if (n_token_types != N_DECISION_TYPES) {
throw std::runtime_error("decision model must have one token type per question type");
}
type_embd = create_tensor(tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_token_types}, 0);
cls_norm = create_tensor(tn(LLM_TENSOR_CLS_NORM, "weight"), {n_embd}, 0);
cls_norm_b = create_tensor(tn(LLM_TENSOR_CLS_NORM, "bias"), {n_embd}, 0);
cls = create_tensor(tn(LLM_TENSOR_CLS, "weight"), {n_embd, n_embd}, 0);
cls_b = create_tensor(tn(LLM_TENSOR_CLS, "bias"), {n_embd}, 0);
cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, 1}, 0);
cls_out_b = create_tensor(tn(LLM_TENSOR_CLS_OUT, "bias"), {1}, 0);
} else {
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
if (output == NULL) {
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
}
}
for (int i = 0; i < n_layer; ++i) {
for (int i = 0; i < n_layer - (int) hparams.n_layer_decision; ++i) {
auto & layer = layers[i];
const bool is_moe_layer = i >= static_cast<int>(hparams.n_layer_dense_lead);
@@ -87,6 +135,9 @@ void llama_model_lfm2::load_arch_tensors(llama_model_loader &) {
}
std::unique_ptr<llm_graph_context> llama_model_lfm2::build_arch_graph(const llm_graph_params & params) const {
if (hparams.n_layer_decision > 0) {
return std::make_unique<graph_decision>(*this, params);
}
if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) {
return std::make_unique<graph<true>>(*this, params);
} else {
@@ -294,6 +345,242 @@ llama_model_lfm2::graph<iswa>::graph(const llama_model & model, const llm_graph_
ggml_build_forward_expand(gf, cur);
}
// media entries (an image or audio prefix) are embeddings, text entries are tokens
static bool lfm2_is_media(const llama_ubatch & ubatch, int64_t i) {
return ubatch.is_mixed() ? ubatch.type[i] != 0 : ubatch.token == nullptr;
}
// non-causal within a sequence, the media never reads the text, so it is a function of the media alone
// in the head, the text and the media only read their own kind
class llm_graph_input_attn_media : public llm_graph_input_attn_no_cache {
public:
llm_graph_input_attn_media(const llama_hparams & hparams, const llama_cparams & cparams, bool is_head) :
llm_graph_input_attn_no_cache(hparams, cparams), is_head(is_head) {}
void set_input(const llama_ubatch * ubatch) override {
const int64_t n_tokens = ubatch->n_tokens;
std::vector<bool> is_media(n_tokens);
for (int64_t i = 0; i < n_tokens; ++i) {
is_media[i] = lfm2_is_media(*ubatch, i);
}
const auto fill_mask = [&](auto * data, auto zero, auto ninf) {
for (int64_t i1 = 0; i1 < n_tokens; ++i1) {
for (int64_t i0 = 0; i0 < n_tokens; ++i0) {
bool visible = ubatch->seq_id[i0][0] == ubatch->seq_id[i1][0];
if (is_head) {
visible = visible && is_media[i0] == is_media[i1];
} else {
visible = visible && !(is_media[i1] && !is_media[i0]);
}
data[i1 * n_tokens + i0] = visible ? zero : ninf;
}
}
};
GGML_ASSERT(ggml_backend_buffer_is_host(self_kq_mask->buffer));
if (self_kq_mask->type == GGML_TYPE_F16) {
fill_mask((ggml_fp16_t *) self_kq_mask->data, ggml_fp32_to_fp16(0.0f), ggml_fp32_to_fp16(-INFINITY));
} else {
fill_mask((float *) self_kq_mask->data, 0.0f, -INFINITY);
}
}
const bool is_head;
};
// 1 where the previous (next) token is the left (right) neighbor in the same sequence
// the last media entry does not read the text on its right
class llm_graph_input_conv_mask : public llm_graph_input_i {
public:
void set_input(const llama_ubatch * ubatch) override {
const int64_t n_tokens = ubatch->n_tokens;
std::vector<float> data_left(n_tokens, 0.0f);
std::vector<float> data_right(n_tokens, 0.0f);
for (int64_t i = 0; i + 1 < n_tokens; ++i) {
const bool is_next = ubatch->seq_id[i][0] == ubatch->seq_id[i + 1][0] && ubatch->pos[i] + 1 == ubatch->pos[i + 1];
data_right[i] = is_next && !(lfm2_is_media(*ubatch, i) && !lfm2_is_media(*ubatch, i + 1));
data_left[i + 1] = is_next;
}
ggml_backend_tensor_set(left, data_left.data(), 0, ggml_nbytes(left));
ggml_backend_tensor_set(right, data_right.data(), 0, ggml_nbytes(right));
}
ggml_tensor * left = nullptr; // F32 [1, n_tokens]
ggml_tensor * right = nullptr; // F32 [1, n_tokens]
};
llama_model_lfm2::graph_decision::graph_decision(const llama_model & model, const llm_graph_params & params) :
llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
const int n_layer_enc = n_layer - hparams.n_layer_decision;
ggml_tensor * cur = build_inp_embd(model.tok_embd);
cb(cur, "model.embed_tokens", -1);
ggml_tensor * inp_pos = build_inp_pos();
ggml_tensor * inp_out_ids = build_inp_out_ids();
const auto type_mask = cparams.flash_attn ? GGML_TYPE_F16 : GGML_TYPE_F32;
llm_graph_input_attn_no_cache * inp_attn[2];
for (bool is_head : {false, true}) {
auto inp = std::make_unique<llm_graph_input_attn_media>(hparams, cparams, is_head);
inp->self_kq_mask = ggml_new_tensor_4d(ctx0, type_mask, n_tokens, n_tokens, 1, 1);
ggml_set_input(inp->self_kq_mask);
inp->self_kq_mask_cnv = inp->self_kq_mask;
inp_attn[is_head] = (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp));
}
auto inp_conv = std::make_unique<llm_graph_input_conv_mask>();
inp_conv->left = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, n_tokens);
inp_conv->right = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, n_tokens);
ggml_set_input(inp_conv->left);
ggml_set_input(inp_conv->right);
ggml_tensor * conv_left = inp_conv->left;
ggml_tensor * conv_right = inp_conv->right;
res->add_input(std::move(inp_conv));
for (int il = 0; il < n_layer_enc; ++il) {
const auto & layer = model.layers[il];
ggml_tensor * inpL = cur;
cur = build_norm(cur, layer.attn_norm, NULL, LLM_NORM_RMS, il);
cb(cur, "model.layers.{}.operator_norm", il);
if (hparams.is_recr(il)) {
ggml_tensor * bcx = build_lora_mm(layer.shortconv.in_proj, cur);
cb(bcx, "model.layers.{}.conv.in_proj", il);
ggml_tensor * b = ggml_view_2d(ctx0, bcx, n_embd, n_tokens, bcx->nb[1], 0 * n_embd * ggml_element_size(bcx));
ggml_tensor * c = ggml_view_2d(ctx0, bcx, n_embd, n_tokens, bcx->nb[1], 1 * n_embd * ggml_element_size(bcx));
ggml_tensor * x = ggml_view_2d(ctx0, bcx, n_embd, n_tokens, bcx->nb[1], 2 * n_embd * ggml_element_size(bcx));
// centred 3-tap conv, a tap outside the sequence reads 0
ggml_tensor * bx = ggml_mul(ctx0, b, x);
ggml_tensor * bxp = ggml_pad_ext(ctx0, bx, 0, 0, 1, 1, 0, 0, 0, 0);
ggml_tensor * prv = ggml_view_2d(ctx0, bxp, n_embd, n_tokens, bxp->nb[1], 0);
ggml_tensor * nxt = ggml_view_2d(ctx0, bxp, n_embd, n_tokens, bxp->nb[1], 2 * bxp->nb[1]);
GGML_ASSERT(hparams.n_shortconv_l_cache == 3);
ggml_tensor * taps = ggml_cont(ctx0, ggml_transpose(ctx0, layer.shortconv.conv));
ggml_tensor * tap0 = ggml_view_1d(ctx0, taps, n_embd, 0 * taps->nb[1]);
ggml_tensor * tap1 = ggml_view_1d(ctx0, taps, n_embd, 1 * taps->nb[1]);
ggml_tensor * tap2 = ggml_view_1d(ctx0, taps, n_embd, 2 * taps->nb[1]);
ggml_tensor * y = ggml_mul(ctx0, bx, tap1);
y = ggml_add(ctx0, y, ggml_mul(ctx0, ggml_mul(ctx0, prv, tap0), conv_left));
y = ggml_add(ctx0, y, ggml_mul(ctx0, ggml_mul(ctx0, nxt, tap2), conv_right));
cb(y, "model.layers.{}.conv.conv", il);
cur = build_lora_mm(layer.shortconv.out_proj, ggml_mul(ctx0, c, y));
cb(cur, "model.layers.{}.conv.out_proj", il);
} else {
auto [q, k, v] = build_qkv(layer, cur, n_embd_head, n_head, hparams.n_head_kv(il), il);
q = build_norm(q, layer.attn_q_norm, NULL, LLM_NORM_RMS, il);
k = build_norm(k, layer.attn_k_norm, NULL, LLM_NORM_RMS, il);
q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor,
attn_factor, beta_fast, beta_slow);
k = ggml_rope_ext(ctx0, k, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor,
attn_factor, beta_fast, beta_slow);
cur = build_attn(inp_attn[0],
layer.wo, NULL, layer.wo_s,
q, k, v, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
cb(cur, "model.layers.{}.self_attn.out_proj", il);
}
cur = ggml_add(ctx0, cur, inpL);
ggml_tensor * ffn_out = build_norm(cur, layer.ffn_norm, NULL, LLM_NORM_RMS, il);
ffn_out = build_ffn(ffn_out,
layer.ffn_up, NULL, NULL,
layer.ffn_gate, NULL, NULL,
layer.ffn_down, NULL, NULL,
NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);
cur = ggml_add(ctx0, cur, ffn_out);
cb(cur, "l_out", il);
}
cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
cb(cur, "result_norm", -1);
cur = build_decision_head(model, cur, inp_attn[1], inp_out_ids);
res->t_embd = cur;
ggml_build_forward_expand(gf, cur);
}
// same as llama_model_modern_bert::graph::build_decision_head(), with the head counts of the head layers
ggml_tensor * llama_model_lfm2::graph_decision::build_decision_head(
const llama_model & model,
ggml_tensor * inp,
llm_graph_input_attn_no_cache * inp_attn,
ggml_tensor * inp_out_ids) {
const int64_t n_embd_head = hparams.n_embd_head_v();
const int n_layer_enc = n_layer - hparams.n_layer_decision;
ggml_tensor * scores = nullptr;
// the question type is not a graph input, so the head is evaluated for each of them
for (uint32_t it = 0; it < N_DECISION_TYPES; ++it) {
ggml_tensor * type_row = ggml_view_1d(ctx0, model.type_embd, n_embd, it * model.type_embd->nb[1]);
ggml_tensor * inpL = ggml_add(ctx0, inp, type_row);
for (int il = n_layer_enc; il < n_layer; ++il) {
const auto & layer = model.layers[il];
ggml_tensor * cur = build_norm(inpL, layer.attn_norm, layer.attn_norm_b, LLM_NORM, il);
cb(cur, "attn_norm", il);
// no positional encoding in the head
auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il);
cur = build_attn(inp_attn,
layer.wo, layer.wo_b, layer.wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
cb(cur, "kqv_out", il);
if (il == n_layer - 1 && inp_out_ids) {
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
}
ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);
cb(ffn_inp, "ffn_inp", il);
cur = build_norm(ffn_inp, layer.ffn_norm, layer.ffn_norm_b, LLM_NORM, il);
cb(cur, "ffn_norm", il);
cur = build_ffn(cur,
layer.ffn_up, layer.ffn_up_b, NULL,
NULL, NULL, NULL,
layer.ffn_down, layer.ffn_down_b, NULL,
NULL,
LLM_FFN_RELU,
LLM_FFN_SEQ, il);
inpL = ggml_add(ctx0, cur, ffn_inp);
}
// scorer
ggml_tensor * cur = build_norm(inpL, model.cls_norm, model.cls_norm_b, LLM_NORM, -1);
cur = ggml_add(ctx0, build_lora_mm(model.cls, cur), model.cls_b);
cur = ggml_gelu_erf(ctx0, cur);
cur = ggml_add(ctx0, build_lora_mm(model.cls_out, cur), model.cls_out_b);
scores = scores ? ggml_concat(ctx0, scores, cur, 0) : cur;
}
cb(scores, "decision_scores", -1);
return scores;
}
// Explicit template instantiations
template struct llama_model_lfm2::graph<true>;
template struct llama_model_lfm2::graph<false>;