Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: support dots3-note vision+audio (#27524)

Browse Source 
* text: conversion

* init impl

* mtmd: conversion

* impl mtmd cpp

* Update gguf-py/gguf/tensor_mapping.py

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
This commit is contained in:
Xuan-Son Nguyen
2026-08-22 10:35:50 +02:00
committed by GitHub
co-authored by Sigbjørn Skjæret
parent 3a653fea93
commit 54ee5ee643
15 changed files with 535 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip.cpp
+122 -7
View File
@@ -514,11 +514,13 @@ ggml_tensor * clip_graph::build_vit(
cb(cur, "ffn_inp_normed", il);
// ffn
cur = build_ffn(cur,
layer.ff_up_w, layer.ff_up_b,
layer.ff_gate_w, layer.ff_gate_b,
layer.ff_down_w, layer.ff_down_b,
ffn_t, il);
cur = layer.ff_gate_exps_w
? build_moe_ffn(cur, layer, ffn_t, il)
: build_ffn(cur,
layer.ff_up_w, layer.ff_up_b,
layer.ff_gate_w, layer.ff_gate_b,
layer.ff_down_w, layer.ff_down_b,
ffn_t, il);
cb(cur, "ffn_out", il);
@@ -699,6 +701,50 @@ ggml_tensor * clip_graph::build_ffn(
return cur;
}
// MoE FFN with sigmoid router and normalized top-k weights (dots3note vision)
// the router runs in fp32; exp_probs_b only affects expert selection, not the weights
ggml_tensor * clip_graph::build_moe_ffn(ggml_tensor * cur, const clip_layer & layer, ffn_op_type type_op, int il) const {
const int64_t n_tokens = cur->ne[1];
const int64_t n_expert = layer.ff_gate_exps_w->ne[2];
const int64_t n_expert_used = std::min((int64_t) hparams.n_expert_used, n_expert);
GGML_ASSERT(n_expert_used > 0);
GGML_ASSERT(type_op == FFN_SILU);
ggml_tensor * probs = ggml_sigmoid(ctx0, build_mm(layer.ff_gate_inp_w, cur)); // [n_expert, n_tokens]
cb(probs, "ffn_moe_probs", il);
ggml_tensor * sel = layer.ff_exp_probs_b
? ggml_add(ctx0, probs, layer.ff_exp_probs_b)
: probs;
ggml_tensor * selected = ggml_top_k(ctx0, sel, n_expert_used); // [n_expert_used, n_tokens]
ggml_tensor * weights = ggml_get_rows(ctx0,
ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens), selected);
weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens);
weights = ggml_div(ctx0, weights, ggml_sum_rows(ctx0, weights));
weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens);
cb(weights, "ffn_moe_weights", il);
cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], 1, n_tokens);
ggml_tensor * gate = ggml_mul_mat_id(ctx0, layer.ff_gate_exps_w, cur, selected); // [n_ff, n_expert_used, n_tokens]
ggml_tensor * up = ggml_mul_mat_id(ctx0, layer.ff_up_exps_w, cur, selected);
cur = ggml_mul(ctx0, ggml_silu(ctx0, gate), up);
cur = ggml_mul_mat_id(ctx0, layer.ff_down_exps_w, cur, selected); // [n_embd, n_expert_used, n_tokens]
cur = ggml_mul(ctx0, cur, weights);
// sum over the selected experts
ggml_tensor * out = nullptr;
for (int64_t i = 0; i < n_expert_used; i++) {
ggml_tensor * v = ggml_view_2d(ctx0, cur, cur->ne[0], n_tokens, cur->nb[2], i * cur->nb[1]);
out = out ? ggml_add(ctx0, out, v) : v;
}
if (n_expert_used == 1) {
out = ggml_cont(ctx0, out);
}
cb(out, "ffn_moe_out", il);
return out;
}
ggml_tensor * clip_graph::build_attn(
ggml_tensor * wo,
ggml_tensor * wo_b,
@@ -933,9 +979,14 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
builder = std::make_unique<clip_graph_pixtral>(ctx, img);
} break;
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V: // same ViT + merger; pyramid MoE is handled by build_vit
{
builder = std::make_unique<clip_graph_dotsocr>(ctx, img);
} break;
case PROJECTOR_TYPE_DOTS3NOTE_A:
{
builder = std::make_unique<clip_graph_dots3note_a>(ctx, img);
} break;
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
{
@@ -1510,6 +1561,25 @@ struct clip_model_loader {
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_DOTS3NOTE_V:
{
hparams.rope_theta = 10000.0f;
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
get_u32(KEY_VISION_N_EXPERT_USED, hparams.n_expert_used);
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_DOTS3NOTE_A:
{
hparams.rope_theta = 10000.0f;
hparams.audio_chunk_len = 60; // in seconds
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 400;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
} break;
case PROJECTOR_TYPE_KIMIVL:
{
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
@@ -2190,12 +2260,20 @@ struct clip_model_loader {
layer.ln_1_b = get_tensor(string_format(TN_LN_1, prefix, il, "bias"), false);
layer.ln_2_b = get_tensor(string_format(TN_LN_2, prefix, il, "bias"), false);
// MoE ffn (dots3note vision pyramid blocks); replaces the dense ffn when present
layer.ff_gate_inp_w = get_tensor(string_format(TN_FFN_GATE_INP, prefix, il, "weight"), false);
layer.ff_gate_exps_w = get_tensor(string_format(TN_FFN_GATE_EXPS, prefix, il, "weight"), false);
layer.ff_up_exps_w = get_tensor(string_format(TN_FFN_UP_EXPS, prefix, il, "weight"), false);
layer.ff_down_exps_w = get_tensor(string_format(TN_FFN_DOWN_EXPS, prefix, il, "weight"), false);
layer.ff_exp_probs_b = get_tensor(string_format(TN_FFN_EXP_PROBS_B, prefix, il, "weight"), false);
const bool is_moe = layer.ff_gate_exps_w != nullptr;
// ffn
layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, prefix, il, "weight"));
layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, prefix, il, "weight"), !is_moe);
layer.ff_up_b = get_tensor(string_format(TN_FFN_UP, prefix, il, "bias"), false);
layer.ff_gate_w = get_tensor(string_format(TN_FFN_GATE, prefix, il, "weight"), false);
layer.ff_gate_b = get_tensor(string_format(TN_FFN_GATE, prefix, il, "bias"), false);
layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "weight"));
layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "weight"), !is_moe);
layer.ff_down_b = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "bias"), false);
// mimovl per-head attention sink bias
@@ -2677,6 +2755,7 @@ struct clip_model_loader {
model.mm_patch_merger_w = get_tensor(string_format(TN_MM_PATCH_MERGER, "weight"), false);
} break;
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
{
model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
@@ -2687,6 +2766,23 @@ struct clip_model_loader {
// post_trunk_norm: applied after all ViT blocks, before the merger
model.post_ln_w = get_tensor(string_format(TN_MM_POST_NORM, "weight"));
} break;
case PROJECTOR_TYPE_DOTS3NOTE_A:
{
model.conv2d_1_w = get_tensor(string_format(TN_CONV2D, 1, "weight"));
model.conv2d_1_b = get_tensor(string_format(TN_CONV2D, 1, "bias"));
model.conv2d_2_w = get_tensor(string_format(TN_CONV2D, 2, "weight"));
model.conv2d_2_b = get_tensor(string_format(TN_CONV2D, 2, "bias"));
model.conv2d_3_w = get_tensor(string_format(TN_CONV2D, 3, "weight"));
model.conv2d_3_b = get_tensor(string_format(TN_CONV2D, 3, "bias"));
model.conv_out_w = get_tensor(string_format(TN_CONV_OUT, "weight")); // no bias
// adapter: LayerNorm -> Linear -> GELU -> Linear
model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"));
model.mm_norm_pre_b = get_tensor(string_format(TN_MM_NORM_PRE, "bias"));
model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight"));
model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias"));
} break;
case PROJECTOR_TYPE_ULTRAVOX:
{
model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
@@ -4075,12 +4171,18 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
} break;
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
{
// dynamic size
int n_merge = ctx->model.hparams.n_merge;
int stride = n_merge * n_merge;
n_patches = CLIP_ALIGN(n_patches, stride) / stride;
} break;
case PROJECTOR_TYPE_DOTS3NOTE_A:
{
// 3x stride-2 conv2d over mel frames
n_patches = (img->nx() + 7) / 8;
} break;
case PROJECTOR_TYPE_PIXTRAL:
case PROJECTOR_TYPE_LIGHTONOCR:
{
@@ -4727,6 +4829,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
set_input_i32("minimax_pos_w", pos_w);
} break;
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
{
const int pw = image_size_width / patch_size;
const int ph = image_size_height / patch_size;
@@ -5217,6 +5320,16 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
set_input_i32("pos_w", pos_data);
} break;
case PROJECTOR_TYPE_DOTS3NOTE_A:
{
GGML_ASSERT(imgs.entries.size() == 1);
const int n_pos = (imgs.entries.front().nx() + 7) / 8; // 3x stride-2 conv2d
std::vector<int32_t> positions(n_pos);
for (int i = 0; i < n_pos; i++) {
positions[i] = i;
}
set_input_i32("positions", positions);
} break;
case PROJECTOR_TYPE_GEMMA4A:
{
GGML_ASSERT(imgs.entries.size() == 1);
@@ -5713,6 +5826,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_PIXTRAL:
case PROJECTOR_TYPE_LIGHTONOCR:
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
case PROJECTOR_TYPE_DOTS3NOTE_A:
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_MLP_NORM:
return ctx->model.mm_3_b->ne[0];