Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: support dots3-note vision+audio (#27524)

Browse Source 
* text: conversion

* init impl

* mtmd: conversion

* impl mtmd cpp

* Update gguf-py/gguf/tensor_mapping.py

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
This commit is contained in:
Xuan-Son Nguyen
2026-08-22 10:35:50 +02:00
committed by GitHub
co-authored by Sigbjørn Skjæret
parent 3a653fea93
commit 54ee5ee643
15 changed files with 535 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip-model.h
+8
View File
@@ -93,6 +93,7 @@ struct clip_hparams {
float eps = 1e-6;
float rope_theta = 0.0;
int32_t n_expert_used = 0;
std::vector<int32_t> feature_layers;
int32_t attn_window_size = 0;
int32_t n_wa_pattern = 0;
@@ -259,6 +260,13 @@ struct clip_layer {
ggml_tensor * ff_down_w = nullptr;
ggml_tensor * ff_down_b = nullptr;
// MoE FFN (dots3note vision pyramid blocks)
ggml_tensor * ff_gate_inp_w = nullptr;
ggml_tensor * ff_gate_exps_w = nullptr;
ggml_tensor * ff_up_exps_w = nullptr;
ggml_tensor * ff_down_exps_w = nullptr;
ggml_tensor * ff_exp_probs_b = nullptr;
// layernorm 2 (or pre-FFN norm)
ggml_tensor * ln_2_w = nullptr;
ggml_tensor * ln_2_b = nullptr;