Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: support DeepSeek-V4-Flash-Vision-Exp (#28133)

Browse Source 
* mtmd: support DeepSeek-V4-Flash-Vision-Exp

* handle min/max token counts from CLI

* rm debugging

* use GGML_ROPE_TYPE_VISION

* nits

* apply review comments

* correct token count
This commit is contained in:
Xuan-Son Nguyen
2026-09-02 16:43:43 +02:00
committed by GitHub
parent 8e93a9773b
commit 7798007a29
13 changed files with 504 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/__init__.py
+1
View File
@@ -286,6 +286,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"CogVLMForCausalLM": "cogvlm",
"DeepseekOCR2ForCausalLM": "deepseek",
"DeepseekOCRForCausalLM": "deepseek",
"DeepseekV4ForCausalLM": "deepseek",
"Dots3NoteForCausalLM": "dots3",
"Dots3NoteForConditionalGeneration": "dots3",
"DotsOCRForCausalLM": "dotsocr",
conversion/deepseek.py
+73
View File
@@ -578,6 +578,9 @@ class DeepseekV4Model(TextModel):
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, gen = item
if (name.startswith(("aligner.", "image_"))
or name.endswith(".ffn.gate.bias_vl")):
return None
if name.startswith("mtp."):
if not cls.mtp_only:
cls._skipped_mtp_tensors += 1
@@ -1018,3 +1021,73 @@ class DeepseekV4DSparkModel(DeepseekV4Model):
self.gguf_writer.add_block_size(self.hparams["dspark_block_size"])
self.gguf_writer.add_target_layers([layer + 1 for layer in self.hparams["dspark_target_layer_ids"]])
@ModelBase.register("DeepseekV4ForCausalLM")
@ModelBase.example("deepseek-ai/DeepSeek-V4-Flash-Vision-Exp")
class DeepseekV4FlashVisionModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
assert self.hparams_vision is not None
# no preprocessor_config.json in the repo; normalization is (x/255 - 0.5) / 0.5
# ref: inference/image_processor.py (load_image)
self.preprocessor_config = {
"image_mean": [0.5, 0.5, 0.5],
"image_std": [0.5, 0.5, 0.5],
**self.preprocessor_config,
}
def get_vision_config(self) -> dict[str, Any] | None:
cfg = self.global_config
if cfg.get("vision_n_layers", 0) == 0:
raise ValueError("DeepseekV4FlashVisionModel requires vision_n_layers > 0 in the model config")
return {
"num_hidden_layers": cfg["vision_n_layers"],
"hidden_size": cfg["vision_dim"],
"num_attention_heads": cfg["vision_n_heads"],
"intermediate_size": cfg["vision_inter_dim"],
"patch_size": cfg["vision_patch_size"],
# dynamic resolution; only used for compat / warmup
"image_size": cfg["vision_patch_size"] * cfg["vision_downsample_ratio"] * 16,
"rope_theta": cfg.get("vision_rope_theta", 10000.0),
"downsample_ratio": cfg["vision_downsample_ratio"],
"min_pixels": cfg["vision_min_pixels"],
}
def set_gguf_parameters(self):
super().set_gguf_parameters()
assert self.hparams_vision is not None
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.DEEPSEEK4V)
# vision RMSNorm eps is the pytorch default, NOT the LLM's rms_norm_eps (1e-20)
# ref: inference/vision.py (RMSNorm)
self.gguf_writer.add_vision_attention_layernorm_eps(1e-6)
self.gguf_writer.add_vision_use_silu(True) # SwiGLU MLP
self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"])
self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"])
# hardcoded on the C++ side (see PROJECTOR_TYPE_DEEPSEEK4V in clip.cpp)
# if future models use different values, add GGUF keys for those
assert self.global_config["vision_max_n_token"] == 384
assert self.global_config["vision_max_wh_ratio"] == 8
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, _ = item
if not (name.startswith(("vision.", "aligner.", "image_"))):
return None
return super().filter_tensors(item)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
assert self.hparams_vision is not None
if name == "vision.patch_embed.proj.weight":
# nn.Linear over flattened (3, p, p) patches == conv2d weight
p = self.hparams_vision["patch_size"]
data_torch = data_torch.reshape(data_torch.shape[0], 3, p, p)
if ".mlp.w1." in name:
# fused SwiGLU gate+up
gate, up = data_torch.chunk(2, dim=0)
yield from super().modify_tensors(gate, name.replace("w1", "w1_gate"), bid)
yield from super().modify_tensors(up, name.replace("w1", "w1_up"), bid)
return
yield from super().modify_tensors(data_torch, name, bid)
gguf-py/gguf/constants.py
+10
View File
@@ -950,6 +950,9 @@ class MODEL_TENSOR(IntEnum):
V_RESMPL_PROJ = auto() # minicpmv
V_RESMPL_QUERY = auto() # minicpmv
V_TOK_EMBD_IMG_BREAK = auto() # pixtral
V_TOK_EMBD_IMG_START = auto() # deepseek4v
V_TOK_EMBD_IMG_END = auto() # deepseek4v
V_TOK_EMBD_IMG_PAD = auto() # deepseek4v
V_MM_PATCH_MERGER = auto() # mistral small 3.1
V_DS_NORM = auto() # qwen3vl
V_DS_FC1 = auto() # qwen3vl
@@ -1696,6 +1699,9 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.V_RESMPL_PROJ: "resampler.proj",
MODEL_TENSOR.V_RESMPL_QUERY: "resampler.query",
MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK: "v.token_embd.img_break", # pixtral
MODEL_TENSOR.V_TOK_EMBD_IMG_START: "v.token_embd.img_start", # deepseek4v
MODEL_TENSOR.V_TOK_EMBD_IMG_END: "v.token_embd.img_end", # deepseek4v
MODEL_TENSOR.V_TOK_EMBD_IMG_PAD: "v.token_embd.img_pad", # deepseek4v
MODEL_TENSOR.V_MM_PATCH_MERGER: "mm.patch_merger", # mistral small 3.1
MODEL_TENSOR.V_DS_NORM: "v.deepstack.{bid}.norm",
MODEL_TENSOR.V_DS_FC1: "v.deepstack.{bid}.fc1",
@@ -2030,6 +2036,9 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.V_RESMPL_PROJ,
MODEL_TENSOR.V_RESMPL_QUERY,
MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK,
MODEL_TENSOR.V_TOK_EMBD_IMG_START,
MODEL_TENSOR.V_TOK_EMBD_IMG_END,
MODEL_TENSOR.V_TOK_EMBD_IMG_PAD,
MODEL_TENSOR.V_MM_PATCH_MERGER,
MODEL_TENSOR.V_MM_MERGER_FC1,
MODEL_TENSOR.V_MM_MERGER_FC2,
@@ -5645,6 +5654,7 @@ class VisionProjectorType:
DOTS3NOTE_A = "dots3note_a" # audio
DEEPSEEKOCR = "deepseekocr"
DEEPSEEKOCR2 = "deepseekocr2"
DEEPSEEK4V = "deepseek4v"
LFM2A = "lfm2a" # audio
MUSIC_FLAMINGO = "musicflamingo" # audio
GLM4V = "glm4v"
gguf-py/gguf/tensor_mapping.py
+23
View File
@@ -1476,6 +1476,7 @@ class TensorNameMap:
## Vision encoder
MODEL_TENSOR.V_MMPROJ: (
"aligner.w{bid}", # deepseek4v (w1 -> mm.1, w2 -> mm.2)
"multi_modal_projector.linear_{bid}",
"mm_projector.proj.linear_{bid}", # Kimi-K2.5
"visual.merger.mlp.{bid}", # qwen2vl
@@ -1515,6 +1516,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_EMBD_PATCH: (
"vision.patch_embed.proj", # deepseek4v
"model.vision_tower.vision_model.embeddings.patch_embedding", # Granite4Vision
"vision_tower.vision_model.embeddings.patch_embedding",
"model.vision_tower.embeddings.patch_embedding", # minicpmv4_6
@@ -1570,6 +1572,7 @@ class TensorNameMap:
# TODO: I think these should all be moved to mapping_cfg?
MODEL_TENSOR.V_ENC_EMBD_IMGNL: (
"image_newline", # deepseek4v
"model.image_newline", # Deepseek-OCR, Granite4Vision
"vit.perceive.image_newline", # HunyuanVL
),
@@ -1580,6 +1583,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_ATTN_QKV: (
"vision.blocks.{bid}.attn.wqkv", # deepseek4v
"visual.blocks.{bid}.attn.qkv", # qwen3vl
"vision_tower.blocks.{bid}.attn.qkv", # dots.ocr
"vision_encoder.blocks.{bid}.attn.qkv", # dots3note
@@ -1667,6 +1671,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_INPUT_NORM: (
"vision.blocks.{bid}.norm1", # deepseek4v
"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.layer_norm1",
"model.vision_tower.encoder.layers.{bid}.layer_norm1", # minicpmv4_6
@@ -1692,6 +1697,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_ATTN_O: (
"vision.blocks.{bid}.attn.wo", # deepseek4v
"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj",
"model.vision_tower.encoder.layers.{bid}.self_attn.out_proj", # minicpmv4_6
@@ -1723,6 +1729,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_POST_ATTN_NORM: (
"vision.blocks.{bid}.norm2", # deepseek4v
"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.layer_norm2",
"model.vision_tower.encoder.layers.{bid}.layer_norm2", # minicpmv4_6
@@ -1749,6 +1756,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_FFN_UP: (
"vision.blocks.{bid}.mlp.w1_up", # deepseek4v (split from fused w1)
"vision_encoder.blocks.{bid}.mlp.fc3", # dots3note
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1",
@@ -1775,6 +1783,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_FFN_GATE: (
"vision.blocks.{bid}.mlp.w1_gate", # deepseek4v (split from fused w1)
"vision_encoder.blocks.{bid}.mlp.fc1", # dots3note
"vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf
"vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral
@@ -1784,6 +1793,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_FFN_DOWN: (
"vision.blocks.{bid}.mlp.w2", # deepseek4v
"vision_encoder.blocks.{bid}.mlp.fc2", # dots3note
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2",
@@ -1869,6 +1879,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_POST_NORM: (
"vision.norm", # deepseek4v
"model.vision_tower.vision_model.post_layernorm", # Granite4Vision
"vision_tower.vision_model.post_layernorm",
"model.vision_tower.post_layernorm", # minicpmv4_6
@@ -1960,6 +1971,18 @@ class TensorNameMap:
"v.token_embd.img_break", # for pixtral, this is a generated vector
),
MODEL_TENSOR.V_TOK_EMBD_IMG_START: (
"image_start", # deepseek4v
),
MODEL_TENSOR.V_TOK_EMBD_IMG_END: (
"image_end", # deepseek4v
),
MODEL_TENSOR.V_TOK_EMBD_IMG_PAD: (
"image_pad", # deepseek4v
),
MODEL_TENSOR.V_MM_PATCH_MERGER: (
"multi_modal_projector.patch_merger.merging_layer", # mistral small 3.1 - hf
"patch_merger.merging_layer", # mistral
tools/mtmd/CMakeLists.txt
+1
View File
@@ -30,6 +30,7 @@ add_library(mtmd
models/models.h
models/cogvlm.cpp
models/conformer.cpp
models/deepseek4v.cpp
models/dots3note.cpp
models/dotsocr.cpp
models/exaone4_5.cpp
tools/mtmd/clip-impl.h
+26 -2
View File
@@ -153,6 +153,9 @@
#define TN_MM_MERGER_FC1 "mm.merger.fc1.%s" // minimax-m3 patch-merge MLP
#define TN_MM_MERGER_FC2 "mm.merger.fc2.%s"
#define TN_TOK_IMG_BREAK "v.token_embd.img_break" // pixtral
#define TN_TOK_IMG_START "v.token_embd.img_start" // deepseek4v
#define TN_TOK_IMG_END "v.token_embd.img_end" // deepseek4v
#define TN_TOK_IMG_PAD "v.token_embd.img_pad" // deepseek4v
#define TN_TOK_GLM_BOI "adapter.boi" // glm-edge (these embeddings are not in text model)
#define TN_TOK_GLM_EOI "adapter.eoi" // glm-edge (these embeddings are not in text model)
#define TN_DEEPSTACK_NORM "v.deepstack.%d.norm.%s" // qwen3vl deepstack
@@ -296,8 +299,8 @@
// hunyuanvl (shared GGUF tensor names)
#define TN_MM_PRE_NORM "mm.pre_norm.%s"
#define TN_TOK_IMG_BEGIN "mm.image_begin"
#define TN_TOK_IMG_END "mm.image_end"
#define TN_MM_IMG_BEGIN "mm.image_begin" // note: legacy name, new models should use v.token_embd.*
#define TN_MM_IMG_END "mm.image_end" // note: legacy name, new models should use v.token_embd.*
// deepseek-ocr
#define TN_SAM_POS_EMBD "v.sam.pos_embd.%s"
@@ -480,6 +483,7 @@ enum projector_type {
PROJECTOR_TYPE_DOTS3NOTE_A,
PROJECTOR_TYPE_DEEPSEEKOCR,
PROJECTOR_TYPE_DEEPSEEKOCR2,
PROJECTOR_TYPE_DEEPSEEK4V,
PROJECTOR_TYPE_LFM2A,
PROJECTOR_TYPE_GLM4V,
PROJECTOR_TYPE_YOUTUVL,
@@ -544,6 +548,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_DOTS3NOTE_A, "dots3note_a"},
{ PROJECTOR_TYPE_DEEPSEEKOCR, "deepseekocr"},
{ PROJECTOR_TYPE_DEEPSEEKOCR2, "deepseekocr2"},
{ PROJECTOR_TYPE_DEEPSEEK4V, "deepseek4v"},
{ PROJECTOR_TYPE_LFM2A, "lfm2a"},
{ PROJECTOR_TYPE_GLM4V, "glm4v"},
{ PROJECTOR_TYPE_YOUTUVL, "youtuvl"},
@@ -655,6 +660,9 @@ struct clip_image_f32 {
// appends a learned newline (or EOI) token after the image
// no model uses it now (Granite4 Vision moved to anyres), kept for future models
bool add_newline = false;
// deepseek4v: number of leading IMAGE_PAD embeddings, aligns IMAGE_START to the LLM compressor ratio
// depends on the chunk position, set at tokenize time (see mtmd_tokenizer::add_media)
int32_t lead_pad = 0;
// llava-next "anyres" tiling, used by Granite4 Vision
// the whole grid is encoded and assembled in a single graph
@@ -771,6 +779,22 @@ static inline void clip_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_
}
}
// deepseek4v: layout of the LLM token block built from the aligner grid
struct dsv4_block_layout {
int rows; // grid rows, padded to an even count
int row_len; // grid width + 1 newline
int pad_last; // trailing pads
int n_out; // total block size, including lead pads and the start/end sentinels
};
static inline dsv4_block_layout dsv4_get_block_layout(int n_llm_w, int n_llm_h, int lead_pad) {
dsv4_block_layout bl;
bl.rows = n_llm_h + (n_llm_h % 2);
bl.row_len = n_llm_w + 1;
bl.pad_last = (bl.rows / 2 * bl.row_len) % 2 * 2;
bl.n_out = lead_pad + 1 + bl.rows * bl.row_len + bl.pad_last + 1;
return bl;
}
//
// logging
//
tools/mtmd/clip-model.h
+9
View File
@@ -100,6 +100,10 @@ struct clip_hparams {
std::unordered_set<int32_t> wa_layer_indexes; // explicit layer indexes that use full attention (for irregular patterns like YoutuVL)
std::vector<int32_t> wa_pattern_mode; // mimovl: per-layer window-attention mode
// deepseek4v: resize solver caps the LLM token count of the aligner grid
int32_t dsv4_max_n_token = 0;
int32_t dsv4_max_wh_ratio = 0;
// deepseek-ocr (sam)
int32_t sam_n_layer = 0;
int32_t sam_n_head = 0;
@@ -724,6 +728,11 @@ struct clip_model {
// pixtral, glm4v
ggml_tensor * token_embd_img_break = nullptr;
// deepseek4v sentinel embeddings (image_newline is reused for IMAGE_NEW_LINE)
ggml_tensor * token_embd_img_start = nullptr;
ggml_tensor * token_embd_img_end = nullptr;
ggml_tensor * token_embd_img_pad = nullptr;
ggml_tensor * mm_patch_merger_w = nullptr;
ggml_tensor * mm_patch_merger_b = nullptr;
tools/mtmd/clip.cpp
+116 -2
View File
@@ -1037,6 +1037,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_kimik25>(ctx, img);
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
builder = std::make_unique<clip_graph_deepseek4v>(ctx, img);
} break;
case PROJECTOR_TYPE_COGVLM:
{
builder = std::make_unique<clip_graph_cogvlm>(ctx, img);
@@ -1585,6 +1589,31 @@ struct clip_model_loader {
hparams.set_limit_image_tokens(2, 4096);
}
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.image_pad_color = {127, 127, 127};
hparams.rope_theta = 10000.0f;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
hparams.dsv4_max_n_token = 384;
hparams.dsv4_max_wh_ratio = 8;
const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
// handle min/max token counts from CLI
if (hparams.custom_image_min_tokens > 0) {
hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area;
}
if (hparams.custom_image_max_tokens > 0) {
// the cap is on the whole token block, keep some room for the resize solver
hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16);
}
hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area;
// a small custom max token count also lowers the min-pixel upscale threshold
hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels);
// avoid OOM on warmup
const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token));
hparams.set_warmup_n_tokens(warmup_side * warmup_side);
} break;
case PROJECTOR_TYPE_GEMMA3:
{
// default value (used by all model sizes in gemma 3 family)
@@ -2714,6 +2743,18 @@ struct clip_model_loader {
model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
// sentinel token embeddings written into the output block
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
model.token_embd_img_start = get_tensor(TN_TOK_IMG_START);
model.token_embd_img_end = get_tensor(TN_TOK_IMG_END);
model.token_embd_img_pad = get_tensor(TN_TOK_IMG_PAD);
} break;
case PROJECTOR_TYPE_PIXTRAL:
{
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
@@ -3161,8 +3202,8 @@ struct clip_model_loader {
model.mm_model_proj_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias"));
model.mm_pre_norm_w = get_tensor(string_format(TN_MM_PRE_NORM, "weight"));
model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight"));
model.mm_img_begin = get_tensor(TN_TOK_IMG_BEGIN);
model.mm_img_end = get_tensor(TN_TOK_IMG_END);
model.mm_img_begin = get_tensor(TN_MM_IMG_BEGIN);
model.mm_img_end = get_tensor(TN_MM_IMG_END);
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR, false);
} break;
@@ -4150,6 +4191,13 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
int y_patch = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
n_patches = x_patch * y_patch;
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
const int out_patch_size = params.patch_size * params.n_merge;
const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size;
const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out;
} break;
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
@@ -5021,6 +5069,58 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
set_input_i32("pos_w", pos_data);
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
// set the 2D positions (mrope layout, only the first 2 channels are used)
int n_patches_per_row = image_size_width / patch_size;
std::vector<int32_t> positions(n_pos * 4, 0);
for (int i = 0; i < n_pos; i++) {
positions[i] = i / n_patches_per_row; // row
positions[n_pos + i] = i % n_patches_per_row; // col
}
set_input_i32("positions", positions);
// token block layout index (see clip_graph_deepseek4v::build)
// rows [0, n_grid) are the aligner output, the sentinels follow
const int n_merge = hparams.n_merge;
const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge;
const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge;
const int n_grid = n_llm_w * n_llm_h;
const int idx_start = n_grid;
const int idx_end = n_grid + 1;
const int idx_newline = n_grid + 2;
const int idx_pad = n_grid + 3;
const int lead_pad = imgs.entries[0].lead_pad;
const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad);
std::vector<int32_t> idx;
idx.reserve(bl.n_out);
for (int i = 0; i < lead_pad; i++) {
idx.push_back(idx_pad);
}
idx.push_back(idx_start);
// pairs of adjacent rows are interleaved column-wise ("N-layout")
// ref: build_image_block in inference/image_processor.py
for (int t = 0; t < bl.rows * bl.row_len; t++) {
const int g = t / (2 * bl.row_len);
const int rem = t % (2 * bl.row_len);
const int c = rem / 2; // column
const int r = 2 * g + rem % 2; // row
if (r >= n_llm_h) {
idx.push_back(idx_pad);
} else if (c == n_llm_w) {
idx.push_back(idx_newline);
} else {
idx.push_back(r * n_llm_w + c);
}
}
for (int i = 0; i < bl.pad_last; i++) {
idx.push_back(idx_pad);
}
idx.push_back(idx_end);
set_input_i32("layout_idx", idx);
} break;
case PROJECTOR_TYPE_GLM_EDGE:
{
// llava and other models
@@ -5762,6 +5862,19 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n");
LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens);
// TEMP debugging (parity validation), will be removed before merge
// when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data]
const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS");
if (dump_path && strcmp(dump_path, "1") != 0) {
FILE * f = fopen(dump_path, "wb");
if (f) {
const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd };
fwrite(hdr, sizeof(hdr), 1, f);
fwrite(emb_data.data(), sizeof(float), emb_data.size(), f);
fclose(f);
}
}
// Print first few values of first token
LOG_INF("Token 0 (first 16 values): ");
for (int i = 0; i < std::min((int64_t)16, n_embd); i++) {
@@ -5866,6 +5979,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_KIMIK25:
case PROJECTOR_TYPE_YASA2:
case PROJECTOR_TYPE_DEEPSEEK4V:
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_HUNYUANVL:
return ctx->model.mm_model_proj->ne[1];
tools/mtmd/models/deepseek4v.cpp
+102
View File
@@ -0,0 +1,102 @@
#include "models.h"
// DeepSeek-V4-Flash-Vision encoder (deepseek4v)
//
// native-resolution ViT (RMSNorm, SwiGLU, 2D RoPE, no CLS / learned pos-embd)
// then the "aligner": 3x3 patch merge (torch.nn.functional.unfold) + 2-layer GELU MLP
//
// the graph outputs the complete LLM token block, built from the aligner output and 4 learned sentinel embeddings:
//
// [PAD]*lead_pad [START] <interleaved rows> [PAD]*pad_last [END]
//
// each aligner row ends with a NEWLINE, an odd row count is padded with a full row of PADs
// pairs of adjacent rows are interleaved column-wise ("N-layout")
// the mapping is precomputed on CPU as the "layout_idx" input (see set_input in clip.cpp)
//
// ref: inference/vision.py and inference/image_processor.py in the HF repo
ggml_cgraph * clip_graph_deepseek4v::build() {
const int n_merge = hparams.n_merge;
// 2D input positions
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches * 4);
ggml_set_name(positions, "positions");
ggml_set_input(positions);
int sections[4] = {d_head/4, d_head/4, 0, 0};
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
return ggml_rope_multi(ctx0, cur, positions, nullptr,
d_head/2, sections, GGML_ROPE_TYPE_VISION,
0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
};
ggml_tensor * inp = build_inp();
ggml_tensor * cur = build_vit(
inp, n_patches,
NORM_TYPE_RMS,
hparams.ffn_op,
nullptr, // no learned pos embd
add_pos);
cb(cur, "vit_out", -1);
// aligner patch merge: zero-pad the patch grid to a multiple of n_merge
// then F.unfold == im2col with a dummy kernel (same trick as pixtral)
{
cur = ggml_reshape_3d(ctx0, cur, n_embd, n_patches_x, n_patches_y);
cur = ggml_permute(ctx0, cur, 2, 0, 1, 3); // [x, y, n_embd]
cur = ggml_cont(ctx0, cur);
const int pad_x = (n_merge - n_patches_x % n_merge) % n_merge;
const int pad_y = (n_merge - n_patches_y % n_merge) % n_merge;
if (pad_x || pad_y) {
cur = ggml_pad(ctx0, cur, pad_x, pad_y, 0, 0);
}
ggml_tensor * kernel = ggml_view_3d(ctx0, cur, n_merge, n_merge, cur->ne[2], 0, 0, 0);
cur = ggml_im2col(ctx0, kernel, cur, n_merge, n_merge, 0, 0, 1, 1, true, inp->type);
cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]);
// aligner MLP (F.gelu in the reference == erf-based gelu)
cur = build_ffn(cur,
model.mm_1_w, model.mm_1_b,
nullptr, nullptr,
model.mm_2_w, model.mm_2_b,
FFN_GELU_ERF,
-1);
cb(cur, "aligner_out", -1);
}
// assemble the token block: append the sentinel embeddings as extra rows
// then reorder everything with the precomputed layout index
{
const int64_t n_embd_out = cur->ne[0];
const int64_t n_grid = cur->ne[1]; // n_llm_w * n_llm_h
// rows n_grid + 0..3, keep in sync with the index computation in set_input
ggml_tensor * sentinels[] = {
model.token_embd_img_start,
model.token_embd_img_end,
model.image_newline,
model.token_embd_img_pad,
};
for (ggml_tensor * tok : sentinels) {
cur = ggml_concat(ctx0, cur, ggml_reshape_2d(ctx0, tok, n_embd_out, 1), 1);
}
const int n_llm_w = CLIP_ALIGN(n_patches_x, n_merge) / n_merge;
const int n_llm_h = CLIP_ALIGN(n_patches_y, n_merge) / n_merge;
const int n_out = dsv4_get_block_layout(n_llm_w, n_llm_h, img.lead_pad).n_out;
GGML_ASSERT(n_grid == n_llm_w * n_llm_h);
ggml_tensor * layout_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_out);
ggml_set_name(layout_idx, "layout_idx");
ggml_set_input(layout_idx);
cur = ggml_get_rows(ctx0, cur, layout_idx);
}
// build the graph
ggml_build_forward_expand(gf, cur);
return gf;
}
tools/mtmd/models/models.h
+5
View File
@@ -34,6 +34,11 @@ struct clip_graph_pixtral : clip_graph {
ggml_cgraph * build() override;
};
struct clip_graph_deepseek4v : clip_graph {
clip_graph_deepseek4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct clip_graph_qwen2vl : clip_graph {
clip_graph_qwen2vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
tools/mtmd/mtmd-image.cpp
+102
View File
@@ -1092,6 +1092,108 @@ clip_image_size mtmd_image_preprocessor_deepseekocr::find_closest_aspect_ratio(
return best_ratio;
}
//
// DeepSeek-V4-Flash-Vision (deepseek4v)
//
// port of load_image / safe_resize / solve_resize_ratio / grid_tokens from inference/image_processor.py
// the resize solver picks the largest target size (multiple of patch_size) whose LLM token block fits max_n_token
//
// ref: grid_tokens()
mtmd_image_preprocessor_deepseek4v::grid_info mtmd_image_preprocessor_deepseek4v::grid_tokens(int best_height, int best_width, int patch_size, int r) {
grid_info g;
g.n_llm_h = ((best_height / patch_size) + r - 1) / r;
g.n_llm_w = ((best_width / patch_size) + r - 1) / r;
g.n_tokens = dsv4_get_block_layout(g.n_llm_w, g.n_llm_h, 0).n_out;
return g;
}
// ref: solve_resize_ratio()
void mtmd_image_preprocessor_deepseek4v::solve_resize_ratio(int height, int width, int p, int r, int max_n_token,
int & best_height, int & best_width) {
const double ratio = (double) height / width;
const double max_w_f = std::sqrt((max_n_token - 2) / ratio + 0.25) - 0.5;
const double max_h_f = max_w_f * ratio;
if (max_w_f < 1.0) {
const int max_w = 1;
int max_h = (max_n_token - 2) / (max_w + 1);
if (max_h % 2 == 1) {
max_h -= 1;
}
best_width = max_w * p * r;
best_height = max_h * p * r;
} else if (max_h_f < 2.0) {
const int max_h = 2;
// guard tiny budgets; cannot be hit with the current lower bound on max_n_token
const int max_w = std::max(((max_n_token - 2) / max_h) - 1, 2);
best_width = max_w * p * r;
best_height = max_h * p * r;
} else {
const int max_w_i = (int) std::floor(max_w_f);
int max_h_i = (int) std::floor(max_h_f);
if (max_h_i % 2 == 1) {
max_h_i -= 1;
}
const double beta = std::min(
(double) max_w_i * p * r / width,
(double) max_h_i * p * r / height);
best_width = (int) std::floor(width * beta / p) * p;
best_height = (int) std::floor(height * beta / p) * p;
}
}
// ref: safe_resize()
void mtmd_image_preprocessor_deepseek4v::safe_resize(int height, int width, int & best_height, int & best_width,
int p, int r, int max_n_token) {
max_n_token -= 4 - 1; // reserve room for the position-dependent lead pads (COMPRESS_PAD_TO - 1)
grid_info g = grid_tokens(best_height, best_width, p, r);
int budget = max_n_token;
while (g.n_tokens > max_n_token) {
solve_resize_ratio(height, width, p, r, budget, best_height, best_width);
g = grid_tokens(best_height, best_width, p, r);
budget -= 1;
}
}
// ref: load_image()
mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out out;
const int p = hparams.patch_size;
const int r = hparams.n_merge;
const int max_n_token = hparams.dsv4_max_n_token;
const int max_wh = hparams.dsv4_max_wh_ratio;
const clip_image_size orig = img.get_size();
int width = orig.width;
int height = orig.height;
if (max_wh > 0 && width > height * max_wh) {
width = height * max_wh;
}
if (hparams.image_min_pixels > 0 && width * height > 0
&& width * height < hparams.image_min_pixels) {
const double up = std::sqrt((double) hparams.image_min_pixels / ((double) width * height));
width = (int) (width * up);
height = (int) (height * up);
}
int best_width = CLIP_ALIGN(width, p);
int best_height = CLIP_ALIGN(height, p);
safe_resize(height, width, best_height, best_width, p, r, max_n_token);
clip_image_u8 resized;
if (max_wh > 0 && orig.width >= max_wh * orig.height) {
// extreme aspect ratio: plain stretch resize, no padding
img_tool::resize(img, resized, {best_width, best_height}, hparams.image_resize_algo, PAD_NONE);
} else {
// aspect-preserving resize + centered padding (PIL ImageOps.pad)
img_tool::resize(img, resized, {best_width, best_height}, hparams.image_resize_algo,
PAD_NEAREST, hparams.image_pad_color);
}
out.append(hparams, resized);
return out;
}
mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out output;
int grid_w = 0;
tools/mtmd/mtmd-image.h
+16
View File
@@ -129,6 +129,22 @@ struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor {
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};
// ref: inference/image_processor.py in the HF repo (DeepSeek-V4-Flash-Vision)
struct mtmd_image_preprocessor_deepseek4v : mtmd_image_preprocessor {
mtmd_image_preprocessor_deepseek4v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
private:
struct grid_info {
int n_llm_h;
int n_llm_w;
int n_tokens; // token count of the block (incl. newline/pad rows and start/end, excl. lead pads)
};
static grid_info grid_tokens(int best_height, int best_width, int patch_size, int r);
static void solve_resize_ratio(int height, int width, int p, int r, int max_n_token, int & best_height, int & best_width);
static void safe_resize(int height, int width, int & best_height, int & best_width, int p, int r, int max_n_token);
};
// custom llava-uhd slicing logic for MiniCPM-V
struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd {
using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;
tools/mtmd/mtmd.cpp
+20 -1
View File
@@ -27,7 +27,7 @@
#include <vector>
// remember to bump this if the serialization format changes
#define MTMD_SERIALIZATION_VERSION 1
#define MTMD_SERIALIZATION_VERSION 2
struct mtmd_serialization {
// note: using 64-bit here for future-proofing
@@ -105,12 +105,14 @@ void clip_image_f32::serialize(mtmd_serialization & ser) const {
// note: buf is intentionally NOT serialized; the loaded clip_image_f32 will always be a placeholder
ser.write(add_viewsep);
ser.write(add_newline);
ser.write(lead_pad);
ser.write((int32_t)nx_);
ser.write((int32_t)ny_);
}
void clip_image_f32::deserialize(mtmd_serialization & ser) {
add_viewsep = ser.read<bool>();
add_newline = ser.read<bool>();
lead_pad = ser.read<int32_t>();
nx_ = ser.read<int32_t>();
ny_ = ser.read<int32_t>();
buf.clear(); // always a placeholder after loading
@@ -824,6 +826,11 @@ struct mtmd_context {
img_end = "<|im_end|>";
image_preproc = std::make_unique<mtmd_image_preprocessor_longest_edge>(ctx_v);
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
// no vocab tokens are added; the start/end/newline markers are learned embeddings emitted by the encoder
image_preproc = std::make_unique<mtmd_image_preprocessor_deepseek4v>(ctx_v);
} break;
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
{
@@ -1451,6 +1458,18 @@ struct mtmd_tokenizer {
return 2;
}
if (ctx->proj_type_v() == PROJECTOR_TYPE_DEEPSEEK4V) {
// the text model perceives input in blocks of N tokens (N = COMPRESS_PAD_TO = 4, same as the CSA compress ratio)
// image need to be aligned to block size, while adding IMAGE_PAD embeddings to the beginning
// TODO @ngxson : maybe refactor this in the future
constexpr int32_t align = 4;
size_t n_past = 0;
for (const auto & e : cur.entries) {
n_past += mtmd_input_chunk_get_n_tokens(&e);
}
preproc_out.entries[0].lead_pad = align - 1 - (int32_t)(n_past % align);
}
size_t n_tokens = 0;
for (auto & e : preproc_out.entries) {
n_tokens += clip_n_output_tokens(ctx->ctx_v, &e);