diff --git a/conversion/__init__.py b/conversion/__init__.py index a5632fcc4..254a3e6c8 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -286,6 +286,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "CogVLMForCausalLM": "cogvlm", "DeepseekOCR2ForCausalLM": "deepseek", "DeepseekOCRForCausalLM": "deepseek", + "DeepseekV4ForCausalLM": "deepseek", "Dots3NoteForCausalLM": "dots3", "Dots3NoteForConditionalGeneration": "dots3", "DotsOCRForCausalLM": "dotsocr", diff --git a/conversion/deepseek.py b/conversion/deepseek.py index 225f8645d..c244e94ec 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -578,6 +578,9 @@ class DeepseekV4Model(TextModel): @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, gen = item + if (name.startswith(("aligner.", "image_")) + or name.endswith(".ffn.gate.bias_vl")): + return None if name.startswith("mtp."): if not cls.mtp_only: cls._skipped_mtp_tensors += 1 @@ -1018,3 +1021,73 @@ class DeepseekV4DSparkModel(DeepseekV4Model): self.gguf_writer.add_block_size(self.hparams["dspark_block_size"]) self.gguf_writer.add_target_layers([layer + 1 for layer in self.hparams["dspark_target_layer_ids"]]) + + +@ModelBase.register("DeepseekV4ForCausalLM") +@ModelBase.example("deepseek-ai/DeepSeek-V4-Flash-Vision-Exp") +class DeepseekV4FlashVisionModel(MmprojModel): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + assert self.hparams_vision is not None + # no preprocessor_config.json in the repo; normalization is (x/255 - 0.5) / 0.5 + # ref: inference/image_processor.py (load_image) + self.preprocessor_config = { + "image_mean": [0.5, 0.5, 0.5], + "image_std": [0.5, 0.5, 0.5], + **self.preprocessor_config, + } + + def get_vision_config(self) -> dict[str, Any] | None: + cfg = self.global_config + if cfg.get("vision_n_layers", 0) == 0: + raise ValueError("DeepseekV4FlashVisionModel requires vision_n_layers > 0 in the model config") + return { + "num_hidden_layers": cfg["vision_n_layers"], + "hidden_size": cfg["vision_dim"], + "num_attention_heads": cfg["vision_n_heads"], + "intermediate_size": cfg["vision_inter_dim"], + "patch_size": cfg["vision_patch_size"], + # dynamic resolution; only used for compat / warmup + "image_size": cfg["vision_patch_size"] * cfg["vision_downsample_ratio"] * 16, + "rope_theta": cfg.get("vision_rope_theta", 10000.0), + "downsample_ratio": cfg["vision_downsample_ratio"], + "min_pixels": cfg["vision_min_pixels"], + } + + def set_gguf_parameters(self): + super().set_gguf_parameters() + assert self.hparams_vision is not None + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.DEEPSEEK4V) + # vision RMSNorm eps is the pytorch default, NOT the LLM's rms_norm_eps (1e-20) + # ref: inference/vision.py (RMSNorm) + self.gguf_writer.add_vision_attention_layernorm_eps(1e-6) + self.gguf_writer.add_vision_use_silu(True) # SwiGLU MLP + self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"]) + self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"]) + # hardcoded on the C++ side (see PROJECTOR_TYPE_DEEPSEEK4V in clip.cpp) + # if future models use different values, add GGUF keys for those + assert self.global_config["vision_max_n_token"] == 384 + assert self.global_config["vision_max_wh_ratio"] == 8 + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, _ = item + if not (name.startswith(("vision.", "aligner.", "image_"))): + return None + return super().filter_tensors(item) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + assert self.hparams_vision is not None + if name == "vision.patch_embed.proj.weight": + # nn.Linear over flattened (3, p, p) patches == conv2d weight + p = self.hparams_vision["patch_size"] + data_torch = data_torch.reshape(data_torch.shape[0], 3, p, p) + + if ".mlp.w1." in name: + # fused SwiGLU gate+up + gate, up = data_torch.chunk(2, dim=0) + yield from super().modify_tensors(gate, name.replace("w1", "w1_gate"), bid) + yield from super().modify_tensors(up, name.replace("w1", "w1_up"), bid) + return + + yield from super().modify_tensors(data_torch, name, bid) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index c99feb3c7..56477c198 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -950,6 +950,9 @@ class MODEL_TENSOR(IntEnum): V_RESMPL_PROJ = auto() # minicpmv V_RESMPL_QUERY = auto() # minicpmv V_TOK_EMBD_IMG_BREAK = auto() # pixtral + V_TOK_EMBD_IMG_START = auto() # deepseek4v + V_TOK_EMBD_IMG_END = auto() # deepseek4v + V_TOK_EMBD_IMG_PAD = auto() # deepseek4v V_MM_PATCH_MERGER = auto() # mistral small 3.1 V_DS_NORM = auto() # qwen3vl V_DS_FC1 = auto() # qwen3vl @@ -1696,6 +1699,9 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.V_RESMPL_PROJ: "resampler.proj", MODEL_TENSOR.V_RESMPL_QUERY: "resampler.query", MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK: "v.token_embd.img_break", # pixtral + MODEL_TENSOR.V_TOK_EMBD_IMG_START: "v.token_embd.img_start", # deepseek4v + MODEL_TENSOR.V_TOK_EMBD_IMG_END: "v.token_embd.img_end", # deepseek4v + MODEL_TENSOR.V_TOK_EMBD_IMG_PAD: "v.token_embd.img_pad", # deepseek4v MODEL_TENSOR.V_MM_PATCH_MERGER: "mm.patch_merger", # mistral small 3.1 MODEL_TENSOR.V_DS_NORM: "v.deepstack.{bid}.norm", MODEL_TENSOR.V_DS_FC1: "v.deepstack.{bid}.fc1", @@ -2030,6 +2036,9 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.V_RESMPL_PROJ, MODEL_TENSOR.V_RESMPL_QUERY, MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK, + MODEL_TENSOR.V_TOK_EMBD_IMG_START, + MODEL_TENSOR.V_TOK_EMBD_IMG_END, + MODEL_TENSOR.V_TOK_EMBD_IMG_PAD, MODEL_TENSOR.V_MM_PATCH_MERGER, MODEL_TENSOR.V_MM_MERGER_FC1, MODEL_TENSOR.V_MM_MERGER_FC2, @@ -5645,6 +5654,7 @@ class VisionProjectorType: DOTS3NOTE_A = "dots3note_a" # audio DEEPSEEKOCR = "deepseekocr" DEEPSEEKOCR2 = "deepseekocr2" + DEEPSEEK4V = "deepseek4v" LFM2A = "lfm2a" # audio MUSIC_FLAMINGO = "musicflamingo" # audio GLM4V = "glm4v" diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 861acfe18..d644d502e 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -1476,6 +1476,7 @@ class TensorNameMap: ## Vision encoder MODEL_TENSOR.V_MMPROJ: ( + "aligner.w{bid}", # deepseek4v (w1 -> mm.1, w2 -> mm.2) "multi_modal_projector.linear_{bid}", "mm_projector.proj.linear_{bid}", # Kimi-K2.5 "visual.merger.mlp.{bid}", # qwen2vl @@ -1515,6 +1516,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_EMBD_PATCH: ( + "vision.patch_embed.proj", # deepseek4v "model.vision_tower.vision_model.embeddings.patch_embedding", # Granite4Vision "vision_tower.vision_model.embeddings.patch_embedding", "model.vision_tower.embeddings.patch_embedding", # minicpmv4_6 @@ -1570,6 +1572,7 @@ class TensorNameMap: # TODO: I think these should all be moved to mapping_cfg? MODEL_TENSOR.V_ENC_EMBD_IMGNL: ( + "image_newline", # deepseek4v "model.image_newline", # Deepseek-OCR, Granite4Vision "vit.perceive.image_newline", # HunyuanVL ), @@ -1580,6 +1583,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_QKV: ( + "vision.blocks.{bid}.attn.wqkv", # deepseek4v "visual.blocks.{bid}.attn.qkv", # qwen3vl "vision_tower.blocks.{bid}.attn.qkv", # dots.ocr "vision_encoder.blocks.{bid}.attn.qkv", # dots3note @@ -1667,6 +1671,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_INPUT_NORM: ( + "vision.blocks.{bid}.norm1", # deepseek4v "model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", "model.vision_tower.encoder.layers.{bid}.layer_norm1", # minicpmv4_6 @@ -1692,6 +1697,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_O: ( + "vision.blocks.{bid}.attn.wo", # deepseek4v "model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", "model.vision_tower.encoder.layers.{bid}.self_attn.out_proj", # minicpmv4_6 @@ -1723,6 +1729,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_POST_ATTN_NORM: ( + "vision.blocks.{bid}.norm2", # deepseek4v "model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", "model.vision_tower.encoder.layers.{bid}.layer_norm2", # minicpmv4_6 @@ -1749,6 +1756,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_FFN_UP: ( + "vision.blocks.{bid}.mlp.w1_up", # deepseek4v (split from fused w1) "vision_encoder.blocks.{bid}.mlp.fc3", # dots3note "model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", @@ -1775,6 +1783,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_FFN_GATE: ( + "vision.blocks.{bid}.mlp.w1_gate", # deepseek4v (split from fused w1) "vision_encoder.blocks.{bid}.mlp.fc1", # dots3note "vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf "vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral @@ -1784,6 +1793,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_FFN_DOWN: ( + "vision.blocks.{bid}.mlp.w2", # deepseek4v "vision_encoder.blocks.{bid}.mlp.fc2", # dots3note "model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", @@ -1869,6 +1879,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_POST_NORM: ( + "vision.norm", # deepseek4v "model.vision_tower.vision_model.post_layernorm", # Granite4Vision "vision_tower.vision_model.post_layernorm", "model.vision_tower.post_layernorm", # minicpmv4_6 @@ -1960,6 +1971,18 @@ class TensorNameMap: "v.token_embd.img_break", # for pixtral, this is a generated vector ), + MODEL_TENSOR.V_TOK_EMBD_IMG_START: ( + "image_start", # deepseek4v + ), + + MODEL_TENSOR.V_TOK_EMBD_IMG_END: ( + "image_end", # deepseek4v + ), + + MODEL_TENSOR.V_TOK_EMBD_IMG_PAD: ( + "image_pad", # deepseek4v + ), + MODEL_TENSOR.V_MM_PATCH_MERGER: ( "multi_modal_projector.patch_merger.merging_layer", # mistral small 3.1 - hf "patch_merger.merging_layer", # mistral diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index e60c9c878..907468e87 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -30,6 +30,7 @@ add_library(mtmd models/models.h models/cogvlm.cpp models/conformer.cpp + models/deepseek4v.cpp models/dots3note.cpp models/dotsocr.cpp models/exaone4_5.cpp diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index f6045093c..72148a4d9 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -153,6 +153,9 @@ #define TN_MM_MERGER_FC1 "mm.merger.fc1.%s" // minimax-m3 patch-merge MLP #define TN_MM_MERGER_FC2 "mm.merger.fc2.%s" #define TN_TOK_IMG_BREAK "v.token_embd.img_break" // pixtral +#define TN_TOK_IMG_START "v.token_embd.img_start" // deepseek4v +#define TN_TOK_IMG_END "v.token_embd.img_end" // deepseek4v +#define TN_TOK_IMG_PAD "v.token_embd.img_pad" // deepseek4v #define TN_TOK_GLM_BOI "adapter.boi" // glm-edge (these embeddings are not in text model) #define TN_TOK_GLM_EOI "adapter.eoi" // glm-edge (these embeddings are not in text model) #define TN_DEEPSTACK_NORM "v.deepstack.%d.norm.%s" // qwen3vl deepstack @@ -296,8 +299,8 @@ // hunyuanvl (shared GGUF tensor names) #define TN_MM_PRE_NORM "mm.pre_norm.%s" -#define TN_TOK_IMG_BEGIN "mm.image_begin" -#define TN_TOK_IMG_END "mm.image_end" +#define TN_MM_IMG_BEGIN "mm.image_begin" // note: legacy name, new models should use v.token_embd.* +#define TN_MM_IMG_END "mm.image_end" // note: legacy name, new models should use v.token_embd.* // deepseek-ocr #define TN_SAM_POS_EMBD "v.sam.pos_embd.%s" @@ -480,6 +483,7 @@ enum projector_type { PROJECTOR_TYPE_DOTS3NOTE_A, PROJECTOR_TYPE_DEEPSEEKOCR, PROJECTOR_TYPE_DEEPSEEKOCR2, + PROJECTOR_TYPE_DEEPSEEK4V, PROJECTOR_TYPE_LFM2A, PROJECTOR_TYPE_GLM4V, PROJECTOR_TYPE_YOUTUVL, @@ -544,6 +548,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_DOTS3NOTE_A, "dots3note_a"}, { PROJECTOR_TYPE_DEEPSEEKOCR, "deepseekocr"}, { PROJECTOR_TYPE_DEEPSEEKOCR2, "deepseekocr2"}, + { PROJECTOR_TYPE_DEEPSEEK4V, "deepseek4v"}, { PROJECTOR_TYPE_LFM2A, "lfm2a"}, { PROJECTOR_TYPE_GLM4V, "glm4v"}, { PROJECTOR_TYPE_YOUTUVL, "youtuvl"}, @@ -655,6 +660,9 @@ struct clip_image_f32 { // appends a learned newline (or EOI) token after the image // no model uses it now (Granite4 Vision moved to anyres), kept for future models bool add_newline = false; + // deepseek4v: number of leading IMAGE_PAD embeddings, aligns IMAGE_START to the LLM compressor ratio + // depends on the chunk position, set at tokenize time (see mtmd_tokenizer::add_media) + int32_t lead_pad = 0; // llava-next "anyres" tiling, used by Granite4 Vision // the whole grid is encoded and assembled in a single graph @@ -771,6 +779,22 @@ static inline void clip_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_ } } +// deepseek4v: layout of the LLM token block built from the aligner grid +struct dsv4_block_layout { + int rows; // grid rows, padded to an even count + int row_len; // grid width + 1 newline + int pad_last; // trailing pads + int n_out; // total block size, including lead pads and the start/end sentinels +}; +static inline dsv4_block_layout dsv4_get_block_layout(int n_llm_w, int n_llm_h, int lead_pad) { + dsv4_block_layout bl; + bl.rows = n_llm_h + (n_llm_h % 2); + bl.row_len = n_llm_w + 1; + bl.pad_last = (bl.rows / 2 * bl.row_len) % 2 * 2; + bl.n_out = lead_pad + 1 + bl.rows * bl.row_len + bl.pad_last + 1; + return bl; +} + // // logging // diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 060938d86..f737ccc24 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -100,6 +100,10 @@ struct clip_hparams { std::unordered_set wa_layer_indexes; // explicit layer indexes that use full attention (for irregular patterns like YoutuVL) std::vector wa_pattern_mode; // mimovl: per-layer window-attention mode + // deepseek4v: resize solver caps the LLM token count of the aligner grid + int32_t dsv4_max_n_token = 0; + int32_t dsv4_max_wh_ratio = 0; + // deepseek-ocr (sam) int32_t sam_n_layer = 0; int32_t sam_n_head = 0; @@ -724,6 +728,11 @@ struct clip_model { // pixtral, glm4v ggml_tensor * token_embd_img_break = nullptr; + + // deepseek4v sentinel embeddings (image_newline is reused for IMAGE_NEW_LINE) + ggml_tensor * token_embd_img_start = nullptr; + ggml_tensor * token_embd_img_end = nullptr; + ggml_tensor * token_embd_img_pad = nullptr; ggml_tensor * mm_patch_merger_w = nullptr; ggml_tensor * mm_patch_merger_b = nullptr; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 46f0437a7..f2e487534 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1037,6 +1037,10 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_DEEPSEEK4V: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_COGVLM: { builder = std::make_unique(ctx, img); @@ -1585,6 +1589,31 @@ struct clip_model_loader { hparams.set_limit_image_tokens(2, 4096); } } break; + case PROJECTOR_TYPE_DEEPSEEK4V: + { + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.image_pad_color = {127, 127, 127}; + hparams.rope_theta = 10000.0f; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + hparams.dsv4_max_n_token = 384; + hparams.dsv4_max_wh_ratio = 8; + const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; + // handle min/max token counts from CLI + if (hparams.custom_image_min_tokens > 0) { + hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area; + } + if (hparams.custom_image_max_tokens > 0) { + // the cap is on the whole token block, keep some room for the resize solver + hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16); + } + hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area; + // a small custom max token count also lowers the min-pixel upscale threshold + hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels); + // avoid OOM on warmup + const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token)); + hparams.set_warmup_n_tokens(warmup_side * warmup_side); + } break; case PROJECTOR_TYPE_GEMMA3: { // default value (used by all model sizes in gemma 3 family) @@ -2714,6 +2743,18 @@ struct clip_model_loader { model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); } break; + case PROJECTOR_TYPE_DEEPSEEK4V: + { + model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias")); + model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); + // sentinel token embeddings written into the output block + model.image_newline = get_tensor(TN_IMAGE_NEWLINE); + model.token_embd_img_start = get_tensor(TN_TOK_IMG_START); + model.token_embd_img_end = get_tensor(TN_TOK_IMG_END); + model.token_embd_img_pad = get_tensor(TN_TOK_IMG_PAD); + } break; case PROJECTOR_TYPE_PIXTRAL: { model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); @@ -3161,8 +3202,8 @@ struct clip_model_loader { model.mm_model_proj_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias")); model.mm_pre_norm_w = get_tensor(string_format(TN_MM_PRE_NORM, "weight")); model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight")); - model.mm_img_begin = get_tensor(TN_TOK_IMG_BEGIN); - model.mm_img_end = get_tensor(TN_TOK_IMG_END); + model.mm_img_begin = get_tensor(TN_MM_IMG_BEGIN); + model.mm_img_end = get_tensor(TN_MM_IMG_END); model.image_newline = get_tensor(TN_IMAGE_NEWLINE); model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR, false); } break; @@ -4150,6 +4191,13 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { int y_patch = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size; n_patches = x_patch * y_patch; } break; + case PROJECTOR_TYPE_DEEPSEEK4V: + { + const int out_patch_size = params.patch_size * params.n_merge; + const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size; + const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size; + n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out; + } break; case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_DOTS_OCR: case PROJECTOR_TYPE_DOTS3NOTE_V: @@ -5021,6 +5069,58 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { } set_input_i32("pos_w", pos_data); } break; + case PROJECTOR_TYPE_DEEPSEEK4V: + { + // set the 2D positions (mrope layout, only the first 2 channels are used) + int n_patches_per_row = image_size_width / patch_size; + std::vector positions(n_pos * 4, 0); + for (int i = 0; i < n_pos; i++) { + positions[i] = i / n_patches_per_row; // row + positions[n_pos + i] = i % n_patches_per_row; // col + } + set_input_i32("positions", positions); + + // token block layout index (see clip_graph_deepseek4v::build) + // rows [0, n_grid) are the aligner output, the sentinels follow + const int n_merge = hparams.n_merge; + const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge; + const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge; + const int n_grid = n_llm_w * n_llm_h; + const int idx_start = n_grid; + const int idx_end = n_grid + 1; + const int idx_newline = n_grid + 2; + const int idx_pad = n_grid + 3; + + const int lead_pad = imgs.entries[0].lead_pad; + const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad); + + std::vector idx; + idx.reserve(bl.n_out); + for (int i = 0; i < lead_pad; i++) { + idx.push_back(idx_pad); + } + idx.push_back(idx_start); + // pairs of adjacent rows are interleaved column-wise ("N-layout") + // ref: build_image_block in inference/image_processor.py + for (int t = 0; t < bl.rows * bl.row_len; t++) { + const int g = t / (2 * bl.row_len); + const int rem = t % (2 * bl.row_len); + const int c = rem / 2; // column + const int r = 2 * g + rem % 2; // row + if (r >= n_llm_h) { + idx.push_back(idx_pad); + } else if (c == n_llm_w) { + idx.push_back(idx_newline); + } else { + idx.push_back(r * n_llm_w + c); + } + } + for (int i = 0; i < bl.pad_last; i++) { + idx.push_back(idx_pad); + } + idx.push_back(idx_end); + set_input_i32("layout_idx", idx); + } break; case PROJECTOR_TYPE_GLM_EDGE: { // llava and other models @@ -5762,6 +5862,19 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n"); LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens); + // TEMP debugging (parity validation), will be removed before merge + // when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data] + const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS"); + if (dump_path && strcmp(dump_path, "1") != 0) { + FILE * f = fopen(dump_path, "wb"); + if (f) { + const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd }; + fwrite(hdr, sizeof(hdr), 1, f); + fwrite(emb_data.data(), sizeof(float), emb_data.size(), f); + fclose(f); + } + } + // Print first few values of first token LOG_INF("Token 0 (first 16 values): "); for (int i = 0; i < std::min((int64_t)16, n_embd); i++) { @@ -5866,6 +5979,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_KIMIK25: case PROJECTOR_TYPE_YASA2: + case PROJECTOR_TYPE_DEEPSEEK4V: return ctx->model.mm_2_w->ne[1]; case PROJECTOR_TYPE_HUNYUANVL: return ctx->model.mm_model_proj->ne[1]; diff --git a/tools/mtmd/models/deepseek4v.cpp b/tools/mtmd/models/deepseek4v.cpp new file mode 100644 index 000000000..ffe8f59d9 --- /dev/null +++ b/tools/mtmd/models/deepseek4v.cpp @@ -0,0 +1,102 @@ +#include "models.h" + +// DeepSeek-V4-Flash-Vision encoder (deepseek4v) +// +// native-resolution ViT (RMSNorm, SwiGLU, 2D RoPE, no CLS / learned pos-embd) +// then the "aligner": 3x3 patch merge (torch.nn.functional.unfold) + 2-layer GELU MLP +// +// the graph outputs the complete LLM token block, built from the aligner output and 4 learned sentinel embeddings: +// +// [PAD]*lead_pad [START] [PAD]*pad_last [END] +// +// each aligner row ends with a NEWLINE, an odd row count is padded with a full row of PADs +// pairs of adjacent rows are interleaved column-wise ("N-layout") +// the mapping is precomputed on CPU as the "layout_idx" input (see set_input in clip.cpp) +// +// ref: inference/vision.py and inference/image_processor.py in the HF repo + +ggml_cgraph * clip_graph_deepseek4v::build() { + const int n_merge = hparams.n_merge; + + // 2D input positions + ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches * 4); + ggml_set_name(positions, "positions"); + ggml_set_input(positions); + + int sections[4] = {d_head/4, d_head/4, 0, 0}; + auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + return ggml_rope_multi(ctx0, cur, positions, nullptr, + d_head/2, sections, GGML_ROPE_TYPE_VISION, + 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + }; + + ggml_tensor * inp = build_inp(); + ggml_tensor * cur = build_vit( + inp, n_patches, + NORM_TYPE_RMS, + hparams.ffn_op, + nullptr, // no learned pos embd + add_pos); + cb(cur, "vit_out", -1); + + // aligner patch merge: zero-pad the patch grid to a multiple of n_merge + // then F.unfold == im2col with a dummy kernel (same trick as pixtral) + { + cur = ggml_reshape_3d(ctx0, cur, n_embd, n_patches_x, n_patches_y); + cur = ggml_permute(ctx0, cur, 2, 0, 1, 3); // [x, y, n_embd] + cur = ggml_cont(ctx0, cur); + + const int pad_x = (n_merge - n_patches_x % n_merge) % n_merge; + const int pad_y = (n_merge - n_patches_y % n_merge) % n_merge; + if (pad_x || pad_y) { + cur = ggml_pad(ctx0, cur, pad_x, pad_y, 0, 0); + } + + ggml_tensor * kernel = ggml_view_3d(ctx0, cur, n_merge, n_merge, cur->ne[2], 0, 0, 0); + cur = ggml_im2col(ctx0, kernel, cur, n_merge, n_merge, 0, 0, 1, 1, true, inp->type); + cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]); + + // aligner MLP (F.gelu in the reference == erf-based gelu) + cur = build_ffn(cur, + model.mm_1_w, model.mm_1_b, + nullptr, nullptr, + model.mm_2_w, model.mm_2_b, + FFN_GELU_ERF, + -1); + cb(cur, "aligner_out", -1); + } + + // assemble the token block: append the sentinel embeddings as extra rows + // then reorder everything with the precomputed layout index + { + const int64_t n_embd_out = cur->ne[0]; + const int64_t n_grid = cur->ne[1]; // n_llm_w * n_llm_h + + // rows n_grid + 0..3, keep in sync with the index computation in set_input + ggml_tensor * sentinels[] = { + model.token_embd_img_start, + model.token_embd_img_end, + model.image_newline, + model.token_embd_img_pad, + }; + for (ggml_tensor * tok : sentinels) { + cur = ggml_concat(ctx0, cur, ggml_reshape_2d(ctx0, tok, n_embd_out, 1), 1); + } + + const int n_llm_w = CLIP_ALIGN(n_patches_x, n_merge) / n_merge; + const int n_llm_h = CLIP_ALIGN(n_patches_y, n_merge) / n_merge; + const int n_out = dsv4_get_block_layout(n_llm_w, n_llm_h, img.lead_pad).n_out; + GGML_ASSERT(n_grid == n_llm_w * n_llm_h); + + ggml_tensor * layout_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_out); + ggml_set_name(layout_idx, "layout_idx"); + ggml_set_input(layout_idx); + + cur = ggml_get_rows(ctx0, cur, layout_idx); + } + + // build the graph + ggml_build_forward_expand(gf, cur); + + return gf; +} diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 10546fa5d..5945c6d92 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -34,6 +34,11 @@ struct clip_graph_pixtral : clip_graph { ggml_cgraph * build() override; }; +struct clip_graph_deepseek4v : clip_graph { + clip_graph_deepseek4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; +}; + struct clip_graph_qwen2vl : clip_graph { clip_graph_qwen2vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 0dda8770f..65c24f4d4 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -1092,6 +1092,108 @@ clip_image_size mtmd_image_preprocessor_deepseekocr::find_closest_aspect_ratio( return best_ratio; } +// +// DeepSeek-V4-Flash-Vision (deepseek4v) +// +// port of load_image / safe_resize / solve_resize_ratio / grid_tokens from inference/image_processor.py +// the resize solver picks the largest target size (multiple of patch_size) whose LLM token block fits max_n_token +// + +// ref: grid_tokens() +mtmd_image_preprocessor_deepseek4v::grid_info mtmd_image_preprocessor_deepseek4v::grid_tokens(int best_height, int best_width, int patch_size, int r) { + grid_info g; + g.n_llm_h = ((best_height / patch_size) + r - 1) / r; + g.n_llm_w = ((best_width / patch_size) + r - 1) / r; + g.n_tokens = dsv4_get_block_layout(g.n_llm_w, g.n_llm_h, 0).n_out; + return g; +} + +// ref: solve_resize_ratio() +void mtmd_image_preprocessor_deepseek4v::solve_resize_ratio(int height, int width, int p, int r, int max_n_token, + int & best_height, int & best_width) { + const double ratio = (double) height / width; + const double max_w_f = std::sqrt((max_n_token - 2) / ratio + 0.25) - 0.5; + const double max_h_f = max_w_f * ratio; + if (max_w_f < 1.0) { + const int max_w = 1; + int max_h = (max_n_token - 2) / (max_w + 1); + if (max_h % 2 == 1) { + max_h -= 1; + } + best_width = max_w * p * r; + best_height = max_h * p * r; + } else if (max_h_f < 2.0) { + const int max_h = 2; + // guard tiny budgets; cannot be hit with the current lower bound on max_n_token + const int max_w = std::max(((max_n_token - 2) / max_h) - 1, 2); + best_width = max_w * p * r; + best_height = max_h * p * r; + } else { + const int max_w_i = (int) std::floor(max_w_f); + int max_h_i = (int) std::floor(max_h_f); + if (max_h_i % 2 == 1) { + max_h_i -= 1; + } + const double beta = std::min( + (double) max_w_i * p * r / width, + (double) max_h_i * p * r / height); + best_width = (int) std::floor(width * beta / p) * p; + best_height = (int) std::floor(height * beta / p) * p; + } +} + +// ref: safe_resize() +void mtmd_image_preprocessor_deepseek4v::safe_resize(int height, int width, int & best_height, int & best_width, + int p, int r, int max_n_token) { + max_n_token -= 4 - 1; // reserve room for the position-dependent lead pads (COMPRESS_PAD_TO - 1) + grid_info g = grid_tokens(best_height, best_width, p, r); + int budget = max_n_token; + while (g.n_tokens > max_n_token) { + solve_resize_ratio(height, width, p, r, budget, best_height, best_width); + g = grid_tokens(best_height, best_width, p, r); + budget -= 1; + } +} + +// ref: load_image() +mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip_image_u8 & img) { + mtmd_image_preproc_out out; + + const int p = hparams.patch_size; + const int r = hparams.n_merge; + const int max_n_token = hparams.dsv4_max_n_token; + const int max_wh = hparams.dsv4_max_wh_ratio; + + const clip_image_size orig = img.get_size(); + int width = orig.width; + int height = orig.height; + if (max_wh > 0 && width > height * max_wh) { + width = height * max_wh; + } + if (hparams.image_min_pixels > 0 && width * height > 0 + && width * height < hparams.image_min_pixels) { + const double up = std::sqrt((double) hparams.image_min_pixels / ((double) width * height)); + width = (int) (width * up); + height = (int) (height * up); + } + int best_width = CLIP_ALIGN(width, p); + int best_height = CLIP_ALIGN(height, p); + safe_resize(height, width, best_height, best_width, p, r, max_n_token); + + clip_image_u8 resized; + if (max_wh > 0 && orig.width >= max_wh * orig.height) { + // extreme aspect ratio: plain stretch resize, no padding + img_tool::resize(img, resized, {best_width, best_height}, hparams.image_resize_algo, PAD_NONE); + } else { + // aspect-preserving resize + centered padding (PIL ImageOps.pad) + img_tool::resize(img, resized, {best_width, best_height}, hparams.image_resize_algo, + PAD_NEAREST, hparams.image_pad_color); + } + + out.append(hparams, resized); + return out; +} + mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) { mtmd_image_preproc_out output; int grid_w = 0; diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index 732e27379..8758c6647 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -129,6 +129,22 @@ struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor { mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; }; +// ref: inference/image_processor.py in the HF repo (DeepSeek-V4-Flash-Vision) +struct mtmd_image_preprocessor_deepseek4v : mtmd_image_preprocessor { + mtmd_image_preprocessor_deepseek4v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; + +private: + struct grid_info { + int n_llm_h; + int n_llm_w; + int n_tokens; // token count of the block (incl. newline/pad rows and start/end, excl. lead pads) + }; + static grid_info grid_tokens(int best_height, int best_width, int patch_size, int r); + static void solve_resize_ratio(int height, int width, int p, int r, int max_n_token, int & best_height, int & best_width); + static void safe_resize(int height, int width, int & best_height, int & best_width, int p, int r, int max_n_token); +}; + // custom llava-uhd slicing logic for MiniCPM-V struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd { using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 5b306180d..d2b88b1e4 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -27,7 +27,7 @@ #include // remember to bump this if the serialization format changes -#define MTMD_SERIALIZATION_VERSION 1 +#define MTMD_SERIALIZATION_VERSION 2 struct mtmd_serialization { // note: using 64-bit here for future-proofing @@ -105,12 +105,14 @@ void clip_image_f32::serialize(mtmd_serialization & ser) const { // note: buf is intentionally NOT serialized; the loaded clip_image_f32 will always be a placeholder ser.write(add_viewsep); ser.write(add_newline); + ser.write(lead_pad); ser.write((int32_t)nx_); ser.write((int32_t)ny_); } void clip_image_f32::deserialize(mtmd_serialization & ser) { add_viewsep = ser.read(); add_newline = ser.read(); + lead_pad = ser.read(); nx_ = ser.read(); ny_ = ser.read(); buf.clear(); // always a placeholder after loading @@ -824,6 +826,11 @@ struct mtmd_context { img_end = "<|im_end|>"; image_preproc = std::make_unique(ctx_v); } break; + case PROJECTOR_TYPE_DEEPSEEK4V: + { + // no vocab tokens are added; the start/end/newline markers are learned embeddings emitted by the encoder + image_preproc = std::make_unique(ctx_v); + } break; case PROJECTOR_TYPE_DOTS_OCR: case PROJECTOR_TYPE_DOTS3NOTE_V: { @@ -1451,6 +1458,18 @@ struct mtmd_tokenizer { return 2; } + if (ctx->proj_type_v() == PROJECTOR_TYPE_DEEPSEEK4V) { + // the text model perceives input in blocks of N tokens (N = COMPRESS_PAD_TO = 4, same as the CSA compress ratio) + // image need to be aligned to block size, while adding IMAGE_PAD embeddings to the beginning + // TODO @ngxson : maybe refactor this in the future + constexpr int32_t align = 4; + size_t n_past = 0; + for (const auto & e : cur.entries) { + n_past += mtmd_input_chunk_get_n_tokens(&e); + } + preproc_out.entries[0].lead_pad = align - 1 - (int32_t)(n_past % align); + } + size_t n_tokens = 0; for (auto & e : preproc_out.entries) { n_tokens += clip_n_output_tokens(ctx->ctx_v, &e);