Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: support DeepSeek-V4-Flash-Vision-Exp (#28133)

Browse Source 
* mtmd: support DeepSeek-V4-Flash-Vision-Exp

* handle min/max token counts from CLI

* rm debugging

* use GGML_ROPE_TYPE_VISION

* nits

* apply review comments

* correct token count
This commit is contained in:
Xuan-Son Nguyen
2026-09-02 16:43:43 +02:00
committed by GitHub
parent 8e93a9773b
commit 7798007a29
13 changed files with 504 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/CMakeLists.txt
+1
View File
@@ -30,6 +30,7 @@ add_library(mtmd
models/models.h
models/cogvlm.cpp
models/conformer.cpp
models/deepseek4v.cpp
models/dots3note.cpp
models/dotsocr.cpp
models/exaone4_5.cpp
tools/mtmd/clip-impl.h
+26 -2
View File
@@ -153,6 +153,9 @@
#define TN_MM_MERGER_FC1 "mm.merger.fc1.%s" // minimax-m3 patch-merge MLP
#define TN_MM_MERGER_FC2 "mm.merger.fc2.%s"
#define TN_TOK_IMG_BREAK "v.token_embd.img_break" // pixtral
#define TN_TOK_IMG_START "v.token_embd.img_start" // deepseek4v
#define TN_TOK_IMG_END "v.token_embd.img_end" // deepseek4v
#define TN_TOK_IMG_PAD "v.token_embd.img_pad" // deepseek4v
#define TN_TOK_GLM_BOI "adapter.boi" // glm-edge (these embeddings are not in text model)
#define TN_TOK_GLM_EOI "adapter.eoi" // glm-edge (these embeddings are not in text model)
#define TN_DEEPSTACK_NORM "v.deepstack.%d.norm.%s" // qwen3vl deepstack
@@ -296,8 +299,8 @@
// hunyuanvl (shared GGUF tensor names)
#define TN_MM_PRE_NORM "mm.pre_norm.%s"
#define TN_TOK_IMG_BEGIN "mm.image_begin"
#define TN_TOK_IMG_END "mm.image_end"
#define TN_MM_IMG_BEGIN "mm.image_begin" // note: legacy name, new models should use v.token_embd.*
#define TN_MM_IMG_END "mm.image_end" // note: legacy name, new models should use v.token_embd.*
// deepseek-ocr
#define TN_SAM_POS_EMBD "v.sam.pos_embd.%s"
@@ -480,6 +483,7 @@ enum projector_type {
PROJECTOR_TYPE_DOTS3NOTE_A,
PROJECTOR_TYPE_DEEPSEEKOCR,
PROJECTOR_TYPE_DEEPSEEKOCR2,
PROJECTOR_TYPE_DEEPSEEK4V,
PROJECTOR_TYPE_LFM2A,
PROJECTOR_TYPE_GLM4V,
PROJECTOR_TYPE_YOUTUVL,
@@ -544,6 +548,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_DOTS3NOTE_A, "dots3note_a"},
{ PROJECTOR_TYPE_DEEPSEEKOCR, "deepseekocr"},
{ PROJECTOR_TYPE_DEEPSEEKOCR2, "deepseekocr2"},
{ PROJECTOR_TYPE_DEEPSEEK4V, "deepseek4v"},
{ PROJECTOR_TYPE_LFM2A, "lfm2a"},
{ PROJECTOR_TYPE_GLM4V, "glm4v"},
{ PROJECTOR_TYPE_YOUTUVL, "youtuvl"},
@@ -655,6 +660,9 @@ struct clip_image_f32 {
// appends a learned newline (or EOI) token after the image
// no model uses it now (Granite4 Vision moved to anyres), kept for future models
bool add_newline = false;
// deepseek4v: number of leading IMAGE_PAD embeddings, aligns IMAGE_START to the LLM compressor ratio
// depends on the chunk position, set at tokenize time (see mtmd_tokenizer::add_media)
int32_t lead_pad = 0;
// llava-next "anyres" tiling, used by Granite4 Vision
// the whole grid is encoded and assembled in a single graph
@@ -771,6 +779,22 @@ static inline void clip_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_
}
}
// deepseek4v: layout of the LLM token block built from the aligner grid
struct dsv4_block_layout {
int rows; // grid rows, padded to an even count
int row_len; // grid width + 1 newline
int pad_last; // trailing pads
int n_out; // total block size, including lead pads and the start/end sentinels
};
static inline dsv4_block_layout dsv4_get_block_layout(int n_llm_w, int n_llm_h, int lead_pad) {
dsv4_block_layout bl;
bl.rows = n_llm_h + (n_llm_h % 2);
bl.row_len = n_llm_w + 1;
bl.pad_last = (bl.rows / 2 * bl.row_len) % 2 * 2;
bl.n_out = lead_pad + 1 + bl.rows * bl.row_len + bl.pad_last + 1;
return bl;
}
//
// logging
//
tools/mtmd/clip-model.h
+9
View File
@@ -100,6 +100,10 @@ struct clip_hparams {
std::unordered_set<int32_t> wa_layer_indexes; // explicit layer indexes that use full attention (for irregular patterns like YoutuVL)
std::vector<int32_t> wa_pattern_mode; // mimovl: per-layer window-attention mode
// deepseek4v: resize solver caps the LLM token count of the aligner grid
int32_t dsv4_max_n_token = 0;
int32_t dsv4_max_wh_ratio = 0;
// deepseek-ocr (sam)
int32_t sam_n_layer = 0;
int32_t sam_n_head = 0;
@@ -724,6 +728,11 @@ struct clip_model {
// pixtral, glm4v
ggml_tensor * token_embd_img_break = nullptr;
// deepseek4v sentinel embeddings (image_newline is reused for IMAGE_NEW_LINE)
ggml_tensor * token_embd_img_start = nullptr;
ggml_tensor * token_embd_img_end = nullptr;
ggml_tensor * token_embd_img_pad = nullptr;
ggml_tensor * mm_patch_merger_w = nullptr;
ggml_tensor * mm_patch_merger_b = nullptr;
tools/mtmd/clip.cpp
+116 -2
View File
@@ -1037,6 +1037,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_kimik25>(ctx, img);
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
builder = std::make_unique<clip_graph_deepseek4v>(ctx, img);
} break;
case PROJECTOR_TYPE_COGVLM:
{
builder = std::make_unique<clip_graph_cogvlm>(ctx, img);
@@ -1585,6 +1589,31 @@ struct clip_model_loader {
hparams.set_limit_image_tokens(2, 4096);
}
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.image_pad_color = {127, 127, 127};
hparams.rope_theta = 10000.0f;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
hparams.dsv4_max_n_token = 384;
hparams.dsv4_max_wh_ratio = 8;
const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
// handle min/max token counts from CLI
if (hparams.custom_image_min_tokens > 0) {
hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area;
}
if (hparams.custom_image_max_tokens > 0) {
// the cap is on the whole token block, keep some room for the resize solver
hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16);
}
hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area;
// a small custom max token count also lowers the min-pixel upscale threshold
hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels);
// avoid OOM on warmup
const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token));
hparams.set_warmup_n_tokens(warmup_side * warmup_side);
} break;
case PROJECTOR_TYPE_GEMMA3:
{
// default value (used by all model sizes in gemma 3 family)
@@ -2714,6 +2743,18 @@ struct clip_model_loader {
model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
// sentinel token embeddings written into the output block
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
model.token_embd_img_start = get_tensor(TN_TOK_IMG_START);
model.token_embd_img_end = get_tensor(TN_TOK_IMG_END);
model.token_embd_img_pad = get_tensor(TN_TOK_IMG_PAD);
} break;
case PROJECTOR_TYPE_PIXTRAL:
{
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
@@ -3161,8 +3202,8 @@ struct clip_model_loader {
model.mm_model_proj_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias"));
model.mm_pre_norm_w = get_tensor(string_format(TN_MM_PRE_NORM, "weight"));
model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight"));
model.mm_img_begin = get_tensor(TN_TOK_IMG_BEGIN);
model.mm_img_end = get_tensor(TN_TOK_IMG_END);
model.mm_img_begin = get_tensor(TN_MM_IMG_BEGIN);
model.mm_img_end = get_tensor(TN_MM_IMG_END);
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR, false);
} break;
@@ -4150,6 +4191,13 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
int y_patch = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
n_patches = x_patch * y_patch;
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
const int out_patch_size = params.patch_size * params.n_merge;
const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size;
const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out;
} break;
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
@@ -5021,6 +5069,58 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
set_input_i32("pos_w", pos_data);
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
// set the 2D positions (mrope layout, only the first 2 channels are used)
int n_patches_per_row = image_size_width / patch_size;
std::vector<int32_t> positions(n_pos * 4, 0);
for (int i = 0; i < n_pos; i++) {
positions[i] = i / n_patches_per_row; // row
positions[n_pos + i] = i % n_patches_per_row; // col
}
set_input_i32("positions", positions);
// token block layout index (see clip_graph_deepseek4v::build)
// rows [0, n_grid) are the aligner output, the sentinels follow
const int n_merge = hparams.n_merge;
const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge;
const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge;
const int n_grid = n_llm_w * n_llm_h;
const int idx_start = n_grid;
const int idx_end = n_grid + 1;
const int idx_newline = n_grid + 2;
const int idx_pad = n_grid + 3;
const int lead_pad = imgs.entries[0].lead_pad;
const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad);
std::vector<int32_t> idx;
idx.reserve(bl.n_out);
for (int i = 0; i < lead_pad; i++) {
idx.push_back(idx_pad);
}
idx.push_back(idx_start);
// pairs of adjacent rows are interleaved column-wise ("N-layout")
// ref: build_image_block in inference/image_processor.py
for (int t = 0; t < bl.rows * bl.row_len; t++) {
const int g = t / (2 * bl.row_len);
const int rem = t % (2 * bl.row_len);
const int c = rem / 2; // column
const int r = 2 * g + rem % 2; // row
if (r >= n_llm_h) {
idx.push_back(idx_pad);
} else if (c == n_llm_w) {
idx.push_back(idx_newline);
} else {
idx.push_back(r * n_llm_w + c);
}
}
for (int i = 0; i < bl.pad_last; i++) {
idx.push_back(idx_pad);
}
idx.push_back(idx_end);
set_input_i32("layout_idx", idx);
} break;
case PROJECTOR_TYPE_GLM_EDGE:
{
// llava and other models
@@ -5762,6 +5862,19 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n");
LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens);
// TEMP debugging (parity validation), will be removed before merge
// when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data]
const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS");
if (dump_path && strcmp(dump_path, "1") != 0) {
FILE * f = fopen(dump_path, "wb");
if (f) {
const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd };
fwrite(hdr, sizeof(hdr), 1, f);
fwrite(emb_data.data(), sizeof(float), emb_data.size(), f);
fclose(f);
}
}
// Print first few values of first token
LOG_INF("Token 0 (first 16 values): ");
for (int i = 0; i < std::min((int64_t)16, n_embd); i++) {
@@ -5866,6 +5979,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_KIMIK25:
case PROJECTOR_TYPE_YASA2:
case PROJECTOR_TYPE_DEEPSEEK4V:
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_HUNYUANVL:
return ctx->model.mm_model_proj->ne[1];
tools/mtmd/models/deepseek4v.cpp
+102
View File
@@ -0,0 +1,102 @@
#include "models.h"
// DeepSeek-V4-Flash-Vision encoder (deepseek4v)
//
// native-resolution ViT (RMSNorm, SwiGLU, 2D RoPE, no CLS / learned pos-embd)
// then the "aligner": 3x3 patch merge (torch.nn.functional.unfold) + 2-layer GELU MLP
//
// the graph outputs the complete LLM token block, built from the aligner output and 4 learned sentinel embeddings:
//
// [PAD]*lead_pad [START] <interleaved rows> [PAD]*pad_last [END]
//
// each aligner row ends with a NEWLINE, an odd row count is padded with a full row of PADs
// pairs of adjacent rows are interleaved column-wise ("N-layout")
// the mapping is precomputed on CPU as the "layout_idx" input (see set_input in clip.cpp)
//
// ref: inference/vision.py and inference/image_processor.py in the HF repo
ggml_cgraph * clip_graph_deepseek4v::build() {
const int n_merge = hparams.n_merge;
// 2D input positions
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches * 4);
ggml_set_name(positions, "positions");
ggml_set_input(positions);
int sections[4] = {d_head/4, d_head/4, 0, 0};
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
return ggml_rope_multi(ctx0, cur, positions, nullptr,
d_head/2, sections, GGML_ROPE_TYPE_VISION,
0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
};
ggml_tensor * inp = build_inp();
ggml_tensor * cur = build_vit(
inp, n_patches,
NORM_TYPE_RMS,
hparams.ffn_op,
nullptr, // no learned pos embd
add_pos);
cb(cur, "vit_out", -1);
// aligner patch merge: zero-pad the patch grid to a multiple of n_merge
// then F.unfold == im2col with a dummy kernel (same trick as pixtral)
{
cur = ggml_reshape_3d(ctx0, cur, n_embd, n_patches_x, n_patches_y);
cur = ggml_permute(ctx0, cur, 2, 0, 1, 3); // [x, y, n_embd]
cur = ggml_cont(ctx0, cur);
const int pad_x = (n_merge - n_patches_x % n_merge) % n_merge;
const int pad_y = (n_merge - n_patches_y % n_merge) % n_merge;
if (pad_x || pad_y) {
cur = ggml_pad(ctx0, cur, pad_x, pad_y, 0, 0);
}
ggml_tensor * kernel = ggml_view_3d(ctx0, cur, n_merge, n_merge, cur->ne[2], 0, 0, 0);
cur = ggml_im2col(ctx0, kernel, cur, n_merge, n_merge, 0, 0, 1, 1, true, inp->type);
cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]);
// aligner MLP (F.gelu in the reference == erf-based gelu)
cur = build_ffn(cur,
model.mm_1_w, model.mm_1_b,
nullptr, nullptr,
model.mm_2_w, model.mm_2_b,
FFN_GELU_ERF,
-1);
cb(cur, "aligner_out", -1);
}
// assemble the token block: append the sentinel embeddings as extra rows
// then reorder everything with the precomputed layout index
{
const int64_t n_embd_out = cur->ne[0];
const int64_t n_grid = cur->ne[1]; // n_llm_w * n_llm_h
// rows n_grid + 0..3, keep in sync with the index computation in set_input
ggml_tensor * sentinels[] = {
model.token_embd_img_start,
model.token_embd_img_end,
model.image_newline,
model.token_embd_img_pad,
};
for (ggml_tensor * tok : sentinels) {
cur = ggml_concat(ctx0, cur, ggml_reshape_2d(ctx0, tok, n_embd_out, 1), 1);
}
const int n_llm_w = CLIP_ALIGN(n_patches_x, n_merge) / n_merge;
const int n_llm_h = CLIP_ALIGN(n_patches_y, n_merge) / n_merge;
const int n_out = dsv4_get_block_layout(n_llm_w, n_llm_h, img.lead_pad).n_out;
GGML_ASSERT(n_grid == n_llm_w * n_llm_h);
ggml_tensor * layout_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_out);
ggml_set_name(layout_idx, "layout_idx");
ggml_set_input(layout_idx);
cur = ggml_get_rows(ctx0, cur, layout_idx);
}
// build the graph
ggml_build_forward_expand(gf, cur);
return gf;
}
tools/mtmd/models/models.h
+5
View File
@@ -34,6 +34,11 @@ struct clip_graph_pixtral : clip_graph {
ggml_cgraph * build() override;
};
struct clip_graph_deepseek4v : clip_graph {
clip_graph_deepseek4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct clip_graph_qwen2vl : clip_graph {
clip_graph_qwen2vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
tools/mtmd/mtmd-image.cpp
+102
View File
@@ -1092,6 +1092,108 @@ clip_image_size mtmd_image_preprocessor_deepseekocr::find_closest_aspect_ratio(
return best_ratio;
}
//
// DeepSeek-V4-Flash-Vision (deepseek4v)
//
// port of load_image / safe_resize / solve_resize_ratio / grid_tokens from inference/image_processor.py
// the resize solver picks the largest target size (multiple of patch_size) whose LLM token block fits max_n_token
//
// ref: grid_tokens()
mtmd_image_preprocessor_deepseek4v::grid_info mtmd_image_preprocessor_deepseek4v::grid_tokens(int best_height, int best_width, int patch_size, int r) {
grid_info g;
g.n_llm_h = ((best_height / patch_size) + r - 1) / r;
g.n_llm_w = ((best_width / patch_size) + r - 1) / r;
g.n_tokens = dsv4_get_block_layout(g.n_llm_w, g.n_llm_h, 0).n_out;
return g;
}
// ref: solve_resize_ratio()
void mtmd_image_preprocessor_deepseek4v::solve_resize_ratio(int height, int width, int p, int r, int max_n_token,
int & best_height, int & best_width) {
const double ratio = (double) height / width;
const double max_w_f = std::sqrt((max_n_token - 2) / ratio + 0.25) - 0.5;
const double max_h_f = max_w_f * ratio;
if (max_w_f < 1.0) {
const int max_w = 1;
int max_h = (max_n_token - 2) / (max_w + 1);
if (max_h % 2 == 1) {
max_h -= 1;
}
best_width = max_w * p * r;
best_height = max_h * p * r;
} else if (max_h_f < 2.0) {
const int max_h = 2;
// guard tiny budgets; cannot be hit with the current lower bound on max_n_token
const int max_w = std::max(((max_n_token - 2) / max_h) - 1, 2);
best_width = max_w * p * r;
best_height = max_h * p * r;
} else {
const int max_w_i = (int) std::floor(max_w_f);
int max_h_i = (int) std::floor(max_h_f);
if (max_h_i % 2 == 1) {
max_h_i -= 1;
}
const double beta = std::min(
(double) max_w_i * p * r / width,
(double) max_h_i * p * r / height);
best_width = (int) std::floor(width * beta / p) * p;
best_height = (int) std::floor(height * beta / p) * p;
}
}
// ref: safe_resize()
void mtmd_image_preprocessor_deepseek4v::safe_resize(int height, int width, int & best_height, int & best_width,
int p, int r, int max_n_token) {
max_n_token -= 4 - 1; // reserve room for the position-dependent lead pads (COMPRESS_PAD_TO - 1)
grid_info g = grid_tokens(best_height, best_width, p, r);
int budget = max_n_token;
while (g.n_tokens > max_n_token) {
solve_resize_ratio(height, width, p, r, budget, best_height, best_width);
g = grid_tokens(best_height, best_width, p, r);
budget -= 1;
}
}
// ref: load_image()
mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out out;
const int p = hparams.patch_size;
const int r = hparams.n_merge;
const int max_n_token = hparams.dsv4_max_n_token;
const int max_wh = hparams.dsv4_max_wh_ratio;
const clip_image_size orig = img.get_size();
int width = orig.width;
int height = orig.height;
if (max_wh > 0 && width > height * max_wh) {
width = height * max_wh;
}
if (hparams.image_min_pixels > 0 && width * height > 0
&& width * height < hparams.image_min_pixels) {
const double up = std::sqrt((double) hparams.image_min_pixels / ((double) width * height));
width = (int) (width * up);
height = (int) (height * up);
}
int best_width = CLIP_ALIGN(width, p);
int best_height = CLIP_ALIGN(height, p);
safe_resize(height, width, best_height, best_width, p, r, max_n_token);
clip_image_u8 resized;
if (max_wh > 0 && orig.width >= max_wh * orig.height) {
// extreme aspect ratio: plain stretch resize, no padding
img_tool::resize(img, resized, {best_width, best_height}, hparams.image_resize_algo, PAD_NONE);
} else {
// aspect-preserving resize + centered padding (PIL ImageOps.pad)
img_tool::resize(img, resized, {best_width, best_height}, hparams.image_resize_algo,
PAD_NEAREST, hparams.image_pad_color);
}
out.append(hparams, resized);
return out;
}
mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out output;
int grid_w = 0;
tools/mtmd/mtmd-image.h
+16
View File
@@ -129,6 +129,22 @@ struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor {
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};
// ref: inference/image_processor.py in the HF repo (DeepSeek-V4-Flash-Vision)
struct mtmd_image_preprocessor_deepseek4v : mtmd_image_preprocessor {
mtmd_image_preprocessor_deepseek4v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
private:
struct grid_info {
int n_llm_h;
int n_llm_w;
int n_tokens; // token count of the block (incl. newline/pad rows and start/end, excl. lead pads)
};
static grid_info grid_tokens(int best_height, int best_width, int patch_size, int r);
static void solve_resize_ratio(int height, int width, int p, int r, int max_n_token, int & best_height, int & best_width);
static void safe_resize(int height, int width, int & best_height, int & best_width, int p, int r, int max_n_token);
};
// custom llava-uhd slicing logic for MiniCPM-V
struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd {
using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;
tools/mtmd/mtmd.cpp
+20 -1
View File
@@ -27,7 +27,7 @@
#include <vector>
// remember to bump this if the serialization format changes
#define MTMD_SERIALIZATION_VERSION 1
#define MTMD_SERIALIZATION_VERSION 2
struct mtmd_serialization {
// note: using 64-bit here for future-proofing
@@ -105,12 +105,14 @@ void clip_image_f32::serialize(mtmd_serialization & ser) const {
// note: buf is intentionally NOT serialized; the loaded clip_image_f32 will always be a placeholder
ser.write(add_viewsep);
ser.write(add_newline);
ser.write(lead_pad);
ser.write((int32_t)nx_);
ser.write((int32_t)ny_);
}
void clip_image_f32::deserialize(mtmd_serialization & ser) {
add_viewsep = ser.read<bool>();
add_newline = ser.read<bool>();
lead_pad = ser.read<int32_t>();
nx_ = ser.read<int32_t>();
ny_ = ser.read<int32_t>();
buf.clear(); // always a placeholder after loading
@@ -824,6 +826,11 @@ struct mtmd_context {
img_end = "<|im_end|>";
image_preproc = std::make_unique<mtmd_image_preprocessor_longest_edge>(ctx_v);
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
// no vocab tokens are added; the start/end/newline markers are learned embeddings emitted by the encoder
image_preproc = std::make_unique<mtmd_image_preprocessor_deepseek4v>(ctx_v);
} break;
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
{
@@ -1451,6 +1458,18 @@ struct mtmd_tokenizer {
return 2;
}
if (ctx->proj_type_v() == PROJECTOR_TYPE_DEEPSEEK4V) {
// the text model perceives input in blocks of N tokens (N = COMPRESS_PAD_TO = 4, same as the CSA compress ratio)
// image need to be aligned to block size, while adding IMAGE_PAD embeddings to the beginning
// TODO @ngxson : maybe refactor this in the future
constexpr int32_t align = 4;
size_t n_past = 0;
for (const auto & e : cur.entries) {
n_past += mtmd_input_chunk_get_n_tokens(&e);
}
preproc_out.entries[0].lead_pad = align - 1 - (int32_t)(n_past % align);
}
size_t n_tokens = 0;
for (auto & e : preproc_out.entries) {
n_tokens += clip_n_output_tokens(ctx->ctx_v, &e);