Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: support DeepSeek-V4-Flash-Vision-Exp (#28133)

Browse Source 
* mtmd: support DeepSeek-V4-Flash-Vision-Exp

* handle min/max token counts from CLI

* rm debugging

* use GGML_ROPE_TYPE_VISION

* nits

* apply review comments

* correct token count
This commit is contained in:
Xuan-Son Nguyen
2026-09-02 16:43:43 +02:00
committed by GitHub
parent 8e93a9773b
commit 7798007a29
13 changed files with 504 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip.cpp
+116 -2
View File
@@ -1037,6 +1037,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_kimik25>(ctx, img);
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
builder = std::make_unique<clip_graph_deepseek4v>(ctx, img);
} break;
case PROJECTOR_TYPE_COGVLM:
{
builder = std::make_unique<clip_graph_cogvlm>(ctx, img);
@@ -1585,6 +1589,31 @@ struct clip_model_loader {
hparams.set_limit_image_tokens(2, 4096);
}
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.image_pad_color = {127, 127, 127};
hparams.rope_theta = 10000.0f;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
hparams.dsv4_max_n_token = 384;
hparams.dsv4_max_wh_ratio = 8;
const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
// handle min/max token counts from CLI
if (hparams.custom_image_min_tokens > 0) {
hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area;
}
if (hparams.custom_image_max_tokens > 0) {
// the cap is on the whole token block, keep some room for the resize solver
hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16);
}
hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area;
// a small custom max token count also lowers the min-pixel upscale threshold
hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels);
// avoid OOM on warmup
const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token));
hparams.set_warmup_n_tokens(warmup_side * warmup_side);
} break;
case PROJECTOR_TYPE_GEMMA3:
{
// default value (used by all model sizes in gemma 3 family)
@@ -2714,6 +2743,18 @@ struct clip_model_loader {
model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
// sentinel token embeddings written into the output block
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
model.token_embd_img_start = get_tensor(TN_TOK_IMG_START);
model.token_embd_img_end = get_tensor(TN_TOK_IMG_END);
model.token_embd_img_pad = get_tensor(TN_TOK_IMG_PAD);
} break;
case PROJECTOR_TYPE_PIXTRAL:
{
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
@@ -3161,8 +3202,8 @@ struct clip_model_loader {
model.mm_model_proj_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias"));
model.mm_pre_norm_w = get_tensor(string_format(TN_MM_PRE_NORM, "weight"));
model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight"));
model.mm_img_begin = get_tensor(TN_TOK_IMG_BEGIN);
model.mm_img_end = get_tensor(TN_TOK_IMG_END);
model.mm_img_begin = get_tensor(TN_MM_IMG_BEGIN);
model.mm_img_end = get_tensor(TN_MM_IMG_END);
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR, false);
} break;
@@ -4150,6 +4191,13 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
int y_patch = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
n_patches = x_patch * y_patch;
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
const int out_patch_size = params.patch_size * params.n_merge;
const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size;
const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out;
} break;
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
@@ -5021,6 +5069,58 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
set_input_i32("pos_w", pos_data);
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
// set the 2D positions (mrope layout, only the first 2 channels are used)
int n_patches_per_row = image_size_width / patch_size;
std::vector<int32_t> positions(n_pos * 4, 0);
for (int i = 0; i < n_pos; i++) {
positions[i] = i / n_patches_per_row; // row
positions[n_pos + i] = i % n_patches_per_row; // col
}
set_input_i32("positions", positions);
// token block layout index (see clip_graph_deepseek4v::build)
// rows [0, n_grid) are the aligner output, the sentinels follow
const int n_merge = hparams.n_merge;
const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge;
const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge;
const int n_grid = n_llm_w * n_llm_h;
const int idx_start = n_grid;
const int idx_end = n_grid + 1;
const int idx_newline = n_grid + 2;
const int idx_pad = n_grid + 3;
const int lead_pad = imgs.entries[0].lead_pad;
const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad);
std::vector<int32_t> idx;
idx.reserve(bl.n_out);
for (int i = 0; i < lead_pad; i++) {
idx.push_back(idx_pad);
}
idx.push_back(idx_start);
// pairs of adjacent rows are interleaved column-wise ("N-layout")
// ref: build_image_block in inference/image_processor.py
for (int t = 0; t < bl.rows * bl.row_len; t++) {
const int g = t / (2 * bl.row_len);
const int rem = t % (2 * bl.row_len);
const int c = rem / 2; // column
const int r = 2 * g + rem % 2; // row
if (r >= n_llm_h) {
idx.push_back(idx_pad);
} else if (c == n_llm_w) {
idx.push_back(idx_newline);
} else {
idx.push_back(r * n_llm_w + c);
}
}
for (int i = 0; i < bl.pad_last; i++) {
idx.push_back(idx_pad);
}
idx.push_back(idx_end);
set_input_i32("layout_idx", idx);
} break;
case PROJECTOR_TYPE_GLM_EDGE:
{
// llava and other models
@@ -5762,6 +5862,19 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n");
LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens);
// TEMP debugging (parity validation), will be removed before merge
// when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data]
const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS");
if (dump_path && strcmp(dump_path, "1") != 0) {
FILE * f = fopen(dump_path, "wb");
if (f) {
const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd };
fwrite(hdr, sizeof(hdr), 1, f);
fwrite(emb_data.data(), sizeof(float), emb_data.size(), f);
fclose(f);
}
}
// Print first few values of first token
LOG_INF("Token 0 (first 16 values): ");
for (int i = 0; i < std::min((int64_t)16, n_embd); i++) {
@@ -5866,6 +5979,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_KIMIK25:
case PROJECTOR_TYPE_YASA2:
case PROJECTOR_TYPE_DEEPSEEK4V:
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_HUNYUANVL:
return ctx->model.mm_model_proj->ne[1];