Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: support DeepSeek-V4-Flash-Vision-Exp (#28133)

Browse Source 
* mtmd: support DeepSeek-V4-Flash-Vision-Exp

* handle min/max token counts from CLI

* rm debugging

* use GGML_ROPE_TYPE_VISION

* nits

* apply review comments

* correct token count
This commit is contained in:
Xuan-Son Nguyen
2026-09-02 16:43:43 +02:00
committed by GitHub
parent 8e93a9773b
commit 7798007a29
13 changed files with 504 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/mtmd.cpp
+20 -1
View File
@@ -27,7 +27,7 @@
#include <vector>
// remember to bump this if the serialization format changes
#define MTMD_SERIALIZATION_VERSION 1
#define MTMD_SERIALIZATION_VERSION 2
struct mtmd_serialization {
// note: using 64-bit here for future-proofing
@@ -105,12 +105,14 @@ void clip_image_f32::serialize(mtmd_serialization & ser) const {
// note: buf is intentionally NOT serialized; the loaded clip_image_f32 will always be a placeholder
ser.write(add_viewsep);
ser.write(add_newline);
ser.write(lead_pad);
ser.write((int32_t)nx_);
ser.write((int32_t)ny_);
}
void clip_image_f32::deserialize(mtmd_serialization & ser) {
add_viewsep = ser.read<bool>();
add_newline = ser.read<bool>();
lead_pad = ser.read<int32_t>();
nx_ = ser.read<int32_t>();
ny_ = ser.read<int32_t>();
buf.clear(); // always a placeholder after loading
@@ -824,6 +826,11 @@ struct mtmd_context {
img_end = "<|im_end|>";
image_preproc = std::make_unique<mtmd_image_preprocessor_longest_edge>(ctx_v);
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
// no vocab tokens are added; the start/end/newline markers are learned embeddings emitted by the encoder
image_preproc = std::make_unique<mtmd_image_preprocessor_deepseek4v>(ctx_v);
} break;
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
{
@@ -1451,6 +1458,18 @@ struct mtmd_tokenizer {
return 2;
}
if (ctx->proj_type_v() == PROJECTOR_TYPE_DEEPSEEK4V) {
// the text model perceives input in blocks of N tokens (N = COMPRESS_PAD_TO = 4, same as the CSA compress ratio)
// image need to be aligned to block size, while adding IMAGE_PAD embeddings to the beginning
// TODO @ngxson : maybe refactor this in the future
constexpr int32_t align = 4;
size_t n_past = 0;
for (const auto & e : cur.entries) {
n_past += mtmd_input_chunk_get_n_tokens(&e);
}
preproc_out.entries[0].lead_pad = align - 1 - (int32_t)(n_past % align);
}
size_t n_tokens = 0;
for (auto & e : preproc_out.entries) {
n_tokens += clip_n_output_tokens(ctx->ctx_v, &e);