From 7894e8b92481a8bb8ffe94f73b656ffe0dd01e10 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Sat, 24 Oct 2026 03:38:25 +0200 Subject: [PATCH] mtmd: move build_inp_temporal_merge to base class (#31065) --- tools/mtmd/clip-graph.h | 4 ++++ tools/mtmd/clip.cpp | 30 ++++++++++++++++++++++++++++++ tools/mtmd/models/ling3vl.cpp | 2 +- tools/mtmd/models/models.h | 1 - tools/mtmd/models/qwen2vl.cpp | 31 +------------------------------ tools/mtmd/models/qwen3vl.cpp | 2 +- 6 files changed, 37 insertions(+), 33 deletions(-) diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/clip-graph.h index 33e546b7d..a0669d1a0 100644 --- a/tools/mtmd/clip-graph.h +++ b/tools/mtmd/clip-graph.h @@ -101,6 +101,10 @@ struct clip_graph { ggml_tensor * build_inp_raw(int channels = 3); + // qwen-vl style temporal merge, handles n_batch == 1 (image) or n_batch == 2 (2 video frames) + // requires patch_embeddings_0 and patch_embeddings_1; returns tensor with shape [w, h, n_embd, 1] + ggml_tensor * build_inp_temporal_merge(); + // f16 if flash attn is enabled, set it with set_input_attn_mask() // idx is only needed when the graph has more than one mask ggml_tensor * build_inp_attn_mask(int64_t n_kv, int64_t n_q, int idx = 0); diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 2ab55c52a..d204708ee 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -584,6 +584,36 @@ ggml_tensor * clip_graph::build_inp_raw(int channels) { return inp_raw; } +// qwen-vl style temporal merge: Conv3D(kt=2) split into 2 Conv2D +ggml_tensor * clip_graph::build_inp_temporal_merge() { + ggml_tensor * inp_raw = build_inp_raw(); + + GGML_ASSERT(img.nx() % (patch_size * 2) == 0); + GGML_ASSERT(img.ny() % (patch_size * 2) == 0); + + const size_t nb1 = ggml_row_size(inp_raw->type, img.nx()); + const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny()); + + if (n_batch == 1) { + // still image input + return ggml_add(ctx0, + ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1), + ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1)); + } else if (n_batch == 2) { + // 2 frames input (video input) + ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw, + img.nx(), img.ny(), 3, nb1, nb2, 0); + ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw, + img.nx(), img.ny(), 3, nb1, nb2, + nb2 * 3); // move to the second frame + return ggml_add(ctx0, + ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1), + ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1)); + } else { + GGML_ASSERT(false && "n_batch > 2 is not supported"); + } +} + static std::string get_attn_mask_name(int idx) { return idx == 0 ? "attn_mask" : "attn_mask_" + std::to_string(idx); } diff --git a/tools/mtmd/models/ling3vl.cpp b/tools/mtmd/models/ling3vl.cpp index ef8eb3a83..fb9eb7c74 100644 --- a/tools/mtmd/models/ling3vl.cpp +++ b/tools/mtmd/models/ling3vl.cpp @@ -14,7 +14,7 @@ ggml_cgraph * clip_graph_ling3vl::build() { // vision M-RoPE, same layout as qwen3vl: [row, col, row, col] quarters int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4}; - ggml_tensor * inp = build_inp_with_temporal_merge(); + ggml_tensor * inp = build_inp_temporal_merge(); // spatial merge { diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 8ae2a5b25..3b34548c4 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -42,7 +42,6 @@ struct clip_graph_deepseek4v : clip_graph { struct clip_graph_qwen2vl : clip_graph { clip_graph_qwen2vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; - ggml_tensor * build_inp_with_temporal_merge(); }; struct clip_graph_qwen3vl : clip_graph_qwen2vl { diff --git a/tools/mtmd/models/qwen2vl.cpp b/tools/mtmd/models/qwen2vl.cpp index 4998d95bc..7516aa23a 100644 --- a/tools/mtmd/models/qwen2vl.cpp +++ b/tools/mtmd/models/qwen2vl.cpp @@ -1,34 +1,5 @@ #include "models.h" -ggml_tensor * clip_graph_qwen2vl::build_inp_with_temporal_merge() { - ggml_tensor * inp_raw = build_inp_raw(); - - GGML_ASSERT(img.nx() % (patch_size * 2) == 0); - GGML_ASSERT(img.ny() % (patch_size * 2) == 0); - - const size_t nb1 = ggml_row_size(inp_raw->type, img.nx()); - const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny()); - - if (n_batch == 1) { - // still image input - return ggml_add(ctx0, - ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1), - ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1)); - } else if (n_batch == 2) { - // 2 frames input (video input) - ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw, - img.nx(), img.ny(), 3, nb1, nb2, 0); - ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw, - img.nx(), img.ny(), 3, nb1, nb2, - nb2 * 3); // move to the second frame - return ggml_add(ctx0, - ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1), - ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1)); - } else { - GGML_ASSERT(false && "n_batch > 2 is not supported"); - } -} - ggml_cgraph * clip_graph_qwen2vl::build() { GGML_ASSERT(model.patch_bias == nullptr); GGML_ASSERT(model.class_embedding == nullptr); @@ -45,7 +16,7 @@ ggml_cgraph * clip_graph_qwen2vl::build() { int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4}; - ggml_tensor * inp = build_inp_with_temporal_merge(); + ggml_tensor * inp = build_inp_temporal_merge(); // second conv dimension { diff --git a/tools/mtmd/models/qwen3vl.cpp b/tools/mtmd/models/qwen3vl.cpp index 48626b221..2e49dcd01 100644 --- a/tools/mtmd/models/qwen3vl.cpp +++ b/tools/mtmd/models/qwen3vl.cpp @@ -13,7 +13,7 @@ ggml_cgraph * clip_graph_qwen3vl::build() { int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4}; - ggml_tensor * inp = build_inp_with_temporal_merge(); + ggml_tensor * inp = build_inp_temporal_merge(); // spatial merge {