mtmd: move build_inp_temporal_merge to base class (#31065)

This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-10-24 03:38:25 +02:00
1 parent edc353842c
commit 7894e8b924
6 files changed
+37 -33

No files matched your search

+4
View File
@@ -101,6 +101,10 @@ struct clip_graph {
ggml_tensor * build_inp_raw(int channels = 3);
// qwen-vl style temporal merge, handles n_batch == 1 (image) or n_batch == 2 (2 video frames)
// requires patch_embeddings_0 and patch_embeddings_1; returns tensor with shape [w, h, n_embd, 1]
ggml_tensor * build_inp_temporal_merge();
// f16 if flash attn is enabled, set it with set_input_attn_mask()
// idx is only needed when the graph has more than one mask
ggml_tensor * build_inp_attn_mask(int64_t n_kv, int64_t n_q, int idx = 0);
+30
View File
@@ -584,6 +584,36 @@ ggml_tensor * clip_graph::build_inp_raw(int channels) {
return inp_raw;
}
// qwen-vl style temporal merge: Conv3D(kt=2) split into 2 Conv2D
ggml_tensor * clip_graph::build_inp_temporal_merge() {
ggml_tensor * inp_raw = build_inp_raw();
GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
const size_t nb1 = ggml_row_size(inp_raw->type, img.nx());
const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny());
if (n_batch == 1) {
// still image input
return ggml_add(ctx0,
ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1),
ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
} else if (n_batch == 2) {
// 2 frames input (video input)
ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw,
img.nx(), img.ny(), 3, nb1, nb2, 0);
ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw,
img.nx(), img.ny(), 3, nb1, nb2,
nb2 * 3); // move to the second frame
return ggml_add(ctx0,
ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1),
ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1));
} else {
GGML_ASSERT(false && "n_batch > 2 is not supported");
}
}
static std::string get_attn_mask_name(int idx) {
return idx == 0 ? "attn_mask" : "attn_mask_" + std::to_string(idx);
}
+1 -1
View File
@@ -14,7 +14,7 @@ ggml_cgraph * clip_graph_ling3vl::build() {
// vision M-RoPE, same layout as qwen3vl: [row, col, row, col] quarters
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
ggml_tensor * inp = build_inp_with_temporal_merge();
ggml_tensor * inp = build_inp_temporal_merge();
// spatial merge
{
-1
View File
@@ -42,7 +42,6 @@ struct clip_graph_deepseek4v : clip_graph {
struct clip_graph_qwen2vl : clip_graph {
clip_graph_qwen2vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * build_inp_with_temporal_merge();
};
struct clip_graph_qwen3vl : clip_graph_qwen2vl {
+1 -30
View File
@@ -1,34 +1,5 @@
#include "models.h"
ggml_tensor * clip_graph_qwen2vl::build_inp_with_temporal_merge() {
ggml_tensor * inp_raw = build_inp_raw();
GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
const size_t nb1 = ggml_row_size(inp_raw->type, img.nx());
const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny());
if (n_batch == 1) {
// still image input
return ggml_add(ctx0,
ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1),
ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
} else if (n_batch == 2) {
// 2 frames input (video input)
ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw,
img.nx(), img.ny(), 3, nb1, nb2, 0);
ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw,
img.nx(), img.ny(), 3, nb1, nb2,
nb2 * 3); // move to the second frame
return ggml_add(ctx0,
ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1),
ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1));
} else {
GGML_ASSERT(false && "n_batch > 2 is not supported");
}
}
ggml_cgraph * clip_graph_qwen2vl::build() {
GGML_ASSERT(model.patch_bias == nullptr);
GGML_ASSERT(model.class_embedding == nullptr);
@@ -45,7 +16,7 @@ ggml_cgraph * clip_graph_qwen2vl::build() {
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
ggml_tensor * inp = build_inp_with_temporal_merge();
ggml_tensor * inp = build_inp_temporal_merge();
// second conv dimension
{
+1 -1
View File
@@ -13,7 +13,7 @@ ggml_cgraph * clip_graph_qwen3vl::build() {
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
ggml_tensor * inp = build_inp_with_temporal_merge();
ggml_tensor * inp = build_inp_temporal_merge();
// spatial merge
{