mtmd: move build_inp_temporal_merge to base class (#31065)
This commit is contained in:
1 parent
edc353842c
commit
7894e8b924
6 files changed
+37
-33
No files matched your search
@@ -101,6 +101,10 @@ struct clip_graph {
|
||||
|
||||
ggml_tensor * build_inp_raw(int channels = 3);
|
||||
|
||||
// qwen-vl style temporal merge, handles n_batch == 1 (image) or n_batch == 2 (2 video frames)
|
||||
// requires patch_embeddings_0 and patch_embeddings_1; returns tensor with shape [w, h, n_embd, 1]
|
||||
ggml_tensor * build_inp_temporal_merge();
|
||||
|
||||
// f16 if flash attn is enabled, set it with set_input_attn_mask()
|
||||
// idx is only needed when the graph has more than one mask
|
||||
ggml_tensor * build_inp_attn_mask(int64_t n_kv, int64_t n_q, int idx = 0);
|
||||
|
||||
@@ -584,6 +584,36 @@ ggml_tensor * clip_graph::build_inp_raw(int channels) {
|
||||
return inp_raw;
|
||||
}
|
||||
|
||||
// qwen-vl style temporal merge: Conv3D(kt=2) split into 2 Conv2D
|
||||
ggml_tensor * clip_graph::build_inp_temporal_merge() {
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
|
||||
GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
|
||||
GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
|
||||
|
||||
const size_t nb1 = ggml_row_size(inp_raw->type, img.nx());
|
||||
const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny());
|
||||
|
||||
if (n_batch == 1) {
|
||||
// still image input
|
||||
return ggml_add(ctx0,
|
||||
ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1),
|
||||
ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
|
||||
} else if (n_batch == 2) {
|
||||
// 2 frames input (video input)
|
||||
ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw,
|
||||
img.nx(), img.ny(), 3, nb1, nb2, 0);
|
||||
ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw,
|
||||
img.nx(), img.ny(), 3, nb1, nb2,
|
||||
nb2 * 3); // move to the second frame
|
||||
return ggml_add(ctx0,
|
||||
ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1),
|
||||
ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1));
|
||||
} else {
|
||||
GGML_ASSERT(false && "n_batch > 2 is not supported");
|
||||
}
|
||||
}
|
||||
|
||||
static std::string get_attn_mask_name(int idx) {
|
||||
return idx == 0 ? "attn_mask" : "attn_mask_" + std::to_string(idx);
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ ggml_cgraph * clip_graph_ling3vl::build() {
|
||||
// vision M-RoPE, same layout as qwen3vl: [row, col, row, col] quarters
|
||||
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
|
||||
|
||||
ggml_tensor * inp = build_inp_with_temporal_merge();
|
||||
ggml_tensor * inp = build_inp_temporal_merge();
|
||||
|
||||
// spatial merge
|
||||
{
|
||||
|
||||
@@ -42,7 +42,6 @@ struct clip_graph_deepseek4v : clip_graph {
|
||||
struct clip_graph_qwen2vl : clip_graph {
|
||||
clip_graph_qwen2vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
ggml_tensor * build_inp_with_temporal_merge();
|
||||
};
|
||||
|
||||
struct clip_graph_qwen3vl : clip_graph_qwen2vl {
|
||||
|
||||
@@ -1,34 +1,5 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_tensor * clip_graph_qwen2vl::build_inp_with_temporal_merge() {
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
|
||||
GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
|
||||
GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
|
||||
|
||||
const size_t nb1 = ggml_row_size(inp_raw->type, img.nx());
|
||||
const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny());
|
||||
|
||||
if (n_batch == 1) {
|
||||
// still image input
|
||||
return ggml_add(ctx0,
|
||||
ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1),
|
||||
ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
|
||||
} else if (n_batch == 2) {
|
||||
// 2 frames input (video input)
|
||||
ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw,
|
||||
img.nx(), img.ny(), 3, nb1, nb2, 0);
|
||||
ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw,
|
||||
img.nx(), img.ny(), 3, nb1, nb2,
|
||||
nb2 * 3); // move to the second frame
|
||||
return ggml_add(ctx0,
|
||||
ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1),
|
||||
ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1));
|
||||
} else {
|
||||
GGML_ASSERT(false && "n_batch > 2 is not supported");
|
||||
}
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_qwen2vl::build() {
|
||||
GGML_ASSERT(model.patch_bias == nullptr);
|
||||
GGML_ASSERT(model.class_embedding == nullptr);
|
||||
@@ -45,7 +16,7 @@ ggml_cgraph * clip_graph_qwen2vl::build() {
|
||||
|
||||
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
|
||||
|
||||
ggml_tensor * inp = build_inp_with_temporal_merge();
|
||||
ggml_tensor * inp = build_inp_temporal_merge();
|
||||
|
||||
// second conv dimension
|
||||
{
|
||||
|
||||
@@ -13,7 +13,7 @@ ggml_cgraph * clip_graph_qwen3vl::build() {
|
||||
|
||||
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
|
||||
|
||||
ggml_tensor * inp = build_inp_with_temporal_merge();
|
||||
ggml_tensor * inp = build_inp_temporal_merge();
|
||||
|
||||
// spatial merge
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user