mtmd: add minicpmv46 downsample (#25993)
* add minicpmv46 downsample Signed-off-by: tc-mb <tianchi_cai@icloud.com> * put downsample mode inside gguf. Signed-off-by: tc-mb <tianchi_cai@icloud.com> * build mtmd_image_preprocessor_llava_uhd Signed-off-by: tc-mb <tianchi_cai@icloud.com> * fix code Signed-off-by: tc-mb <tianchi_cai@icloud.com> * add convert Signed-off-by: tc-mb <tianchi_cai@icloud.com> * add 4x ignore vit merger Signed-off-by: tc-mb <tianchi_cai@icloud.com> --------- Signed-off-by: tc-mb <tianchi_cai@icloud.com>
This commit is contained in:
tc-mb
+135
-180
@@ -114,14 +114,12 @@ ggml_cgraph * clip_graph_minicpmv::build() {
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_minicpmv4_6::build() {
|
||||
const int insert_lid = hparams.insert_layer_id;
|
||||
const int n_pos = n_patches;
|
||||
const int half_h = n_patches_y / 2;
|
||||
const int half_w = n_patches_x / 2;
|
||||
const int n_ds = half_h * half_w; // after ViT merger 2x2 downsample
|
||||
const int qh = half_h / 2;
|
||||
const int qw = half_w / 2;
|
||||
const int n_ds2 = qh * qw; // after final merger 2x2 downsample
|
||||
const bool is_4x = hparams.n_merge == 2;
|
||||
const int n_pos = n_patches;
|
||||
const int half_h = n_patches_y / 2;
|
||||
const int half_w = n_patches_x / 2;
|
||||
const int n_ds = half_h * half_w;
|
||||
const int n_out = is_4x ? n_ds : (half_h / 2) * (half_w / 2);
|
||||
|
||||
auto add_i32_input = [&](const char * name, int n) {
|
||||
ggml_tensor * t = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n);
|
||||
@@ -134,29 +132,39 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() {
|
||||
ggml_tensor * positions = add_i32_input("positions", n_pos);
|
||||
ggml_tensor * learned_pos_embd = ggml_get_rows(ctx0, model.position_embeddings, positions);
|
||||
|
||||
// ViT merger window reorder indices + block-diagonal mask
|
||||
// (mask layout follows qwen2vl: -inf except for 4x4 blocks on the diagonal,
|
||||
// so each window-major group of 4 tokens only attends to itself)
|
||||
ggml_tensor * vit_merger_window_idx = add_i32_input("vit_merger_window_idx", n_pos);
|
||||
ggml_tensor * vit_merger_inv_window_idx = add_i32_input("vit_merger_inv_window_idx", n_pos);
|
||||
ggml_tensor * vit_merger_window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);
|
||||
ggml_set_name(vit_merger_window_mask, "vit_merger_window_mask");
|
||||
ggml_set_input(vit_merger_window_mask);
|
||||
if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
|
||||
vit_merger_window_mask = ggml_cast(ctx0, vit_merger_window_mask, GGML_TYPE_F16);
|
||||
ggml_tensor * vit_merger_window_idx = nullptr;
|
||||
ggml_tensor * vit_merger_inv_window_idx = nullptr;
|
||||
ggml_tensor * vit_merger_window_mask = nullptr;
|
||||
ggml_tensor * vit_merger_ds_idx_0 = nullptr;
|
||||
ggml_tensor * vit_merger_ds_idx_1 = nullptr;
|
||||
ggml_tensor * vit_merger_ds_idx_2 = nullptr;
|
||||
ggml_tensor * vit_merger_ds_idx_3 = nullptr;
|
||||
|
||||
if (!is_4x) {
|
||||
// ViT merger window reorder indices + block-diagonal mask
|
||||
// (mask layout follows qwen2vl: -inf except for 4x4 blocks on the diagonal,
|
||||
// so each window-major group of 4 tokens only attends to itself)
|
||||
vit_merger_window_idx = add_i32_input("vit_merger_window_idx", n_pos);
|
||||
vit_merger_inv_window_idx = add_i32_input("vit_merger_inv_window_idx", n_pos);
|
||||
vit_merger_window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);
|
||||
ggml_set_name(vit_merger_window_mask, "vit_merger_window_mask");
|
||||
ggml_set_input(vit_merger_window_mask);
|
||||
if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
|
||||
vit_merger_window_mask = ggml_cast(ctx0, vit_merger_window_mask, GGML_TYPE_F16);
|
||||
}
|
||||
|
||||
// ViT merger 2x2 downsample gather indices
|
||||
vit_merger_ds_idx_0 = add_i32_input("vit_merger_ds_idx_0", n_ds);
|
||||
vit_merger_ds_idx_1 = add_i32_input("vit_merger_ds_idx_1", n_ds);
|
||||
vit_merger_ds_idx_2 = add_i32_input("vit_merger_ds_idx_2", n_ds);
|
||||
vit_merger_ds_idx_3 = add_i32_input("vit_merger_ds_idx_3", n_ds);
|
||||
}
|
||||
|
||||
// ViT merger 2x2 downsample gather indices
|
||||
ggml_tensor * vit_merger_ds_idx_0 = add_i32_input("vit_merger_ds_idx_0", n_ds);
|
||||
ggml_tensor * vit_merger_ds_idx_1 = add_i32_input("vit_merger_ds_idx_1", n_ds);
|
||||
ggml_tensor * vit_merger_ds_idx_2 = add_i32_input("vit_merger_ds_idx_2", n_ds);
|
||||
ggml_tensor * vit_merger_ds_idx_3 = add_i32_input("vit_merger_ds_idx_3", n_ds);
|
||||
|
||||
// final merger 2x2 downsample gather indices
|
||||
ggml_tensor * merger_ds_idx_0 = add_i32_input("merger_ds_idx_0", n_ds2);
|
||||
ggml_tensor * merger_ds_idx_1 = add_i32_input("merger_ds_idx_1", n_ds2);
|
||||
ggml_tensor * merger_ds_idx_2 = add_i32_input("merger_ds_idx_2", n_ds2);
|
||||
ggml_tensor * merger_ds_idx_3 = add_i32_input("merger_ds_idx_3", n_ds2);
|
||||
ggml_tensor * merger_ds_idx_0 = add_i32_input("merger_ds_idx_0", n_out);
|
||||
ggml_tensor * merger_ds_idx_1 = add_i32_input("merger_ds_idx_1", n_out);
|
||||
ggml_tensor * merger_ds_idx_2 = add_i32_input("merger_ds_idx_2", n_out);
|
||||
ggml_tensor * merger_ds_idx_3 = add_i32_input("merger_ds_idx_3", n_out);
|
||||
|
||||
// patch embedding + positional embedding
|
||||
ggml_tensor * inp = build_inp();
|
||||
@@ -169,150 +177,10 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() {
|
||||
cb(inpL, "pre_ln", -1);
|
||||
}
|
||||
|
||||
// ViT layers 0..insert_layer_id (inclusive)
|
||||
// Mirrors the separate-qkv path of clip_graph::build_vit so the two manually
|
||||
// unrolled segments around the ViT merger read like build_vit() expansions.
|
||||
for (int il = 0; il <= insert_lid; il++) {
|
||||
auto & layer = model.layers[il];
|
||||
ggml_tensor * cur = inpL;
|
||||
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cb(cur, "layer_inp_normed", il);
|
||||
|
||||
{
|
||||
ggml_tensor * Qcur = build_mm(layer.q_w, cur);
|
||||
if (layer.q_b) {
|
||||
Qcur = ggml_add(ctx0, Qcur, layer.q_b);
|
||||
}
|
||||
ggml_tensor * Kcur = build_mm(layer.k_w, cur);
|
||||
if (layer.k_b) {
|
||||
Kcur = ggml_add(ctx0, Kcur, layer.k_b);
|
||||
}
|
||||
ggml_tensor * Vcur = build_mm(layer.v_w, cur);
|
||||
if (layer.v_b) {
|
||||
Vcur = ggml_add(ctx0, Vcur, layer.v_b);
|
||||
}
|
||||
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
|
||||
cb(Qcur, "Qcur", il);
|
||||
cb(Kcur, "Kcur", il);
|
||||
cb(Vcur, "Vcur", il);
|
||||
|
||||
cur = build_attn(layer.o_w, layer.o_b, Qcur, Kcur, Vcur, nullptr, kq_scale, il);
|
||||
cb(cur, "attn_out", il);
|
||||
}
|
||||
|
||||
if (layer.ls_1_w) {
|
||||
cur = ggml_mul(ctx0, cur, layer.ls_1_w);
|
||||
cb(cur, "attn_out_scaled", il);
|
||||
}
|
||||
cur = ggml_add(ctx0, cur, inpL);
|
||||
inpL = cur;
|
||||
cb(cur, "ffn_inp", il);
|
||||
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cb(cur, "ffn_inp_normed", il);
|
||||
|
||||
cur = build_ffn(cur, layer.ff_up_w, layer.ff_up_b, layer.ff_gate_w, layer.ff_gate_b,
|
||||
layer.ff_down_w, layer.ff_down_b, hparams.ffn_op, il);
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
if (layer.ls_2_w) {
|
||||
cur = ggml_mul(ctx0, cur, layer.ls_2_w);
|
||||
cb(cur, "ffn_out_scaled", il);
|
||||
}
|
||||
cur = ggml_add(ctx0, inpL, cur);
|
||||
cb(cur, "layer_out", il);
|
||||
|
||||
inpL = cur;
|
||||
}
|
||||
|
||||
// ViT merger: window self-attention
|
||||
// Tokens are reordered to window-major (4 tokens per window are contiguous),
|
||||
// and a block-diagonal mask restricts attention to within each window. This
|
||||
// mirrors the qwen2vl windowed-attention pattern so build_attn() can pick the
|
||||
// flash-attention path when available.
|
||||
{
|
||||
ggml_tensor * residual = inpL;
|
||||
ggml_tensor * cur = build_norm(inpL,
|
||||
model.vit_merger_ln1_w, model.vit_merger_ln1_b,
|
||||
NORM_TYPE_NORMAL, eps, -1);
|
||||
cb(cur, "vit_merger_attn_inp_normed", -1);
|
||||
|
||||
cur = ggml_get_rows(ctx0, cur, vit_merger_window_idx);
|
||||
cb(cur, "vit_merger_window_reorder", -1);
|
||||
|
||||
ggml_tensor * Qcur = build_mm(model.vit_merger_attn_q_w, cur);
|
||||
if (model.vit_merger_attn_q_b) {
|
||||
Qcur = ggml_add(ctx0, Qcur, model.vit_merger_attn_q_b);
|
||||
}
|
||||
ggml_tensor * Kcur = build_mm(model.vit_merger_attn_k_w, cur);
|
||||
if (model.vit_merger_attn_k_b) {
|
||||
Kcur = ggml_add(ctx0, Kcur, model.vit_merger_attn_k_b);
|
||||
}
|
||||
ggml_tensor * Vcur = build_mm(model.vit_merger_attn_v_w, cur);
|
||||
if (model.vit_merger_attn_v_b) {
|
||||
Vcur = ggml_add(ctx0, Vcur, model.vit_merger_attn_v_b);
|
||||
}
|
||||
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
|
||||
cb(Qcur, "vit_merger_Qcur", -1);
|
||||
cb(Kcur, "vit_merger_Kcur", -1);
|
||||
cb(Vcur, "vit_merger_Vcur", -1);
|
||||
|
||||
cur = build_attn(model.vit_merger_attn_o_w, model.vit_merger_attn_o_b,
|
||||
Qcur, Kcur, Vcur, vit_merger_window_mask, kq_scale, -1);
|
||||
cb(cur, "vit_merger_attn_out", -1);
|
||||
|
||||
cur = ggml_get_rows(ctx0, cur, vit_merger_inv_window_idx);
|
||||
inpL = ggml_add(ctx0, cur, residual);
|
||||
cb(inpL, "vit_merger_attn_residual", -1);
|
||||
}
|
||||
|
||||
// ViT merger: 2x2 spatial downsample + MLP (4 tokens -> 1)
|
||||
{
|
||||
ggml_tensor * p0 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_0);
|
||||
ggml_tensor * p1 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_1);
|
||||
ggml_tensor * p2 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_2);
|
||||
ggml_tensor * p3 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_3);
|
||||
|
||||
ggml_tensor * mean_res = ggml_add(ctx0, p0, p1);
|
||||
mean_res = ggml_add(ctx0, mean_res, p2);
|
||||
mean_res = ggml_add(ctx0, mean_res, p3);
|
||||
mean_res = ggml_scale(ctx0, mean_res, 0.25f);
|
||||
cb(mean_res, "vit_merger_ds_mean_res", -1);
|
||||
|
||||
ggml_tensor * cat = ggml_concat(ctx0, p0, p1, 0);
|
||||
cat = ggml_concat(ctx0, cat, p2, 0);
|
||||
cat = ggml_concat(ctx0, cat, p3, 0);
|
||||
|
||||
ggml_tensor * cur = build_norm(cat,
|
||||
model.vit_merger_ds_ln_w, model.vit_merger_ds_ln_b,
|
||||
NORM_TYPE_NORMAL, eps, -1);
|
||||
cb(cur, "vit_merger_ds_normed", -1);
|
||||
|
||||
// ViTWindowAttentionMerger downsample MLP uses gelu_pytorch_tanh (FFN_GELU)
|
||||
cur = build_ffn(cur,
|
||||
model.vit_merger_ds_up_w, model.vit_merger_ds_up_b,
|
||||
nullptr, nullptr,
|
||||
model.vit_merger_ds_down_w, model.vit_merger_ds_down_b,
|
||||
FFN_GELU, -1);
|
||||
cb(cur, "vit_merger_ds_mlp_out", -1);
|
||||
|
||||
inpL = ggml_add(ctx0, cur, mean_res);
|
||||
cb(inpL, "vit_merger_ds_out", -1);
|
||||
}
|
||||
|
||||
// ViT layers (insert_layer_id+1)..n_layer-1, operating on the downsampled tokens
|
||||
{
|
||||
const int64_t n_pos_ds = n_ds;
|
||||
for (int il = insert_lid + 1; il < n_layer; il++) {
|
||||
auto build_vit_layers = [&](ggml_tensor * input, int il_begin, int il_end, int64_t n_pos_layer) {
|
||||
for (int il = il_begin; il < il_end; il++) {
|
||||
auto & layer = model.layers[il];
|
||||
ggml_tensor * cur = inpL;
|
||||
ggml_tensor * cur = input;
|
||||
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cb(cur, "layer_inp_normed", il);
|
||||
@@ -331,9 +199,9 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() {
|
||||
Vcur = ggml_add(ctx0, Vcur, layer.v_b);
|
||||
}
|
||||
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos_ds);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos_ds);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos_ds);
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos_layer);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos_layer);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos_layer);
|
||||
cb(Qcur, "Qcur", il);
|
||||
cb(Kcur, "Kcur", il);
|
||||
cb(Vcur, "Vcur", il);
|
||||
@@ -346,8 +214,8 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() {
|
||||
cur = ggml_mul(ctx0, cur, layer.ls_1_w);
|
||||
cb(cur, "attn_out_scaled", il);
|
||||
}
|
||||
cur = ggml_add(ctx0, cur, inpL);
|
||||
inpL = cur;
|
||||
cur = ggml_add(ctx0, cur, input);
|
||||
input = cur;
|
||||
cb(cur, "ffn_inp", il);
|
||||
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
|
||||
@@ -361,11 +229,98 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() {
|
||||
cur = ggml_mul(ctx0, cur, layer.ls_2_w);
|
||||
cb(cur, "ffn_out_scaled", il);
|
||||
}
|
||||
cur = ggml_add(ctx0, inpL, cur);
|
||||
cb(cur, "layer_out", il);
|
||||
|
||||
inpL = cur;
|
||||
input = ggml_add(ctx0, input, cur);
|
||||
cb(input, "layer_out", il);
|
||||
}
|
||||
return input;
|
||||
};
|
||||
|
||||
if (!is_4x) {
|
||||
const int insert_lid = hparams.insert_layer_id;
|
||||
|
||||
inpL = build_vit_layers(inpL, 0, insert_lid + 1, n_pos);
|
||||
|
||||
// ViT merger: window self-attention
|
||||
// Tokens are reordered to window-major (4 tokens per window are contiguous),
|
||||
// and a block-diagonal mask restricts attention to within each window. This
|
||||
// mirrors the qwen2vl windowed-attention pattern so build_attn() can pick the
|
||||
// flash-attention path when available.
|
||||
{
|
||||
ggml_tensor * residual = inpL;
|
||||
ggml_tensor * cur = build_norm(inpL,
|
||||
model.vit_merger_ln1_w, model.vit_merger_ln1_b,
|
||||
NORM_TYPE_NORMAL, eps, -1);
|
||||
cb(cur, "vit_merger_attn_inp_normed", -1);
|
||||
|
||||
cur = ggml_get_rows(ctx0, cur, vit_merger_window_idx);
|
||||
cb(cur, "vit_merger_window_reorder", -1);
|
||||
|
||||
ggml_tensor * Qcur = build_mm(model.vit_merger_attn_q_w, cur);
|
||||
if (model.vit_merger_attn_q_b) {
|
||||
Qcur = ggml_add(ctx0, Qcur, model.vit_merger_attn_q_b);
|
||||
}
|
||||
ggml_tensor * Kcur = build_mm(model.vit_merger_attn_k_w, cur);
|
||||
if (model.vit_merger_attn_k_b) {
|
||||
Kcur = ggml_add(ctx0, Kcur, model.vit_merger_attn_k_b);
|
||||
}
|
||||
ggml_tensor * Vcur = build_mm(model.vit_merger_attn_v_w, cur);
|
||||
if (model.vit_merger_attn_v_b) {
|
||||
Vcur = ggml_add(ctx0, Vcur, model.vit_merger_attn_v_b);
|
||||
}
|
||||
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
|
||||
cb(Qcur, "vit_merger_Qcur", -1);
|
||||
cb(Kcur, "vit_merger_Kcur", -1);
|
||||
cb(Vcur, "vit_merger_Vcur", -1);
|
||||
|
||||
cur = build_attn(model.vit_merger_attn_o_w, model.vit_merger_attn_o_b,
|
||||
Qcur, Kcur, Vcur, vit_merger_window_mask, kq_scale, -1);
|
||||
cb(cur, "vit_merger_attn_out", -1);
|
||||
|
||||
cur = ggml_get_rows(ctx0, cur, vit_merger_inv_window_idx);
|
||||
inpL = ggml_add(ctx0, cur, residual);
|
||||
cb(inpL, "vit_merger_attn_residual", -1);
|
||||
}
|
||||
|
||||
// ViT merger: 2x2 spatial downsample + MLP (4 tokens -> 1)
|
||||
{
|
||||
ggml_tensor * p0 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_0);
|
||||
ggml_tensor * p1 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_1);
|
||||
ggml_tensor * p2 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_2);
|
||||
ggml_tensor * p3 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_3);
|
||||
|
||||
ggml_tensor * mean_res = ggml_add(ctx0, p0, p1);
|
||||
mean_res = ggml_add(ctx0, mean_res, p2);
|
||||
mean_res = ggml_add(ctx0, mean_res, p3);
|
||||
mean_res = ggml_scale(ctx0, mean_res, 0.25f);
|
||||
cb(mean_res, "vit_merger_ds_mean_res", -1);
|
||||
|
||||
ggml_tensor * cat = ggml_concat(ctx0, p0, p1, 0);
|
||||
cat = ggml_concat(ctx0, cat, p2, 0);
|
||||
cat = ggml_concat(ctx0, cat, p3, 0);
|
||||
|
||||
ggml_tensor * cur = build_norm(cat,
|
||||
model.vit_merger_ds_ln_w, model.vit_merger_ds_ln_b,
|
||||
NORM_TYPE_NORMAL, eps, -1);
|
||||
cb(cur, "vit_merger_ds_normed", -1);
|
||||
|
||||
// ViTWindowAttentionMerger downsample MLP uses gelu_pytorch_tanh (FFN_GELU)
|
||||
cur = build_ffn(cur,
|
||||
model.vit_merger_ds_up_w, model.vit_merger_ds_up_b,
|
||||
nullptr, nullptr,
|
||||
model.vit_merger_ds_down_w, model.vit_merger_ds_down_b,
|
||||
FFN_GELU, -1);
|
||||
cb(cur, "vit_merger_ds_mlp_out", -1);
|
||||
|
||||
inpL = ggml_add(ctx0, cur, mean_res);
|
||||
cb(inpL, "vit_merger_ds_out", -1);
|
||||
}
|
||||
|
||||
inpL = build_vit_layers(inpL, insert_lid + 1, n_layer, n_ds);
|
||||
} else {
|
||||
inpL = build_vit_layers(inpL, 0, n_layer, n_pos);
|
||||
}
|
||||
|
||||
if (model.post_ln_w) {
|
||||
|
||||
Reference in New Issue
Block a user