mtmd : fix Muse Glimmer vision embedding order (#32433)

Assisted-by: Claude
This commit is contained in:
Sidhartha Reddy Potu authored and GitHub committed 2026-11-17 00:29:02 +01:00
1 parent 1ef8c5d684
commit 9581154a22
1 file changed
+9 -1
+9 -1
View File
@@ -34,8 +34,16 @@ ggml_cgraph * mtmd_graph_muse_glimmer::build() {
ggml_tensor * sp_mask = build_inp_attn_mask(n_tok, n_tok);
// patchify via build_inp (conv2d over raw pixels) + bilinear-resized learned pos-emb
// pos-emb grid is resized with height and width swapped, to match the trainer reference
const int n_per_side = (int) std::sqrt(model.position_embeddings->ne[1]);
ggml_tensor * pos_embd = ggml_reshape_3d(ctx0, model.position_embeddings, n_embd, n_per_side, n_per_side);
pos_embd = ggml_permute(ctx0, pos_embd, 2, 0, 1, 3);
pos_embd = ggml_interpolate(ctx0, pos_embd, n_patches_y, n_patches_x, n_embd, 1, GGML_SCALE_MODE_BILINEAR);
pos_embd = ggml_permute(ctx0, pos_embd, 1, 2, 0, 3);
pos_embd = ggml_cont_2d(ctx0, pos_embd, n_embd, n_tok);
ggml_tensor * x = build_inp(); // [n_embd, n_tok, 1]
x = ggml_add(ctx0, x, resize_position_embeddings(GGML_SCALE_MODE_BILINEAR));
x = ggml_add(ctx0, x, pos_embd);
cb(x, "after_posemb", -1);
// group patches into pgrid x pgrid windows (sparse attention order)