mtmd: fix Granite4 Vision image sequence assembly (#26653)

* mtmd: fix granite 4v grid assembly

(cherry picked from commit 91f82eb1b489bff0dc3f649edf2cb9e0b7f656e9)

* mtmd: fix truncation for scaled image height and width before unpad

Signed-off-by: Hemanth Battu <hbattu@ibm.com>

* mtmd: remove MTMD_DUMP_EMBD debug scaffolding

Signed-off-by: Hemanth Battu <hbattu@ibm.com>

* clean up comments, clarify about anyres_info excluded from serialization

* add_newline is now dead code

---------

Signed-off-by: Hemanth Battu <hbattu@ibm.com>
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
Co-authored-by: Hemanth Battu <hbattu@ibm.com>

tools/mtmd/clip-impl.h

@@ -6,6 +6,7 @@
 
 #include <array>
 #include <climits>
+#include <cmath>
 #include <cstdarg>
 #include <cinttypes>
 #include <string>
@@ -642,9 +643,25 @@ struct mtmd_serialization; // forward declaration
 struct clip_image_f32 {
     // marks the global view in e.g., DeepSeek-OCR Models
     bool add_viewsep = false;
-    // whether a learned newline (or EOI) token should be appended after the image (eg Granite4 Vision)
+    // appends a learned newline (or EOI) token after the image
+    // no model uses it now (Granite4 Vision moved to anyres), kept for future models
     bool add_newline = false;
 
+    // llava-next "anyres" tiling, used by Granite4 Vision
+    // the whole grid is encoded and assembled in a single graph
+    // NOTE: excluded from serialized: a deserialized image is always a placeholder, which is never encoded
+    struct anyres_info {
+        int grid_x = 0; // tiles per row, 0 means the image is not tiled
+        int grid_y = 0; // tiles per column
+        int orig_nx = 0; // size of the source image, used to drop the padding tokens
+        int orig_ny = 0;
+
+        bool is_tiled() const {
+            return grid_x > 0 && grid_y > 0;
+        }
+    };
+    anyres_info anyres;
+
     clip_image_size get_size() const {
         return { nx_, ny_ };
     }
@@ -726,6 +743,25 @@ struct clip_image_f32 {
     }
 };
 
+// token area kept after removing the padding added by the anyres resize
+// ref: https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/llava_next/modeling_llava_next.py#L109
+static inline void clip_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_h,
+                                     int & off_x, int & off_y, int & out_w, int & out_h) {
+    off_x = 0;
+    off_y = 0;
+    out_w = cur_w;
+    out_h = cur_h;
+    if ((float) orig_w / orig_h > (float) cur_w / cur_h) {
+        const int new_h = (int) std::floor((double) orig_h * cur_w / orig_w + 1e-7);
+        off_y = (cur_h - new_h) / 2;
+        out_h = cur_h - 2 * off_y;
+    } else {
+        const int new_w = (int) std::floor((double) orig_w * cur_h / orig_h + 1e-7);
+        off_x = (cur_w - new_w) / 2;
+        out_w = cur_w - 2 * off_x;
+    }
+}
+
 //
 // logging
 //

tools/mtmd/clip.cpp

@@ -4229,18 +4229,20 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
         case PROJECTOR_TYPE_GRANITE4_VISION:
             {
                 // Per-tile output token count: each projector block outputs
-                // query_side^2 tokens per window × n^2 windows.
+                // query_side^2 tokens per window x n^2 windows.
-                // For 384×384 input: n = 24/8 = 3, query_side = 4 → 144.
+                // For 384x384 input: n = 24/8 = 3, query_side = 4 -> 144.
                 const int window_side = ctx->model.hparams.downsample_window_side;
                 const int query_side  = ctx->model.hparams.downsample_query_side;
                 const int side        = img->nx() / params.patch_size;
                 const int n           = side / window_side;
-                n_patches             = (query_side * n) * (query_side * n);
+                const int out_side    = query_side * n;
-                if (img->add_newline) {
+                n_patches             = out_side * out_side;
-                    // For single-tile case: append 1 newline row.
+                if (img->anyres.is_tiled()) {
-                    // For multi-tile rowwise: handled by caller, but here we
+                    // overview tile, then the unpadded tile grid with one newline per row
-                    // report the per-tile count including one trailing newline.
+                    int off_x, off_y, w, h;
-                    n_patches += 1;
+                    clip_anyres_unpad(img->anyres.grid_x * out_side, img->anyres.grid_y * out_side,
+                                      img->anyres.orig_nx, img->anyres.orig_ny, off_x, off_y, w, h);
+                    n_patches += h * (w + 1);
                 }
             } break;
         default:
@@ -5505,10 +5507,18 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
                     return idx;
                 };
 
+                // the same permutation is applied to every tile of the stacked image
                 auto upload = [&](const std::string & name, const std::vector<int32_t> & idx) {
                     ggml_tensor * t = ggml_graph_get_tensor(gf, name.c_str());
                     GGML_ASSERT(t);
-                    ggml_backend_tensor_set(t, idx.data(), 0, idx.size() * sizeof(int32_t));
+                    GGML_ASSERT(ggml_nelements(t) % (int64_t) idx.size() == 0);
+                    const int n_rep = ggml_nelements(t) / idx.size();
+                    std::vector<int32_t> buf;
+                    buf.reserve(idx.size() * n_rep);
+                    for (int i = 0; i < n_rep; ++i) {
+                        buf.insert(buf.end(), idx.begin(), idx.end());
+                    }
+                    ggml_backend_tensor_set(t, buf.data(), 0, ggml_nbytes(t));
                 };
 
                 // Stage 1b only uses block 0's permutations; future stages

tools/mtmd/models/granite4-vision.cpp

@@ -14,18 +14,39 @@
  *   Stage 1a: SigLIP vision tower (N layers, post-norm)
  *   Stage 1b: WindowQFormer blocks (deepstack + spatial)
  *   Stage 1c: Concatenate and pack outputs
- *   Stage 1d: Append newline tokens if add_newline is set
+ *   Stage 1d: Assemble the anyres tiles into one token sequence
  */
 
 // ---------------------------------------------------------------------------
 // Member method implementations
 // ---------------------------------------------------------------------------
 
+// split the stacked tiles into the batch axis, then run the usual patch embedding
+ggml_tensor * clip_graph_granite4_vision::build_tile_inp() {
+    ggml_tensor * inp_raw = build_inp_raw();
+
+    if (n_tiles > 1) {
+        const int px = img.nx();
+        inp_raw = ggml_reshape_4d(ctx0, inp_raw, px * px, n_tiles, 3, 1);
+        inp_raw = ggml_cont(ctx0, ggml_permute(ctx0, inp_raw, 0, 2, 1, 3));
+        inp_raw = ggml_reshape_4d(ctx0, inp_raw, px, px, 3, n_tiles);
+    }
+
+    ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
+    inp = ggml_reshape_3d(ctx0, inp, tile_side * tile_side, n_embd, n_tiles);
+    inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
+    if (model.patch_bias) {
+        inp = ggml_add(ctx0, inp, model.patch_bias);
+    }
+    return inp;
+}
+
 ggml_tensor * clip_graph_granite4_vision::gather(
         ggml_tensor * src,
         const std::string & name,
         int idx_len) {
-    ggml_tensor * idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, idx_len);
+    // one index row per tile, all rows hold the same permutation
+    ggml_tensor * idx = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, idx_len, n_tiles);
     ggml_set_name(idx, name.c_str());
     ggml_set_input(idx);
     return ggml_get_rows(ctx0, src, idx);
@@ -36,12 +57,15 @@ ggml_tensor * clip_graph_granite4_vision::interp_down(
         int side,
         int new_side) {
     const int n_embd = src->ne[0];
-    ggml_tensor * t = ggml_reshape_4d(ctx0, src, n_embd, side, side, 1);
+    ggml_tensor * t = ggml_reshape_4d(ctx0, src, n_embd, side, side, n_tiles);
     t = ggml_cont(ctx0, ggml_permute(ctx0, t, 2, 0, 1, 3));
+    // fold the tile axis into the channel axis, ggml_pool_2d only pools the first two axes
+    t = ggml_reshape_3d(ctx0, t, side, side, n_embd * n_tiles);
     const int kernel = side / new_side;
     t = ggml_pool_2d(ctx0, t, GGML_OP_POOL_AVG, kernel, kernel, kernel, kernel, 0, 0);
+    t = ggml_reshape_4d(ctx0, t, new_side, new_side, n_embd, n_tiles);
     t = ggml_cont(ctx0, ggml_permute(ctx0, t, 1, 2, 0, 3));
-    return ggml_reshape_2d(ctx0, t, n_embd, new_side * new_side);
+    return ggml_reshape_3d(ctx0, t, n_embd, new_side * new_side, n_tiles);
 }
 
 // ---------------------------------------------------------------------------
@@ -63,6 +87,7 @@ ggml_tensor * clip_graph_granite4_vision::build_block(
     const int n = image_side / window_side;
     const int new_side = n * query_side;
     const int n_windows = n * n;
+    const int n_win_all = n_windows * n_tiles; // windows of every tile, batched together
     const int enc_len = window_side * window_side;
     const int query_len = query_side * query_side;
 
@@ -82,7 +107,7 @@ ggml_tensor * clip_graph_granite4_vision::build_block(
         ggml_tensor * enc_flat = gather(x,
             "g4v_blk" + std::to_string(bid) + "_win_idx",
             image_side * image_side);
-        enc = ggml_reshape_3d(ctx0, enc_flat, n_embd, enc_len, n_windows);
+        enc = ggml_reshape_3d(ctx0, enc_flat, n_embd, enc_len, n_win_all);
     }
     cbx(enc, "enc");
 
@@ -104,7 +129,7 @@ ggml_tensor * clip_graph_granite4_vision::build_block(
         ggml_tensor * dw_flat = gather(d,
             "g4v_blk" + std::to_string(bid) + "_qwin_idx",
             new_side * new_side);
-        ggml_tensor * dw = ggml_reshape_3d(ctx0, dw_flat, n_embd, query_len, n_windows);
+        ggml_tensor * dw = ggml_reshape_3d(ctx0, dw_flat, n_embd, query_len, n_win_all);
         q_in = ggml_add(ctx0, dw, blk.qf_proj_query);
     }
     cbx(q_in, "query_embeds");
@@ -140,12 +165,12 @@ ggml_tensor * clip_graph_granite4_vision::build_block(
         ggml_tensor * K = linear(q, pl.k_w, pl.k_b);
         ggml_tensor * V = linear(q, pl.v_w, pl.v_b);
 
-        Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_windows);
+        Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_win_all);
-        K = ggml_reshape_4d(ctx0, K, d_h, n_head, nq, n_windows);
+        K = ggml_reshape_4d(ctx0, K, d_h, n_head, nq, n_win_all);
-        V = ggml_reshape_4d(ctx0, V, d_h, n_head, nq, n_windows);
+        V = ggml_reshape_4d(ctx0, V, d_h, n_head, nq, n_win_all);
 
         sa_out = build_attn(pl.o_w, pl.o_b, Q, K, V, nullptr, scale, bid);
-        sa_out = ggml_reshape_3d(ctx0, sa_out, n_embd, nq, n_windows);
+        sa_out = ggml_reshape_3d(ctx0, sa_out, n_embd, nq, n_win_all);
 
         sa_out = ggml_add(ctx0, sa_out, q);
         sa_out = build_norm(sa_out, pl.ln_1_w, pl.ln_1_b,
@@ -166,13 +191,13 @@ ggml_tensor * clip_graph_granite4_vision::build_block(
         ggml_tensor * K = linear(e_in, pl.cross_attn_k_w, pl.cross_attn_k_b);
         ggml_tensor * V = linear(e_in, pl.cross_attn_v_w, pl.cross_attn_v_b);
 
-        Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_windows);
+        Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_win_all);
-        K = ggml_reshape_4d(ctx0, K, d_h, n_head, nkv, n_windows);
+        K = ggml_reshape_4d(ctx0, K, d_h, n_head, nkv, n_win_all);
-        V = ggml_reshape_4d(ctx0, V, d_h, n_head, nkv, n_windows);
+        V = ggml_reshape_4d(ctx0, V, d_h, n_head, nkv, n_win_all);
 
         ca_out = build_attn(pl.cross_attn_o_w, pl.cross_attn_o_b,
                             Q, K, V, nullptr, scale, bid);
-        ca_out = ggml_reshape_3d(ctx0, ca_out, n_embd, nq, n_windows);
+        ca_out = ggml_reshape_3d(ctx0, ca_out, n_embd, nq, n_win_all);
 
         ca_out = ggml_add(ctx0, ca_out, sa_out);
         ca_out = build_norm(ca_out, pl.cross_attn_norm_w, pl.cross_attn_norm_b,
@@ -183,13 +208,13 @@ ggml_tensor * clip_graph_granite4_vision::build_block(
     // 6c. FFN
     ggml_tensor * ffn;
     {
-        ggml_tensor * t = ggml_reshape_2d(ctx0, ca_out, n_embd, query_len * n_windows);
+        ggml_tensor * t = ggml_reshape_2d(ctx0, ca_out, n_embd, query_len * n_win_all);
         t = build_mm(pl.ff_up_w, t);
         if (pl.ff_up_b) t = ggml_add(ctx0, t, pl.ff_up_b);
         t = ggml_gelu_erf(ctx0, t);
         t = build_mm(pl.ff_down_w, t);
         if (pl.ff_down_b) t = ggml_add(ctx0, t, pl.ff_down_b);
-        t = ggml_reshape_3d(ctx0, t, n_embd, query_len, n_windows);
+        t = ggml_reshape_3d(ctx0, t, n_embd, query_len, n_win_all);
         ffn = ggml_add(ctx0, t, ca_out);
         ffn = build_norm(ffn, pl.ln_2_w, pl.ln_2_b, NORM_TYPE_NORMAL, qformer_eps, bid);
     }
@@ -198,7 +223,7 @@ ggml_tensor * clip_graph_granite4_vision::build_block(
     // 7. _unwin back to raster
     ggml_tensor * unwinned;
     {
-        ggml_tensor * flat = ggml_reshape_2d(ctx0, ffn, n_embd, query_len * n_windows);
+        ggml_tensor * flat = ggml_reshape_3d(ctx0, ffn, n_embd, query_len * n_windows, n_tiles);
         unwinned = gather(flat,
             "g4v_blk" + std::to_string(bid) + "_unwin_idx",
             new_side * new_side);
@@ -244,13 +269,42 @@ ggml_tensor * clip_graph_granite4_vision::build_newline_row(ggml_context * ctx0)
     return ggml_reshape_2d(ctx0, nl_row_2d, n_mmproj_embd, 1);
 }
 
-// Append a single newline row at the end of the tile output.
+// Assemble [overview, tile(0,0), tile(0,1), ...] into one token sequence:
-ggml_tensor * clip_graph_granite4_vision::append_rowwise_newlines(ggml_context * ctx0, ggml_tensor * tile_output) {
+// the overview tokens first, then the tile grid read in raster order with one newline per row.
-    // For the single-tile case, append one newline row at the end.
+// ref: https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/llava_next/modeling_llava_next.py#L266
-    // For the multi-tile rowwise case, this will be called per-tile
+ggml_tensor * clip_graph_granite4_vision::build_anyres_assembly(ggml_tensor * cur, int out_side) {
-    // (though currently only the single-tile path uses it).
+    const int n_dim  = cur->ne[0];
-    ggml_tensor * nl_row = build_newline_row(ctx0);
+    const int grid_x = anyres.grid_x;
-    return ggml_concat(ctx0, tile_output, nl_row, 1);
+    const int grid_y = anyres.grid_y;
+    const int cur_w  = grid_x * out_side;
+    const int cur_h  = grid_y * out_side;
+    GGML_ASSERT(cur->ne[1] == out_side * out_side);
+    GGML_ASSERT(cur->ne[2] == 1 + grid_x * grid_y);
+
+    ggml_tensor * base = ggml_view_2d(ctx0, cur, n_dim, out_side * out_side, cur->nb[1], 0);
+
+    ggml_tensor * tiles = ggml_view_3d(ctx0, cur, n_dim, out_side * out_side, grid_x * grid_y,
+                                       cur->nb[1], cur->nb[2], cur->nb[2]);
+
+    // (n_dim*out_side, out_side, grid_x, grid_y) -> interleave the tiles of a grid row
+    tiles = ggml_reshape_4d(ctx0, tiles, n_dim * out_side, out_side, grid_x, grid_y);
+    tiles = ggml_cont(ctx0, ggml_permute(ctx0, tiles, 0, 2, 1, 3));
+    tiles = ggml_reshape_3d(ctx0, tiles, n_dim, cur_w, cur_h);
+
+    // drop the tokens that only cover the padding added when resizing to the grid
+    int off_x, off_y, w, h;
+    clip_anyres_unpad(cur_w, cur_h, anyres.orig_nx, anyres.orig_ny, off_x, off_y, w, h);
+    if (w != cur_w || h != cur_h) {
+        tiles = ggml_cont(ctx0, ggml_view_3d(ctx0, tiles, n_dim, w, h,
+                                             tiles->nb[1], tiles->nb[2],
+                                             off_x * tiles->nb[1] + off_y * tiles->nb[2]));
+    }
+
+    ggml_tensor * nl = ggml_repeat_4d(ctx0, build_newline_row(ctx0), n_dim, 1, h, 1);
+    tiles = ggml_concat(ctx0, tiles, nl, 1);
+    tiles = ggml_reshape_2d(ctx0, tiles, n_dim, (w + 1) * h);
+
+    return ggml_concat(ctx0, base, tiles, 1);
 }
 
 ggml_cgraph * clip_graph_granite4_vision::build() {
@@ -260,10 +314,12 @@ ggml_cgraph * clip_graph_granite4_vision::build() {
     GGML_ASSERT(!model.qf_proj_blocks.empty());
 
     // --- Stage 1a: SigLIP encoder producing intermediate hidden states ---
-    ggml_tensor * inp = build_inp();
+    ggml_tensor * inp = build_tile_inp();
     inp = ggml_add(ctx0, inp, model.position_embeddings);
     cb(inp, "pos_embed", -1);
 
+    const int tile_n_patches = tile_side * tile_side;
+
     ggml_tensor * inpL = inp;
     std::vector<ggml_tensor *> layer_outs(n_layer, nullptr);
 
@@ -281,12 +337,13 @@ ggml_cgraph * clip_graph_granite4_vision::build() {
         ggml_tensor * Vcur = build_mm(layer.v_w, cur);
         if (layer.v_b) Vcur = ggml_add(ctx0, Vcur, layer.v_b);
 
-        Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_patches);
+        Qcur = ggml_reshape_4d(ctx0, Qcur, d_head, n_head, tile_n_patches, n_tiles);
-        Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_patches);
+        Kcur = ggml_reshape_4d(ctx0, Kcur, d_head, n_head, tile_n_patches, n_tiles);
-        Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_patches);
+        Vcur = ggml_reshape_4d(ctx0, Vcur, d_head, n_head, tile_n_patches, n_tiles);
 
         cur = build_attn(layer.o_w, layer.o_b,
                          Qcur, Kcur, Vcur, nullptr, kq_scale, il);
+        cur = ggml_reshape_3d(ctx0, cur, n_embd, tile_n_patches, n_tiles);
 
         cur = ggml_add(ctx0, cur, inpL);
         inpL = cur;
@@ -318,7 +375,7 @@ ggml_cgraph * clip_graph_granite4_vision::build() {
         ggml_tensor * stream = build_block(
             blk, h, bid,
             hparams.proj_spatial_offsets[bid],
-            n_patches_x,
+            tile_side,
             hparams.downsample_window_side,
             hparams.downsample_query_side,
             qformer_eps);
@@ -326,10 +383,11 @@ ggml_cgraph * clip_graph_granite4_vision::build() {
         mmproj = mmproj ? ggml_concat(ctx0, mmproj, stream, 0) : stream;
     }
 
-    // --- Stage 1d: Append newline tokens if add_newline is set ---
+    // --- Stage 1d: assemble the tiles and weave in the newline tokens ---
-    if (add_newline) {
+    if (anyres.is_tiled()) {
-        mmproj = append_rowwise_newlines(ctx0, mmproj);
+        const int out_side = tile_side / hparams.downsample_window_side * hparams.downsample_query_side;
-        ggml_set_name(mmproj, "g4v_mmproj_out_nl");
+        mmproj = build_anyres_assembly(mmproj, out_side);
+        ggml_set_name(mmproj, "g4v_mmproj_out_anyres");
     } else {
         ggml_set_name(mmproj, "g4v_mmproj_out");
     }

tools/mtmd/models/models.h

@@ -402,16 +402,19 @@ struct clip_graph_exaone4_5 : clip_graph {
 struct clip_graph_granite4_vision : clip_graph {
     clip_graph_granite4_vision(clip_ctx * ctx, const clip_image_f32 & img)
         : clip_graph(ctx, img),
-          add_newline(img.add_newline) {}
+          anyres(img.anyres),
+          n_tiles(img.ny() / img.nx()),
+          tile_side(img.nx() / patch_size) {}
 
     ggml_cgraph * build() override;
 
 private:
-    // The graph is per-tile since only batch-size 1 is supported in clip. As
+    // the input image is a stack of tiles on the Y axis: [overview, tile(0,0), tile(0,1), ...]
-    // such, this value is set at construct time based on the tile that will be
+    const clip_image_f32::anyres_info anyres;
-    // encoded, then used during build to determine how to handle newlines.
+    const int n_tiles;
-    const bool add_newline;
+    const int tile_side; // patches per tile side
 
+    ggml_tensor * build_tile_inp();
     ggml_tensor * gather(ggml_tensor * src, const std::string & name, int idx_len);
     ggml_tensor * interp_down(ggml_tensor * src, int side, int new_side);
     ggml_tensor * build_block(const qf_block & blk, ggml_tensor * h, int bid,
@@ -419,7 +422,7 @@ private:
                               int query_side, float qformer_eps);
 
     ggml_tensor * build_newline_row(ggml_context * ctx0);
-    ggml_tensor * append_rowwise_newlines(ggml_context * ctx0, ggml_tensor * tile_output);
+    ggml_tensor * build_anyres_assembly(ggml_tensor * cur, int out_side);
 };
 
 struct clip_graph_muse_glimmer : clip_graph {

tools/mtmd/mtmd-image.cpp

@@ -1602,17 +1602,50 @@ mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_im
 }
 
 mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_image_u8 & img) {
-    auto output = mtmd_image_preprocessor_llava_uhd::preprocess(img);
+    GGML_ASSERT(!hparams.image_res_candidates.empty());
-    if (output.entries.size() == 0) {
+
-        // Single-tile (overview only): append one newline row.
+    const clip_image_size orig_size = img.get_size();
-        output.overview.add_newline = true;
+    const int             tile_size = hparams.image_size;
-    } else {
+
-        // Multi-tile: overview gets no newline, grid tiles get one.
+    // llava-next always encodes an overview plus a grid of tiles, even for small images
-        output.overview.add_newline = false;
+    const clip_image_size refined_size = select_best_resolution(orig_size, hparams.image_res_candidates);
-        for (size_t i = 0; i < output.entries.size(); ++i) {
+    const int             grid_x       = refined_size.width  / tile_size;
-            output.entries[i].add_newline = true;
+    const int             grid_y       = refined_size.height / tile_size;
+
+    clip_image_u8 overview;
+    img_tool::resize(img, overview, {tile_size, tile_size}, hparams.image_resize_algo_ov,
+                        hparams.image_pad_ov, hparams.image_pad_color_ov);
+
+    clip_image_u8 refined;
+    img_tool::resize(img, refined, refined_size, hparams.image_resize_algo_rf,
+                        hparams.image_pad_rf, hparams.image_pad_color_rf);
+
+    // stack the overview and the tiles on the Y axis, so the whole grid goes through one graph
+    clip_image_u8 stacked;
+    stacked.set_size({tile_size, tile_size * (1 + grid_x * grid_y)}, false);
+    auto copy_tile = [&](const clip_image_u8 & src, int src_x, int src_y, int dst_idx) {
+        for (int py = 0; py < tile_size; py++) {
+            for (int px = 0; px < tile_size; px++) {
+                stacked.set_pixel(px, dst_idx * tile_size + py, src.get_pixel(src_x + px, src_y + py));
+            }
+        }
+    };
+    copy_tile(overview, 0, 0, 0);
+    for (int ty = 0; ty < grid_y; ty++) {
+        for (int tx = 0; tx < grid_x; tx++) {
+            copy_tile(refined, tx * tile_size, ty * tile_size, 1 + ty * grid_x + tx);
         }
     }
+
+    LOG_DBG("%s: grid size: %d x %d (%d tiles) + overview\n", __func__, grid_x, grid_y, grid_x * grid_y);
+
+    mtmd_image_preproc_out output;
+    output.append(hparams, stacked, true);
+    auto & entry = output.entries.back();
+    entry.anyres.grid_x  = grid_x;
+    entry.anyres.grid_y  = grid_y;
+    entry.anyres.orig_nx = orig_size.width;
+    entry.anyres.orig_ny = orig_size.height;
     return output;
 }
 

tools/mtmd/mtmd-image.h

@@ -85,9 +85,6 @@ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor {
 protected:
     clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false);
 
-private:
-    clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max);
-
     /**
      * Selects the best resolution from a list of possible resolutions based on the original size.
      *
@@ -104,6 +101,9 @@ private:
      * @return The best fit resolution
      */
     clip_image_size select_best_resolution(const clip_image_size & original_size, const std::vector<clip_image_size> & possible_resolutions);
+
+private:
+    clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max);
     int ensure_divide(int length, int patch_size);
     clip_image_size get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false);
     clip_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio);
@@ -225,7 +225,7 @@ struct mtmd_image_preprocessor_youtuvl : mtmd_image_preprocessor {
     mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
 };
 
-// similar to llava_uhd, but has add_newline
+// llava-next "anyres": stacks the overview and all tiles into one image, assembled by clip in a single graph
 struct mtmd_image_preprocessor_granite : mtmd_image_preprocessor_llava_uhd {
     mtmd_image_preprocessor_granite(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
     mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;

tools/mtmd/mtmd.cpp

@@ -891,10 +891,10 @@ struct mtmd_context {
                 } break;
             case PROJECTOR_TYPE_GRANITE4_VISION:
                 {
-                    img_beg = "<image>";
+                    // ... (image embeddings) \n ...
-                    img_end = "";
+                    img_beg = "";
+                    img_end = "\n";
                     image_preproc = std::make_unique<mtmd_image_preprocessor_granite>(ctx_v);
-                    ov_img_first = true;
                 } break;
             default:
                 throw std::runtime_error(string_format("%s: unexpected vision projector type %d\n", __func__, proj));