Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: fix Granite4 Vision image sequence assembly (#26653)

Browse Source 
* mtmd: fix granite 4v grid assembly

(cherry picked from commit 91f82eb1b489bff0dc3f649edf2cb9e0b7f656e9)

* mtmd: fix truncation for scaled image height and width before unpad

Signed-off-by: Hemanth Battu <hbattu@ibm.com>

* mtmd: remove MTMD_DUMP_EMBD debug scaffolding

Signed-off-by: Hemanth Battu <hbattu@ibm.com>

* clean up comments, clarify about anyres_info excluded from serialization

* add_newline is now dead code

---------

Signed-off-by: Hemanth Battu <hbattu@ibm.com>
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
Co-authored-by: Hemanth Battu <hbattu@ibm.com>
This commit is contained in:
Hemanth Battu
2026-08-15 00:25:26 +02:00
committed by GitHub
co-authored by Xuan Son Nguyen Hemanth Battu
parent 16d222fc5e
commit 9d57ce456c
7 changed files with 205 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/models/models.h
+9 -6
View File
@@ -402,16 +402,19 @@ struct clip_graph_exaone4_5 : clip_graph {
struct clip_graph_granite4_vision : clip_graph {
clip_graph_granite4_vision(clip_ctx * ctx, const clip_image_f32 & img)
: clip_graph(ctx, img),
add_newline(img.add_newline) {}
anyres(img.anyres),
n_tiles(img.ny() / img.nx()),
tile_side(img.nx() / patch_size) {}
ggml_cgraph * build() override;
private:
// The graph is per-tile since only batch-size 1 is supported in clip. As
// such, this value is set at construct time based on the tile that will be
// encoded, then used during build to determine how to handle newlines.
const bool add_newline;
// the input image is a stack of tiles on the Y axis: [overview, tile(0,0), tile(0,1), ...]
const clip_image_f32::anyres_info anyres;
const int n_tiles;
const int tile_side; // patches per tile side
ggml_tensor * build_tile_inp();
ggml_tensor * gather(ggml_tensor * src, const std::string & name, int idx_len);
ggml_tensor * interp_down(ggml_tensor * src, int side, int new_side);
ggml_tensor * build_block(const qf_block & blk, ggml_tensor * h, int bid,
@@ -419,7 +422,7 @@ private:
int query_side, float qformer_eps);
ggml_tensor * build_newline_row(ggml_context * ctx0);
ggml_tensor * append_rowwise_newlines(ggml_context * ctx0, ggml_tensor * tile_output);
ggml_tensor * build_anyres_assembly(ggml_tensor * cur, int out_side);
};
struct clip_graph_muse_glimmer : clip_graph {