Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: fix Granite4 Vision image sequence assembly (#26653)

Browse Source 
* mtmd: fix granite 4v grid assembly

(cherry picked from commit 91f82eb1b489bff0dc3f649edf2cb9e0b7f656e9)

* mtmd: fix truncation for scaled image height and width before unpad

Signed-off-by: Hemanth Battu <hbattu@ibm.com>

* mtmd: remove MTMD_DUMP_EMBD debug scaffolding

Signed-off-by: Hemanth Battu <hbattu@ibm.com>

* clean up comments, clarify about anyres_info excluded from serialization

* add_newline is now dead code

---------

Signed-off-by: Hemanth Battu <hbattu@ibm.com>
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
Co-authored-by: Hemanth Battu <hbattu@ibm.com>
This commit is contained in:
Hemanth Battu
2026-08-15 00:25:26 +02:00
committed by GitHub
co-authored by Xuan Son Nguyen Hemanth Battu
parent 16d222fc5e
commit 9d57ce456c
7 changed files with 205 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/mtmd-image.cpp
+42 -9
View File
@@ -1602,17 +1602,50 @@ mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_im
}
mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_image_u8 & img) {
auto output = mtmd_image_preprocessor_llava_uhd::preprocess(img);
if (output.entries.size() == 0) {
// Single-tile (overview only): append one newline row.
output.overview.add_newline = true;
} else {
// Multi-tile: overview gets no newline, grid tiles get one.
output.overview.add_newline = false;
for (size_t i = 0; i < output.entries.size(); ++i) {
output.entries[i].add_newline = true;
GGML_ASSERT(!hparams.image_res_candidates.empty());
const clip_image_size orig_size = img.get_size();
const int tile_size = hparams.image_size;
// llava-next always encodes an overview plus a grid of tiles, even for small images
const clip_image_size refined_size = select_best_resolution(orig_size, hparams.image_res_candidates);
const int grid_x = refined_size.width / tile_size;
const int grid_y = refined_size.height / tile_size;
clip_image_u8 overview;
img_tool::resize(img, overview, {tile_size, tile_size}, hparams.image_resize_algo_ov,
hparams.image_pad_ov, hparams.image_pad_color_ov);
clip_image_u8 refined;
img_tool::resize(img, refined, refined_size, hparams.image_resize_algo_rf,
hparams.image_pad_rf, hparams.image_pad_color_rf);
// stack the overview and the tiles on the Y axis, so the whole grid goes through one graph
clip_image_u8 stacked;
stacked.set_size({tile_size, tile_size * (1 + grid_x * grid_y)}, false);
auto copy_tile = [&](const clip_image_u8 & src, int src_x, int src_y, int dst_idx) {
for (int py = 0; py < tile_size; py++) {
for (int px = 0; px < tile_size; px++) {
stacked.set_pixel(px, dst_idx * tile_size + py, src.get_pixel(src_x + px, src_y + py));
}
}
};
copy_tile(overview, 0, 0, 0);
for (int ty = 0; ty < grid_y; ty++) {
for (int tx = 0; tx < grid_x; tx++) {
copy_tile(refined, tx * tile_size, ty * tile_size, 1 + ty * grid_x + tx);
}
}
LOG_DBG("%s: grid size: %d x %d (%d tiles) + overview\n", __func__, grid_x, grid_y, grid_x * grid_y);
mtmd_image_preproc_out output;
output.append(hparams, stacked, true);
auto & entry = output.entries.back();
entry.anyres.grid_x = grid_x;
entry.anyres.grid_y = grid_y;
entry.anyres.orig_nx = orig_size.width;
entry.anyres.orig_ny = orig_size.height;
return output;
}