Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: fix Granite4 Vision image sequence assembly (#26653)

Browse Source 
* mtmd: fix granite 4v grid assembly

(cherry picked from commit 91f82eb1b489bff0dc3f649edf2cb9e0b7f656e9)

* mtmd: fix truncation for scaled image height and width before unpad

Signed-off-by: Hemanth Battu <hbattu@ibm.com>

* mtmd: remove MTMD_DUMP_EMBD debug scaffolding

Signed-off-by: Hemanth Battu <hbattu@ibm.com>

* clean up comments, clarify about anyres_info excluded from serialization

* add_newline is now dead code

---------

Signed-off-by: Hemanth Battu <hbattu@ibm.com>
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
Co-authored-by: Hemanth Battu <hbattu@ibm.com>
This commit is contained in:
Hemanth Battu
2026-08-15 00:25:26 +02:00
committed by GitHub
co-authored by Xuan Son Nguyen Hemanth Battu
parent 16d222fc5e
commit 9d57ce456c
7 changed files with 205 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/mtmd-image.h
+4 -4
View File
@@ -85,9 +85,6 @@ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor {
protected:
clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false);
private:
clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max);
/**
* Selects the best resolution from a list of possible resolutions based on the original size.
*
@@ -104,6 +101,9 @@ private:
* @return The best fit resolution
*/
clip_image_size select_best_resolution(const clip_image_size & original_size, const std::vector<clip_image_size> & possible_resolutions);
private:
clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max);
int ensure_divide(int length, int patch_size);
clip_image_size get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false);
clip_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio);
@@ -225,7 +225,7 @@ struct mtmd_image_preprocessor_youtuvl : mtmd_image_preprocessor {
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};
// similar to llava_uhd, but has add_newline
// llava-next "anyres": stacks the overview and all tiles into one image, assembled by clip in a single graph
struct mtmd_image_preprocessor_granite : mtmd_image_preprocessor_llava_uhd {
mtmd_image_preprocessor_granite(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;