Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: fix Granite4 Vision image sequence assembly (#26653)

Browse Source 
* mtmd: fix granite 4v grid assembly

(cherry picked from commit 91f82eb1b489bff0dc3f649edf2cb9e0b7f656e9)

* mtmd: fix truncation for scaled image height and width before unpad

Signed-off-by: Hemanth Battu <hbattu@ibm.com>

* mtmd: remove MTMD_DUMP_EMBD debug scaffolding

Signed-off-by: Hemanth Battu <hbattu@ibm.com>

* clean up comments, clarify about anyres_info excluded from serialization

* add_newline is now dead code

---------

Signed-off-by: Hemanth Battu <hbattu@ibm.com>
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
Co-authored-by: Hemanth Battu <hbattu@ibm.com>
This commit is contained in:
Hemanth Battu
2026-08-15 00:25:26 +02:00
committed by GitHub
co-authored by Xuan Son Nguyen Hemanth Battu
parent 16d222fc5e
commit 9d57ce456c
7 changed files with 205 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip-impl.h
+37 -1
View File
@@ -6,6 +6,7 @@
#include <array>
#include <climits>
#include <cmath>
#include <cstdarg>
#include <cinttypes>
#include <string>
@@ -642,9 +643,25 @@ struct mtmd_serialization; // forward declaration
struct clip_image_f32 {
// marks the global view in e.g., DeepSeek-OCR Models
bool add_viewsep = false;
// whether a learned newline (or EOI) token should be appended after the image (eg Granite4 Vision)
// appends a learned newline (or EOI) token after the image
// no model uses it now (Granite4 Vision moved to anyres), kept for future models
bool add_newline = false;
// llava-next "anyres" tiling, used by Granite4 Vision
// the whole grid is encoded and assembled in a single graph
// NOTE: excluded from serialized: a deserialized image is always a placeholder, which is never encoded
struct anyres_info {
int grid_x = 0; // tiles per row, 0 means the image is not tiled
int grid_y = 0; // tiles per column
int orig_nx = 0; // size of the source image, used to drop the padding tokens
int orig_ny = 0;
bool is_tiled() const {
return grid_x > 0 && grid_y > 0;
}
};
anyres_info anyres;
clip_image_size get_size() const {
return { nx_, ny_ };
}
@@ -726,6 +743,25 @@ struct clip_image_f32 {
}
};
// token area kept after removing the padding added by the anyres resize
// ref: https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/llava_next/modeling_llava_next.py#L109
static inline void clip_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_h,
int & off_x, int & off_y, int & out_w, int & out_h) {
off_x = 0;
off_y = 0;
out_w = cur_w;
out_h = cur_h;
if ((float) orig_w / orig_h > (float) cur_w / cur_h) {
const int new_h = (int) std::floor((double) orig_h * cur_w / orig_w + 1e-7);
off_y = (cur_h - new_h) / 2;
out_h = cur_h - 2 * off_y;
} else {
const int new_w = (int) std::floor((double) orig_w * cur_h / orig_h + 1e-7);
off_x = (cur_w - new_w) / 2;
out_w = cur_w - 2 * off_x;
}
}
//
// logging
//