Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: Unlimited-OCR fix max_tiles, setting in converter (#25614)

Browse Source
This commit is contained in:
Saba Fallah
2026-08-05 15:30:14 +02:00
committed by GitHub
parent cd0fa6051a
commit b06aa774c0
2 changed files with 21 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/deepseek.py
+14 -1
View File
@@ -17,8 +17,11 @@ from .base import LazyTorchTensor, MmprojModel, ModelBase, TextModel, gguf, logg
from .qwen import QwenModel from .qwen import QwenModel
@ModelBase.register("DeepseekOCRForCausalLM", "UnlimitedOCRForCausalLM") @ModelBase.register("DeepseekOCRForCausalLM")
class DeepseekOCRVisionModel(MmprojModel): class DeepseekOCRVisionModel(MmprojModel):
# HF dynamic_preprocess() max_num, which differs per model
preproc_max_tiles = 9
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR
@@ -43,6 +46,9 @@ class DeepseekOCRVisionModel(MmprojModel):
# @bluebread: there's no window_size in config but just add it here anyway # @bluebread: there's no window_size in config but just add it here anyway
self.gguf_writer.add_vision_window_size(self.hparams.get("window_size", 14)) self.gguf_writer.add_vision_window_size(self.hparams.get("window_size", 14))
self.gguf_writer.add_vision_preproc_min_tiles(2)
self.gguf_writer.add_vision_preproc_max_tiles(self.preproc_max_tiles)
# SAM configuration # SAM configuration
sam_hparams = hparams['sam'] sam_hparams = hparams['sam']
self.gguf_writer.add_vision_sam_layers_count(sam_hparams['layers']) self.gguf_writer.add_vision_sam_layers_count(sam_hparams['layers'])
@@ -93,8 +99,15 @@ class DeepseekOCRVisionModel(MmprojModel):
return super().filter_tensors((name, gen)) return super().filter_tensors((name, gen))
@ModelBase.register("UnlimitedOCRForCausalLM")
class UnlimitedOCRVisionModel(DeepseekOCRVisionModel):
preproc_max_tiles = 32
@ModelBase.register("DeepseekOCR2ForCausalLM") @ModelBase.register("DeepseekOCR2ForCausalLM")
class DeepseekOCR2VisionModel(DeepseekOCRVisionModel): class DeepseekOCR2VisionModel(DeepseekOCRVisionModel):
preproc_max_tiles = 6
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR2 self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR2
tools/mtmd/clip.cpp
+7
View File
@@ -1761,6 +1761,10 @@ struct clip_model_loader {
// qwen2 encoder is GQA, requires KEY_N_HEAD_KV // qwen2 encoder is GQA, requires KEY_N_HEAD_KV
get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv); get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
} }
// unlimited-ocr shares the v1 projector but tiles up to 32
get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false);
get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false);
GGML_ASSERT(hparams.preproc_min_tiles <= hparams.preproc_max_tiles);
} break; } break;
case PROJECTOR_TYPE_HUNYUANVL: case PROJECTOR_TYPE_HUNYUANVL:
{ {
@@ -1909,6 +1913,9 @@ struct clip_model_loader {
if (hparams.image_max_pixels > 0) { if (hparams.image_max_pixels > 0) {
LOG_INF("%s: image_max_pixels: %d%s\n", __func__, hparams.image_max_pixels, hparams.custom_image_max_tokens > 0 ? " (custom value)" : ""); LOG_INF("%s: image_max_pixels: %d%s\n", __func__, hparams.image_max_pixels, hparams.custom_image_max_tokens > 0 ? " (custom value)" : "");
} }
if (hparams.preproc_max_tiles > 0) {
LOG_INF("%s: preproc_tiles: %d - %d\n", __func__, hparams.preproc_min_tiles, hparams.preproc_max_tiles);
}
} else if (is_audio) { } else if (is_audio) {
LOG_INF("\n--- audio hparams ---\n"); LOG_INF("\n--- audio hparams ---\n");
LOG_INF("%s: n_mel_bins: %d\n", __func__, hparams.n_mel_bins); LOG_INF("%s: n_mel_bins: %d\n", __func__, hparams.n_mel_bins);