diff --git a/conversion/chatterbox.py b/conversion/chatterbox.py index bca0a2633..d438228e5 100644 --- a/conversion/chatterbox.py +++ b/conversion/chatterbox.py @@ -355,7 +355,7 @@ class ChatterboxMmprojModel(MmprojModel): def set_gguf_parameters(self): self.gguf_writer.add_file_type(self.ftype) - # speaker encoder, the front-end hparams are hard-coded in clip.cpp + # speaker encoder, the front-end hparams are hard-coded in tools/mtmd self.gguf_writer.add_clip_has_audio_encoder(True) self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.CHATTERBOX_SPKENC) self.gguf_writer.add_audio_projection_dim(80) diff --git a/conversion/deepseek.py b/conversion/deepseek.py index 581111da5..d559b604e 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -1078,7 +1078,7 @@ class DeepseekV4FlashVisionModel(MmprojModel): self.gguf_writer.add_vision_use_silu(True) # SwiGLU MLP self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"]) self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"]) - # hardcoded on the C++ side (see PROJECTOR_TYPE_DEEPSEEK4V in clip.cpp) + # hardcoded on the C++ side (see PROJECTOR_TYPE_DEEPSEEK4V in tools/mtmd) # if future models use different values, add GGUF keys for those assert self.global_config["vision_max_n_token"] == 384 assert self.global_config["vision_max_wh_ratio"] == 8 diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py index 53980c1d0..6b098a037 100644 --- a/conversion/deepseek41.py +++ b/conversion/deepseek41.py @@ -458,7 +458,7 @@ class DeepseekV41VisionModel(DeepseekV4FlashVisionModel): self.gguf_writer.add_vision_use_silu(True) self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"]) self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"]) - # hardcoded in clip.cpp (PROJECTOR_TYPE_DEEPSEEK41V); add GGUF keys if a later model differs + # hardcoded in tools/mtmd (PROJECTOR_TYPE_DEEPSEEK41V); add GGUF keys if a later model differs assert self.hparams_vision["max_image_tokens"] == 1024, self.hparams_vision["max_image_tokens"] assert self.hparams_vision["max_wh_ratio"] is None, self.hparams_vision["max_wh_ratio"] assert self.hparams_vision["rope_theta"] == 10000, self.hparams_vision["rope_theta"] diff --git a/conversion/drama3.py b/conversion/drama3.py index 7c590a014..ac01f481c 100644 --- a/conversion/drama3.py +++ b/conversion/drama3.py @@ -77,7 +77,7 @@ class Drama3MmprojModel(MmprojModel): self.gguf_writer.add_gen_audio_head_count(cfg["n_head"]) self.gguf_writer.add_gen_audio_head_count_kv(cfg["n_local_heads"]) self.gguf_writer.add_gen_audio_attention_layernorm_eps(cfg["norm_eps"]) - # other tower and codec hparams are hardcoded in clip.cpp + # other tower and codec hparams are hardcoded in tools/mtmd @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: diff --git a/conversion/lfm3.py b/conversion/lfm3.py index a0287d4bb..38fbb5a31 100644 --- a/conversion/lfm3.py +++ b/conversion/lfm3.py @@ -181,11 +181,11 @@ class LFM3VLVisionMixin(MmprojModel): def _set_lfm3vl_gguf_parameters(self): assert self.hparams_vision is not None - # clip.cpp hardcodes the GELU projector and its input norm eps + # mtmd hardcodes the GELU projector and its input norm eps if (self.global_config["projector_hidden_act"] != "gelu" or not self.global_config["projector_bias"] or self.global_config["projector_layer_norm_eps"] != 1e-6): raise ValueError("unsupported projector_hidden_act / projector_bias / projector_layer_norm_eps") - # clip.cpp hardcodes the vision rope base, the RGB patchifier and the 3x3 spatial conv + # mtmd hardcodes the vision rope base, the RGB patchifier and the 3x3 spatial conv if (self.hparams_vision["rope_parameters"]["rope_theta"] != 1e6 or self.hparams_vision["in_channels"] != 3 or self.hparams_vision["conv_kernel_size"] != 3): raise ValueError("unsupported vision rope_theta / in_channels / conv_kernel_size") @@ -347,7 +347,7 @@ class LFM3AudioMmprojModel(MmprojModel): "feed_forward.w3": gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP, "feed_forward.w2": gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN, } - # the codec graph in clip.cpp hardcodes the liquid-codec FSQ front end and decoder shape, conversion fails on a mismatch + # the codec graph in mtmd hardcodes the liquid-codec FSQ front end and decoder shape, conversion fails on a mismatch _codec = { "model_type": "liquid_codec", "fsq_levels": [8, 7, 6, 6], "n_q": 12, "quantizer_dim": 256, "encoder_dim": 512, "stft_n_fft": 1280, "stft_hop_length": 320, "stft_win_length": 1280, "stft_frame_stack": 6, @@ -405,7 +405,7 @@ class LFM3AudioMmprojModel(MmprojModel): if fe[k] != v: raise ValueError(f"unsupported feature_extractor.{k}={fe[k]!r}, expected {v!r}") - # clip.cpp hardcodes the encoder rope base + # mtmd hardcodes the encoder rope base if self.hparams_audio["rope_parameters"]["rope_theta"] != 1e6: raise ValueError("unsupported audio encoder rope_theta") @@ -420,7 +420,7 @@ class LFM3AudioMmprojModel(MmprojModel): if not self._audio_output: return - # audio generation: depthformer hparams, the codec ones are constant and live in clip.cpp + # audio generation: depthformer hparams, the codec ones are constant and live in mtmd gen = self.get_generation_config() for k, v in self._generation.items(): if gen[k] != v: diff --git a/conversion/minicpm.py b/conversion/minicpm.py index d8fceb0c0..8c7be8a9a 100644 --- a/conversion/minicpm.py +++ b/conversion/minicpm.py @@ -161,7 +161,7 @@ class MiniCPMV4_6VisionModel(MmprojModel): # In MiniCPM-V 4.6 `vision_config.image_size` (980) describes the SigLIP # positional embedding bucket grid (70 x 70), while the per-slice processing # resolution is the preprocessor's `scale_resolution` (typically 448). - # The CLIP loader in tools/mtmd/clip.cpp consumes `clip.vision.image_size` + # The mmproj loader in tools/mtmd/mtmd-model.cpp consumes `clip.vision.image_size` # as the slice size and warmup resolution, so report `scale_resolution` there # to match the upstream MiniCPMV4_6ImageProcessorPil slicing rules. scale_resolution = self.preprocessor_config.get( @@ -173,7 +173,7 @@ class MiniCPMV4_6VisionModel(MmprojModel): super().set_gguf_parameters() assert self.hparams_vision is not None - # projector type string is consumed by clip_projector_type_from_string() in clip.cpp + # projector type string is consumed by projector_type_from_string() in tools/mtmd/mtmd-impl.h self.gguf_writer.add_clip_projector_type(self.projector_type) self.gguf_writer.add_vision_projector_scale_factor( @@ -243,7 +243,7 @@ class MiniCPMV4_7VisionModel(MiniCPMV4_6VisionModel): projector_type = gguf.VisionProjectorType.MINICPMV4_7 # MiniCPMV4_7ImageProcessorPil default default_scale_resolution = 448 - # rows of v.tok_embd_sep, the order must match clip_suffix_rows() in clip-impl.h + # rows of v.tok_embd_sep, the order must match mtmd_suffix_rows() in tools/mtmd/mtmd-impl.h tok_embd_sep = ["", "", "", "\n"] def get_downsample_mode(self) -> str: diff --git a/conversion/nemotron.py b/conversion/nemotron.py index 066149744..c818a73e1 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -208,7 +208,7 @@ class NemotronNanoV2VLModel(MmprojModel): return name, gen def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - # RADIO's pos_embed doesn't have .weight suffix, but clip.cpp expects it + # RADIO's pos_embed doesn't have .weight suffix, but mtmd expects it if "patch_generator.pos_embed" in name: # Downsample position embeddings for fixed 512x512 image size import torch.nn.functional as F diff --git a/conversion/pockettts.py b/conversion/pockettts.py index 1b5c84111..c9198a221 100644 --- a/conversion/pockettts.py +++ b/conversion/pockettts.py @@ -302,7 +302,7 @@ class PocketTTSMmprojModel(MmprojModel): # voice-prompt encoder: mimi encoder + speaker_proj if self.has_audio_encoder: self.gguf_writer.add_clip_has_audio_encoder(True) - # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models + # note: the 24kHz sample rate is hardcoded on the mtmd side, like the other audio models self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC) self.gguf_writer.add_audio_projection_dim(self.n_embd_text) self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"]) @@ -314,7 +314,7 @@ class PocketTTSMmprojModel(MmprojModel): self.gguf_writer.add_audio_num_mel_bins(1) # generation: flow-matching decoder + mimi decoder - # the SEANet and flow net hparams are constant across the family, clip.cpp holds them + # the SEANet and flow net hparams are constant across the family, mtmd holds them self.gguf_writer.add_clip_has_gen_audio_encoder(True) self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_GEN) self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text) diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py index cdb1e0c9b..28bb6e095 100644 --- a/conversion/qwen3tts.py +++ b/conversion/qwen3tts.py @@ -235,11 +235,11 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel): # handle speaker encoder config self.gguf_writer.add_audio_projection_dim(self.n_embd_text) - # mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the clip.cpp default) + # mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the mtmd default) self.gguf_writer.add_audio_num_mel_bins(128) # 3 SE-Res2Net stages; the stem conv, mfa, asp and fc are not counted here self.gguf_writer.add_audio_block_count(3) - # ECAPA-TDNN has no attention/FFN, these are dummy to allow clip.cpp to load it + # ECAPA-TDNN has no attention/FFN, these are dummy to allow mtmd to load it self.gguf_writer.add_audio_embedding_length(1536) self.gguf_writer.add_audio_head_count(1) self.gguf_writer.add_audio_feed_forward_length(1536) @@ -256,7 +256,7 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel): self.gguf_writer.add_gen_audio_head_count(code_predictor_config["num_attention_heads"]) self.gguf_writer.add_gen_audio_head_count_kv(code_predictor_config["num_key_value_heads"]) self.gguf_writer.add_gen_audio_attention_layernorm_eps(code_predictor_config["rms_norm_eps"]) - # note: code2wav hparams are hardcoded on the mtmd/clip.cpp side for now, not written here + # note: code2wav hparams are hardcoded on the mtmd side for now, not written here def _wav_decoder_config(self) -> dict[str, Any] | None: # code2wav has its own config.json, inside the speech_tokenizer dir diff --git a/docs/development/HOWTO-add-model.md b/docs/development/HOWTO-add-model.md index abf6503f2..558363f4e 100644 --- a/docs/development/HOWTO-add-model.md +++ b/docs/development/HOWTO-add-model.md @@ -128,7 +128,7 @@ Note: to debug the inference graph: you can use [llama-eval-callback](/examples/ If the new model supports multimodal inputs, you will need to add a new encoder definition in `libmtmd`. You can find more information about llama.cpp's multimodal support in [the docs](../multimodal.md) and in the `tools/mtmd` source directory. 1. In the conversion script, make sure you add a subclass that extends `MmprojModel` or another class that inherits from the same base class. -2. Add the encoder definition in `clip.cpp`. +2. Add the encoder definition: hparams and tensor loading in `mtmd-model.cpp`, graph inputs in `mtmd-subcontext.cpp`. 3. Implement the preprocessor in `mtmd.cpp`. In most cases, you can reuse an existing preprocessor. 4. Implement the encoder GGML graph, either in a dedicated file if the model is truly different from existing ones, or by reusing an existing implementation (for example: siglip, pixtral, or qwen) and adding a model-specific projector. diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 61c51c8fd..5f9cbf1e3 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -589,7 +589,7 @@ class GGUFType: class MODEL_ARCH(IntEnum): - MMPROJ = auto() # dummy arch for clip.cpp + MMPROJ = auto() # dummy arch for mmproj LLAMA = auto() LLAMA4 = auto() DECI = auto() @@ -1508,7 +1508,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { - MODEL_ARCH.MMPROJ: "clip", # dummy arch for clip.cpp + MODEL_ARCH.MMPROJ: "clip", # dummy arch for mmproj files (tools/mtmd) MODEL_ARCH.LLAMA: "llama", MODEL_ARCH.LLAMA4: "llama4", MODEL_ARCH.DECI: "deci", @@ -2200,14 +2200,14 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.A_GEN_CODE_EMBD: "a.gen.code.embd", MODEL_TENSOR.A_GEN_CODE_HEAD: "a.gen.code.head", MODEL_TENSOR.A_GEN_CODE_OUT_EMBD: "a.gen.code.out_embd", - MODEL_TENSOR.A_GEN_CODE_ATTN_NORM: "a.gen.code.blk.{bid}.ln1", # reuses the generic clip.cpp block loader (TN_LN_1) + MODEL_TENSOR.A_GEN_CODE_ATTN_NORM: "a.gen.code.blk.{bid}.ln1", # reuses the generic mtmd block loader (TN_LN_1) MODEL_TENSOR.A_GEN_CODE_ATTN_Q: "a.gen.code.blk.{bid}.attn_q", MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM: "a.gen.code.blk.{bid}.attn_q_norm", MODEL_TENSOR.A_GEN_CODE_ATTN_K: "a.gen.code.blk.{bid}.attn_k", MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM: "a.gen.code.blk.{bid}.attn_k_norm", MODEL_TENSOR.A_GEN_CODE_ATTN_V: "a.gen.code.blk.{bid}.attn_v", MODEL_TENSOR.A_GEN_CODE_ATTN_OUT: "a.gen.code.blk.{bid}.attn_out", - MODEL_TENSOR.A_GEN_CODE_FFN_NORM: "a.gen.code.blk.{bid}.ln2", # reuses the generic clip.cpp block loader (TN_LN_2) + MODEL_TENSOR.A_GEN_CODE_FFN_NORM: "a.gen.code.blk.{bid}.ln2", # reuses the generic mtmd block loader (TN_LN_2) MODEL_TENSOR.A_GEN_CODE_FFN_GATE: "a.gen.code.blk.{bid}.ffn_gate", MODEL_TENSOR.A_GEN_CODE_FFN_UP: "a.gen.code.blk.{bid}.ffn_up", MODEL_TENSOR.A_GEN_CODE_FFN_DOWN: "a.gen.code.blk.{bid}.ffn_down", diff --git a/skills/add-new-model/SKILL.md b/skills/add-new-model/SKILL.md index 710a1ebb4..3e4d178eb 100644 --- a/skills/add-new-model/SKILL.md +++ b/skills/add-new-model/SKILL.md @@ -50,7 +50,7 @@ Skill-specific addition: before writing `src/models/.cpp`, read at least 1 ## Step 4 - Optional: multimodal encoder -Only do this if the contributor flagged a vision/audio encoder in Step 0. Follow HOWTO-add-model.md section 4 and `docs/multimodal.md` for the actual touch points (`MmprojModel` subclass, `clip.cpp`, `mtmd.cpp`, encoder graph in `tools/mtmd/models`, etc.). +Only do this if the contributor flagged a vision/audio encoder in Step 0. Follow HOWTO-add-model.md section 4 and `docs/multimodal.md` for the actual touch points (`MmprojModel` subclass, `mtmd-model.cpp`, `mtmd-subcontext.cpp`, `mtmd.cpp`, encoder graph in `tools/mtmd/models`, etc.). Skill-specific addition, and read this carefully: **whether the multimodal encoder can be bundled into the same PR as the base text-model support depends on how conventional the change is.** It's OK to bundle it if the encoder support is conventional - i.e. no new infra or logic is needed, it's just a new cgraph reusing existing preprocessing/projector machinery (e.g. siglip/pixtral/qwen with just a new projector). If it requires anything beyond that - a new preprocessor, non-standard projector logic, or changes to shared `libmtmd` infra/logic - STOP, tell the contributor this is non-conventional, and have them land the text model first with the encoder as a dedicated follow-up PR. Do not let this decision pass silently - call it out explicitly to the contributor before writing any `clip.cpp`/`mtmd.cpp` code. diff --git a/src/models/clip.cpp b/src/models/clip.cpp index 537766aeb..3bf7ac3b8 100644 --- a/src/models/clip.cpp +++ b/src/models/clip.cpp @@ -14,5 +14,5 @@ void llama_model_clip::load_arch_tensors(llama_model_loader &) { [[noreturn]] std::unique_ptr llama_model_clip::build_arch_graph(const llm_graph_params &) const { - GGML_ABORT("CLIP has no inference graph via llama_model dispatch; runtime lives in tools/mtmd/clip.cpp"); + GGML_ABORT("CLIP has no inference graph via llama_model dispatch; runtime lives in tools/mtmd"); } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 016fca3c3..a17f6b43e 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -6561,7 +6561,7 @@ struct test_rope : public test_case { a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0); ggml_set_name(a, "view_of_a"); } else if (v == 2) { - // second-half slice along dim 0 (mimics build_rope_2d in clip.cpp). + // second-half slice along dim 0 (mimics build_rope_2d in tools/mtmd/mtmd-graph.cpp). // The non-zero view offset (ne_a[0] * elem_size) often produces a // non-aligned buffer offset, which exercises backends' alignment paths. auto ne = ne_a; ne[0] *= 2; diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp index e9c6ac856..a0201bdde 100644 --- a/tests/test-mtmd-impl.cpp +++ b/tests/test-mtmd-impl.cpp @@ -3,6 +3,7 @@ #include "ggml-backend.h" #include "ggml-cpp.h" #include "mtmd-audio.h" +#include "mtmd-subcontext.h" #include "mtmd-helper.h" #include "mtmd-image.h" #include "mtmd-internal.h" @@ -52,12 +53,12 @@ struct test_registry { // MAKE_TEST(test_image_preprocessor_longest_edge) { - clip_hparams hparams; + mtmd_hparams hparams; hparams.image_longest_edge = 1540; hparams.patch_size = 14; hparams.n_merge = 2; - const std::vector> cases = { + const std::vector> cases = { { { 463, 463 }, { 476, 476 } }, { { 463, 300 }, { 476, 308 } }, { { 448, 280 }, { 448, 280 } }, @@ -65,7 +66,7 @@ MAKE_TEST(test_image_preprocessor_longest_edge) { }; for (const auto & [input, expected] : cases) { - const clip_image_size actual = mtmd_image_preprocessor_longest_edge::get_target_size(hparams, input); + const mtmd_image_size actual = mtmd_image_preprocessor_longest_edge::get_target_size(hparams, input); const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height); t.assert_equal(name + " width", expected.width, actual.width); t.assert_equal(name + " height", expected.height, actual.height); @@ -75,7 +76,7 @@ MAKE_TEST(test_image_preprocessor_longest_edge) { MAKE_TEST(test_image_preprocessor_dyn_size) { // { patch_size, image size, expected size } // a side that is an even number plus .5 merged patches rounds down, like Python round() in smart_resize - const std::vector> cases = { + const std::vector> cases = { { 16, { 768, 400 }, { 768, 384 } }, { 16, { 800, 336 }, { 800, 320 } }, { 16, { 400, 400 }, { 384, 384 } }, @@ -84,13 +85,13 @@ MAKE_TEST(test_image_preprocessor_dyn_size) { }; for (const auto & [patch_size, input, expected] : cases) { - clip_hparams hparams; + mtmd_hparams hparams; hparams.patch_size = patch_size; hparams.n_merge = 2; hparams.image_round_half_even = true; hparams.set_limit_image_tokens(8, 4096); - const clip_image_size actual = mtmd_image_preprocessor_dyn_size::get_target_size(hparams, input); + const mtmd_image_size actual = mtmd_image_preprocessor_dyn_size::get_target_size(hparams, input); const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height); t.assert_equal(name + " width", expected.width, actual.width); t.assert_equal(name + " height", expected.height, actual.height); @@ -98,13 +99,13 @@ MAKE_TEST(test_image_preprocessor_dyn_size) { } MAKE_TEST(test_image_preprocessor_qwen3vl_video) { - clip_hparams hparams; + mtmd_hparams hparams; hparams.patch_size = 16; hparams.n_merge = 2; const mtmd_image_preprocessor_qwen3vl preproc(hparams); // { n_frames, frame size, expected size from Qwen3VLVideoProcessor smart_resize } - const std::vector> cases = { + const std::vector> cases = { { 20, { 720, 358 }, { 704, 352 } }, // 720 / 32 = 22.5 rounds half to even { 768, { 1920, 1080 }, { 224, 128 } }, // over the longest_edge budget { 4, { 16, 40 }, { 32, 64 } }, // edge below the factor is upscaled first @@ -114,7 +115,7 @@ MAKE_TEST(test_image_preprocessor_qwen3vl_video) { }; for (const auto & [n_frames, input, expected] : cases) { - clip_image_size actual; + mtmd_image_size actual; const std::string name = std::to_string(n_frames) + " frames of " + std::to_string(input.width) + "x" + std::to_string(input.height); if (!t.assert_true(name + " has a frame size", preproc.get_video_frame_size(n_frames, input.width, input.height, actual))) { continue; @@ -124,7 +125,7 @@ MAKE_TEST(test_image_preprocessor_qwen3vl_video) { } } -static std::string image_size_to_str(const clip_image_size & size) { +static std::string image_size_to_str(const mtmd_image_size & size) { return std::to_string(size.width) + "x" + std::to_string(size.height); } @@ -143,8 +144,8 @@ static std::string resize_algo_to_str(resize_algo algo) { } // pseudo-random pixels: s = s * 1103515245 + 12345 from s = 1, one byte (s >> 16) & 255 per channel -static clip_image_u8 make_test_image(const clip_image_size & size) { - clip_image_u8 img; +static mtmd_image_u8 make_test_image(const mtmd_image_size & size) { + mtmd_image_u8 img; img.set_size(size, false); uint32_t s = 1; auto next = [&s]() { @@ -160,7 +161,7 @@ static clip_image_u8 make_test_image(const clip_image_size & size) { } // FNV-1a of the pixels -static uint32_t hash_pixels(const clip_image_u8 & img, uint32_t hash = 2166136261u) { +static uint32_t hash_pixels(const mtmd_image_u8 & img, uint32_t hash = 2166136261u) { for (uint8_t v : img.get_ro_buf()) { hash = (hash ^ v) * 16777619u; } @@ -168,7 +169,7 @@ static uint32_t hash_pixels(const clip_image_u8 & img, uint32_t hash = 216613626 } MAKE_TEST(test_image_preprocessor_lfm2) { - clip_hparams hparams; + mtmd_hparams hparams; hparams.patch_size = 16; hparams.n_merge = 2; hparams.set_limit_image_tokens(64, 256); @@ -178,7 +179,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) { hparams.preproc_max_pixels_tolerance = 2.0f; // { image size, expected tiling } - const std::vector> cases = { + const std::vector> cases = { { { 704, 704 }, false }, // 720 / (patch_size * n_merge) is exactly 22.5, so this only matches HF // if round_by_factor rounds half to even (22) instead of away from zero (23) @@ -198,7 +199,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) { } // { image size, expected size from HF smart_resize } - const std::vector> resize_cases = { + const std::vector> resize_cases = { { { 400, 600 }, { 384, 608 } }, // 400 / 32 = 12.5 rounds half to even { { 480, 336 }, { 480, 320 } }, { { 64, 169 }, { 160, 416 } }, // 169 * beta / 32 is exactly 13 in double @@ -211,7 +212,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) { } // { image size, expected grid from HF } - const std::vector> grid_cases = { + const std::vector> grid_cases = { { { 720, 1600 }, { 2, 4 } }, // 0.45 is an exact tie between 2/4 and 2/5 in double, not in float { { 420, 1440 }, { 1, 3 } }, { { 255, 2160 }, { 1, 8 } }, @@ -231,7 +232,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) { } MAKE_TEST(test_image_preprocessor_lfm2_torch_resize) { - clip_hparams hparams; + mtmd_hparams hparams; hparams.patch_size = 16; hparams.n_merge = 2; hparams.set_limit_image_tokens(64, 256); @@ -243,7 +244,7 @@ MAKE_TEST(test_image_preprocessor_lfm2_torch_resize) { hparams.image_pad_rf = PAD_NONE; // { resize algo, image size, expected hash of the tiles and the thumbnail from Lfm2VlImageProcessor (torchvision 0.29, native uint8 resize: arm64, or x86 with AVX2/AVX512) } - const std::vector> cases = { + const std::vector> cases = { { RESIZE_ALGO_BILINEAR, { 620, 410 }, 0xf5e9c941 }, // 608x416 { RESIZE_ALGO_BILINEAR, { 2000, 1500 }, 0x3331f56c }, // 3x2 tiles, 576x416 thumbnail { RESIZE_ALGO_BICUBIC, { 620, 410 }, 0x2b34eac9 }, @@ -338,14 +339,14 @@ MAKE_TEST(test_temporal_merge_grouping) { // MAKE_TEST(test_video_config) { - clip_hparams hparams; + mtmd_hparams hparams; hparams.patch_size = 16; hparams.n_merge = 2; const mtmd_image_preprocessor_qwen3vl preproc(hparams); // { template, n_frames and fps of the source, expected fps, n_frames and frame size } // the 1920x1080 sizes are from Qwen3VLVideoProcessor smart_resize - const std::vector> cases = { + const std::vector> cases = { { MTMD_VIDEO_TEMPLATE_QWEN3VL, 36000, 30.0f, 2.0f, 768, { 224, 128 } }, { MTMD_VIDEO_TEMPLATE_LFM3, 36000, 30.0f, 2.0f, 768, { 224, 128 } }, { MTMD_VIDEO_TEMPLATE_LFM3_OMNI, 36000, 30.0f, 2.0f, 384, { 320, 192 } }, @@ -405,13 +406,13 @@ static std::string video_sequence_to_str(testing & t, mtmd_helper_video_seq & vi } MAKE_TEST(test_video_sequence_qwen3vl) { - clip_hparams hparams; + mtmd_hparams hparams; hparams.patch_size = 16; hparams.n_merge = 2; const mtmd_image_preprocessor_qwen3vl preproc(hparams); // { source video, expected frame size and sequence from Qwen3VLProcessor (transformers 4.57.1) } - const std::vector> cases = { + const std::vector> cases = { { { 720, 358, 24.0f, 240 }, { 704, 352 }, "<0.3 seconds>[0][13]<1.3 seconds>[25][38]<2.4 seconds>[50][63]<3.4 seconds>[75][88]<4.5 seconds>[101][113]" "<5.5 seconds>[126][138]<6.6 seconds>[151][164]<7.6 seconds>[176][189]<8.6 seconds>[201][214]<9.7 seconds>[226][239]" }, @@ -439,7 +440,7 @@ MAKE_TEST(test_video_sequence_qwen3vl) { // the soundtrack of each group comes after its frames MAKE_TEST(test_video_sequence_lfm3_omni) { - clip_hparams hparams; + mtmd_hparams hparams; hparams.patch_size = 16; hparams.n_merge = 2; const mtmd_image_preprocessor_lfm3vl preproc(hparams); @@ -462,7 +463,7 @@ MAKE_TEST(test_video_sequence_lfm3_omni) { // MAKE_TEST(test_audio_preprocessor_conformer) { - clip_hparams hparams; + mtmd_hparams hparams; hparams.n_mel_bins = 128; hparams.audio_sample_rate = 16000; hparams.audio_n_fft = 512; @@ -504,7 +505,7 @@ MAKE_TEST(test_audio_preprocessor_conformer) { if (!t.assert_true("preprocess", !out.entries.empty() && preproc.compute(out))) { return; } - const clip_image_f32 & mel = out.entries[0]; + const mtmd_image_f32 & mel = out.entries[0]; t.assert_equal("n_len", 101, mel.nx()); for (const auto & [bin, frame, expected] : cases) { @@ -515,7 +516,7 @@ MAKE_TEST(test_audio_preprocessor_conformer) { } MAKE_TEST(test_audio_preprocessor_threads) { - clip_hparams hparams; + mtmd_hparams hparams; hparams.n_mel_bins = 128; hparams.audio_sample_rate = 16000; hparams.audio_n_fft = 512; @@ -539,7 +540,7 @@ MAKE_TEST(test_audio_preprocessor_threads) { if (!t.assert_true("preprocess", ref.entries.size() == 1 && ref.needs_compute())) { return; } - const clip_image_size shape = ref.entries[0].get_size(); + const mtmd_image_size shape = ref.entries[0].get_size(); if (!t.assert_true("compute", preproc.compute(ref))) { return; } @@ -586,7 +587,7 @@ MAKE_TEST(test_audio_log_mel_precision) { } std::vector out((size_t) n_frames * n_mel); - if (!t.assert_true("log mel", clip_audio_log_mel(config, samples.data(), cache.hann_window.data(), n_frames, out.data()))) { + if (!t.assert_true("log mel", mtmd_audio_log_mel(config, samples.data(), cache.hann_window.data(), n_frames, out.data()))) { return; } @@ -650,8 +651,8 @@ MAKE_TEST(test_audio_log_mel_blocks) { std::vector all((size_t) n_frames * n_mel); std::vector sub((size_t) n_sub * n_mel); const float * window = cache.hann_window.data(); - if (!t.assert_true("all", clip_audio_log_mel(config, samples.data(), window, n_frames, all.data())) || - !t.assert_true("sub", clip_audio_log_mel(config, samples.data() + (size_t) off * hop, window, n_sub, sub.data()))) { + if (!t.assert_true("all", mtmd_audio_log_mel(config, samples.data(), window, n_frames, all.data())) || + !t.assert_true("sub", mtmd_audio_log_mel(config, samples.data() + (size_t) off * hop, window, n_sub, sub.data()))) { return; } diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index fd8b24efd..c19815d67 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -22,11 +22,13 @@ set(MTMD_CORE_SOURCES mtmd-helper-gen.cpp mtmd-helper-common.h mtmd-helper.h - clip.cpp - clip.h - clip-impl.h - clip-model.h - clip-graph.h + mtmd-model.cpp + mtmd-subcontext.cpp + mtmd-graph.cpp + mtmd-subcontext.h + mtmd-impl.h + mtmd-model.h + mtmd-graph.h ) add_library(mtmd diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md index 5dad86100..4ca410c67 100644 --- a/tools/mtmd/README-dev.md +++ b/tools/mtmd/README-dev.md @@ -7,6 +7,7 @@ Please refer to [multimodal.md](../../docs/multimodal.md) for a broader context. In short: - `libmtmd` started as a wrapper around `libllava` / `clip.cpp` - Various components that used to be in `clip.cpp` are moved progressively to mtmd. For example, preprocessor is now part of mtmd +- `clip.cpp` has since been split into `mtmd-model` (hparams and weights, one `mtmd_model` per modality), `mtmd-subcontext` (backends and compute, one `mtmd_subcontext` per modality) and `mtmd-graph` (graph builders, see `models/`). Only the GGUF format keeps the `clip.` name (metadata keys and the `clip` architecture), for compatibility with existing mmproj files ## Terminologies @@ -15,6 +16,26 @@ In short: - tiles / slices: for llava-uhd-style models, the preprocessor breaks a large input into smaller square images called tiles or slices - chunk: a mtmd_input_chunk represents a preprocessed input that can then be passed through `mtmd_encode()` +## Structure + +Overview of the `libmtmd` structure: + +- `mtmd_context` (`mtmd.cpp`): the public handle + - `mtmd_models`: one mmproj file, loaded by `mtmd_model_load()` + - `mtmd_model` `v` / `a` / `gen_a` (`mtmd-model.h`): one per modality, holds hparams, weights (`ctx_data` + `buf`) and requirements + - `mtmd_subcontext` `ctx_v` / `ctx_a` / `ctx_gen_a` (`mtmd-subcontext.h`): one per model, holds a const ref to it + - backends + `sched`: GPU/CPU backends, compute buffers, resolved flash attn + - `mtmd_graph` (`mtmd-graph.h`, `models/`): built for each `encode()` call, temporary + - `mtmd_image_preprocessor` / `mtmd_audio_preprocessor` (`mtmd-image.h`, `mtmd-audio.h`): read the hparams of their model +- `mtmd_gen_state` / `mtmd_audio_stream`: public, one per sequence / stream + - `mtmd_subctx_gen_state` / `mtmd_subctx_enc_state`: allocated on the backend of a subcontext, see `gen_state_init()` / `enc_state_init()` + +Rules of thumb: +- `mtmd_model` is read-only once loaded. Anything that depends on the backend (scheduler, compute buffers, flash attn resolution, warmup) belongs to `mtmd_subcontext` +- A model declares its runtime needs (e.g. `max_nodes`, `flash_attn_auto`) in `load_hparams`, the subcontext applies them +- Models must outlive their subcontexts, `mtmd_context` declares `models` first for that reason +- Backends are owned by each subcontext for now; they will move to `mtmd_context` so that subcontexts can share them + ## Pipeline A typical pipeline of the core libmtmd is as follows: @@ -67,21 +88,21 @@ Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system i - Backbone model should be converted to a normal text model (loadable via `libllama`) - If model used hard-coded embedding row ID, append them to token embeddings and assign token name for them (see `qwen3tts.py`) - If model have a specific output logits head for audio codes (usually semantic code), keep the head as-is and pad the logits at inference time (see `src/models/qwen3vl.cpp`) - - Sidecar models (code2wav, bigvgan, etc) must live inside the mmproj GGUF (but can be in different `clip_context` if necessary) + - Sidecar models (code2wav, bigvgan, etc) must live inside the mmproj GGUF (but can be in different `mtmd_subcontext` if necessary) - Note: it should use `ggml_build_forward_select` to select graphs if multiple graphs living in the same context - - Reuse existing GGUF metadata key name and tensor name whenever possible; think twice before adding extensive changes to GGUF writer. For example, Qwen3-TTS hard-code part of the hparams to `clip.cpp` as they won't likely to change. + - Reuse existing GGUF metadata key name and tensor name whenever possible; think twice before adding extensive changes to GGUF writer. For example, Qwen3-TTS hard-code part of the hparams in `mtmd-model.cpp` as they won't likely to change. - For tensor naming: - Prefixed with `a.*` for tensors used by speaker encoder pipeline - Prefixed with `a.gen.*` for generation stages (code / mel-spectrogram / PCM generation) - For GGUF metadata: - Reuse as many existing keys as possible - - In most cases, you can hard-code model configs in the model graph class, or in `clip_hparams` + - In most cases, you can hard-code model configs in the model graph class, or in `mtmd_hparams` - If some values need to be exposed to the `mtmd_helper` layer, hard-code them in `mtmd_helper` and distinguish by pipeline and `mtmd_gen_audio_info::model_variant` if necessary - Do NOT add new GGUF metadata or new fields to `mtmd_gen_audio_info` unless you can prove that you absolutely need them 4. Make sure most of the changes happen inside `mtmd-helper-gen.cpp`. A good PR looks like this: - 10-20% changes is to add new backbone (text) model and conversion - 60% changes inside `mtmd-helper-gen.cpp` - - 10% changes inside `libmtmd` and `clip.cpp` systems + - 10% changes inside `libmtmd` core (`mtmd-model`, `mtmd-subcontext`, `models/`) - The rest downstream code (CLI, server) should have no changes at all 5. Update usage documentation in `tools/tts/README.md` diff --git a/tools/mtmd/README.md b/tools/mtmd/README.md index 701941947..f0290aae5 100644 --- a/tools/mtmd/README.md +++ b/tools/mtmd/README.md @@ -32,7 +32,7 @@ Consequently, running a multimodal model typically requires two GGUF files: As outlined in the history, `libmtmd` is the modern library designed to replace the original `llava.cpp` implementation for handling multimodal inputs. -Built upon `clip.cpp` (similar to `llava.cpp`), `libmtmd` offers several advantages: +Originally built upon `clip.cpp` (similar to `llava.cpp`), `libmtmd` offers several advantages: - **Unified Interface:** Aims to consolidate interaction for various multimodal models. - **Improved UX/DX:** Features a more intuitive API, inspired by the `Processor` class in the Hugging Face `transformers` library. - **Flexibility:** Designed to support multiple input types (text, audio, images) while respecting the wide variety of chat templates used by different models. diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h deleted file mode 100644 index f8fd41ef5..000000000 --- a/tools/mtmd/clip.h +++ /dev/null @@ -1,200 +0,0 @@ -#pragma once - -#include "ggml.h" -#include "mtmd.h" - -#include -#include - -#include - -// !!! Internal header, to be used by mtmd only !!! - -#define MTMD_INTERNAL_HEADER - -struct clip_ctx; -struct clip_enc_state; -struct clip_gen_state; - -struct clip_image_size { - int width; - int height; - bool operator==(const clip_image_size & other) const { - return width == other.width && height == other.height; - } - bool operator!=(const clip_image_size & other) const { - return !(*this == other); - } - int area() const { - // avoid overflow when computing area - GGML_ASSERT(width >= 0 && width <= 46000); - GGML_ASSERT(height >= 0 && height <= 46000); - return width * height; - } -}; - -struct clip_image_f32; -struct clip_image_f32_batch; - -enum clip_modality { - CLIP_MODALITY_VISION, - CLIP_MODALITY_AUDIO, - CLIP_MODALITY_GEN_AUDIO, -}; - -enum clip_flash_attn_type { - CLIP_FLASH_ATTN_TYPE_AUTO = -1, - CLIP_FLASH_ATTN_TYPE_DISABLED = 0, - CLIP_FLASH_ATTN_TYPE_ENABLED = 1, -}; - -struct clip_context_params { - bool use_gpu; - ggml_backend_dev_t device; - enum clip_flash_attn_type flash_attn_type; - int image_min_tokens; - int image_max_tokens; - bool warmup; - ggml_backend_sched_eval_callback cb_eval; - void * cb_eval_user_data; - bool no_alloc; - mtmd_progress_callback progress_callback; - void * progress_callback_user_data; - int n_threads; -}; - -struct clip_init_result { - struct clip_ctx * ctx_v; // vision context - struct clip_ctx * ctx_a; // audio context - struct clip_ctx * ctx_gen_a; // audio generation context -}; - -struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params); - -// max number of output tokens per image, -1 if not dynamic size -int clip_get_image_max_tokens(const struct clip_ctx * ctx); - -void clip_free(struct clip_ctx * ctx); - -// TODO: should be enum, not string -const char * clip_patch_merge_type(const struct clip_ctx * ctx); - -int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img); - -// for M-RoPE, this will be the number of token positions in X and Y directions -// for other models, X will be the total number of tokens and Y will be 1 -int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img); -int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img); - -// this should be equal to the embedding dimension of the text model -int clip_n_mmproj_embd(const struct clip_ctx * ctx); - -ggml_backend_sched_t clip_get_sched(const struct clip_ctx * ctx); - -// compute the log-mel of n_frames frames, the waveform must already be padded and window must hold one whole frame -// out has the layout [n_frames, n_mel]; like the encoders, must not run at the same time as another pass on config.sched -struct mtmd_audio_mel_config; -bool clip_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out); - -// TODO: remove clip_image_encode() and always use batched version -bool clip_image_encode (struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector & out_vec); -bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, std::vector & out_batch_embd); - -enum clip_gen_process_type { - CLIP_GEN_PROCESS_GEN_UNKNOWN, - CLIP_GEN_PROCESS_GEN_CODE, // h_state to codes - CLIP_GEN_PROCESS_GEN_WAV, // codes to raw PCM audio - CLIP_GEN_PROCESS_TTS, // full utterance of codes to raw PCM audio - CLIP_GEN_PROCESS_TTS_VOCODE, // internal: mel + source stft to istft input - CLIP_GEN_PROCESS_TOKENIZE, // raw PCM audio to semantic speech tokens -}; - -// gen-side voice reference (chatterbox): speech tokens, mel-rate features and the speaker vector -struct clip_gen_ref { - std::vector tokens; - std::vector feat; - std::vector spk; -}; - -struct clip_encode_params { - int n_threads = 1; - const clip_image_f32_batch * imgs = nullptr; - std::vector * out_embd = nullptr; - - // for audio gen, imgs has exactly one entry: hidden state from backbone (GEN_CODE) or unused (GEN_WAV) - clip_gen_process_type gen_process = CLIP_GEN_PROCESS_GEN_UNKNOWN; - - // GEN_CODE: out_embd receives the embd to feed back to the backbone - int32_t code0 = 0; // semantic code sampled by the backbone - int32_t top_k = 50; - float top_p = 1.0f; - std::vector * out_codes = nullptr; // this frame's 16 sampled codes - std::vector * out_feats = nullptr; // continuous counterpart of out_codes - float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders - bool * out_is_eos = nullptr; - - // TOKENIZE: speech embedding rows of the produced codes - std::vector * out_code_embd = nullptr; - - // GEN_WAV - const std::vector * codes = nullptr; // this frame's 16 RVQ codes - const std::vector * feats = nullptr; // continuous counterpart of codes - std::vector * out_audio = nullptr; // decoded PCM samples, F32 - const clip_gen_ref * ref = nullptr; // TTS reference conditioning of the voice - // TTS_VOCODE internal stage inputs - const std::vector * mel_in = nullptr; - const std::vector * sstft_in = nullptr; - int vocode_n_mel = 0; - int vocode_n_stft = 0; - std::vector * out_spec = nullptr; - - // audio gen: state of one sequence, clip_encode() updates it in place - struct clip_gen_state * gen_state = nullptr; - - // streaming audio input, updated in place by clip_encode() - struct clip_enc_state * enc_state = nullptr; -}; -bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params); - -// persistent state of a causal audio encoder, for streaming input (one per stream) -// it holds the sliding window of past keys/values, so each clip_encode() call only takes the new frames -// n_frames_max is the maximum number of encoder frames per call -struct clip_enc_state * clip_enc_state_init(struct clip_ctx * ctx, int32_t n_frames_max); -void clip_enc_state_free(struct clip_enc_state * state); - -// per-sequence state of audio generation: RNG and the slots kept between GEN_WAV calls -// seed UINT32_MAX means random; reset() clears the slots and keeps the RNG stream -struct clip_gen_state * clip_gen_state_init(struct clip_ctx * ctx, uint32_t seed); -void clip_gen_state_free(struct clip_gen_state * state); -void clip_gen_state_reset(struct clip_gen_state * state); - -// read a chatterbox tensor by name as F32, returns the element count (0 if not found), out may be null to query the size -size_t clip_cbx_read_tensor(struct clip_ctx * ctx, const char * name, float * out, size_t n_max); - -// a predefined voice: backbone rows, plus the gen-side reference when the gen stack needs one -struct clip_gen_voice { - std::vector rows; - clip_gen_ref ref; -}; - -const std::map & clip_gen_get_voices(const struct clip_ctx * ctx); - -bool clip_is_llava(const struct clip_ctx * ctx); -// note for contributor: this clip_is_(model) pattern is deprecated -// do NOT add new functions like this - -bool clip_has_vision_encoder(const struct clip_ctx * ctx); -bool clip_has_audio_encoder(const struct clip_ctx * ctx); - -bool clip_support_batch(const struct clip_ctx * ctx); - -int clip_model_n_temporal_merge(const struct clip_ctx * ctx); // TODO @ngxson : remove, refactor this - -std::map clip_get_mem_usage(const struct clip_ctx * ctx); - -struct clip_cap { - bool has_vision; - bool has_audio; - bool has_gen_audio; -}; -struct clip_cap clip_get_cap(const char * fname); diff --git a/tools/mtmd/debug/mtmd-debug.cpp b/tools/mtmd/debug/mtmd-debug.cpp index 2719dae9b..894ad7d6f 100644 --- a/tools/mtmd/debug/mtmd-debug.cpp +++ b/tools/mtmd/debug/mtmd-debug.cpp @@ -81,7 +81,7 @@ int main(int argc, char ** argv) { llama_init = common_init_from_params(params); { auto * model = llama_init->model(); - const char * clip_path = params.mmproj.path.c_str(); + const char * mmproj_path = params.mmproj.path.c_str(); mtmd_context_params mparams = mtmd_context_params_default(); mparams.use_gpu = params.mmproj_use_gpu; mparams.device = params.mmproj_device; @@ -96,9 +96,9 @@ int main(int argc, char ** argv) { mparams.cb_eval_user_data = &cb_data; mparams.cb_eval = common_debug_cb_eval; } - ctx_mtmd.reset(mtmd_init_from_file(clip_path, model, mparams)); + ctx_mtmd.reset(mtmd_init_from_file(mmproj_path, model, mparams)); if (!ctx_mtmd.get()) { - LOG_ERR("Failed to load vision model from %s\n", clip_path); + LOG_ERR("Failed to load vision model from %s\n", mmproj_path); exit(1); } } diff --git a/tools/mtmd/models/chatterbox-gen.cpp b/tools/mtmd/models/chatterbox-gen.cpp index 00a9a8d81..017329bf4 100644 --- a/tools/mtmd/models/chatterbox-gen.cpp +++ b/tools/mtmd/models/chatterbox-gen.cpp @@ -3,7 +3,7 @@ // Chatterbox generation graphs: flow encoder, cfm estimator, s3 tokenizer and hift vocoder // x [C, T]: y = W x + b with torch Linear weights stored as [in, out] -ggml_tensor * clip_graph_chatterbox_base::cbx_linear(ggml_tensor * w, ggml_tensor * b, ggml_tensor * x) const { +ggml_tensor * mtmd_graph_chatterbox_base::cbx_linear(ggml_tensor * w, ggml_tensor * b, ggml_tensor * x) const { ggml_tensor * y = ggml_mul_mat(ctx0, w, x); if (b) { y = ggml_add(ctx0, y, b); @@ -12,7 +12,7 @@ ggml_tensor * clip_graph_chatterbox_base::cbx_linear(ggml_tensor * w, ggml_tenso } // conv1d over time with asymmetric zero padding: x [C, T] -> [OC, T_out], kernel [K, IC, OC] -ggml_tensor * clip_graph_chatterbox_base::cbx_conv1d(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, +ggml_tensor * mtmd_graph_chatterbox_base::cbx_conv1d(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, int stride, int pad_l, int pad_r) const { ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, C] if (pad_l > 0) { @@ -34,7 +34,7 @@ ggml_tensor * clip_graph_chatterbox_base::cbx_conv1d(ggml_tensor * k, ggml_tenso } // x [C, T] -> symmetric-padded dilated conv -> [OC, T] -ggml_tensor * clip_graph_chatterbox_base::cbx_conv1d_dil(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, +ggml_tensor * mtmd_graph_chatterbox_base::cbx_conv1d_dil(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, int pad, int dil) const { ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); ggml_tensor * y = ggml_conv_1d(ctx0, k, xt, 1, pad, dil); @@ -61,7 +61,7 @@ static ggml_tensor * cbx_rel_shift(ggml_context * ctx0, ggml_tensor * bd, int T) } // espnet rel-pos self attention block, pre-norm, x [512, T], pos [512, 2T-1] -ggml_tensor * clip_graph_chatterbox::enc_layer(const clip_chatterbox::enc_layer & l, ggml_tensor * x, +ggml_tensor * mtmd_graph_chatterbox::enc_layer(const mtmd_chatterbox::enc_layer & l, ggml_tensor * x, ggml_tensor * pos, int T) { const int n_head = 8; const int d_head = 64; @@ -121,14 +121,14 @@ static ggml_tensor * cbx_mish(ggml_context * ctx0, ggml_tensor * x) { } // causal block: conv k3 left-padded, layer norm over channels, mish; x [C, T] -ggml_tensor * clip_graph_chatterbox::causal_block(const clip_chatterbox::causal_block & b, ggml_tensor * x) { +ggml_tensor * mtmd_graph_chatterbox::causal_block(const mtmd_chatterbox::causal_block & b, ggml_tensor * x) { x = cbx_conv1d(b.conv_w, b.conv_b, x, 1, (int) b.conv_w->ne[0] - 1, 0); x = build_norm(x, b.norm_w, b.norm_b, NORM_TYPE_NORMAL, 1e-5f, -1); return cbx_mish(ctx0, x); } // resnet block with time conditioning; x [C, T], temb [1024] -ggml_tensor * clip_graph_chatterbox::resnet(const clip_chatterbox::resnet & r, ggml_tensor * x, ggml_tensor * temb) { +ggml_tensor * mtmd_graph_chatterbox::resnet(const mtmd_chatterbox::resnet & r, ggml_tensor * x, ggml_tensor * temb) { ggml_tensor * h = causal_block(r.block1, x); ggml_tensor * tproj = cbx_linear(r.mlp_w, r.mlp_b, cbx_mish(ctx0, temb)); h = ggml_add(ctx0, h, tproj); // broadcast [256, 1] over T @@ -138,7 +138,7 @@ ggml_tensor * clip_graph_chatterbox::resnet(const clip_chatterbox::resnet & r, g } // diffusers-style transformer block, full attention; x [256, T] -ggml_tensor * clip_graph_chatterbox::tfm_block(const clip_chatterbox::tfm_block & b, ggml_tensor * x) { +ggml_tensor * mtmd_graph_chatterbox::tfm_block(const mtmd_chatterbox::tfm_block & b, ggml_tensor * x) { const int n_head = 8; const int d_head = 64; const int T = (int) x->ne[1]; @@ -157,7 +157,7 @@ ggml_tensor * clip_graph_chatterbox::tfm_block(const clip_chatterbox::tfm_block } // one estimator evaluation; x_noise, mu, cond [80, T], spks [80], temb [1024] -ggml_tensor * clip_graph_chatterbox::estimator(ggml_tensor * x_noise, ggml_tensor * mu, ggml_tensor * spks, +ggml_tensor * mtmd_graph_chatterbox::estimator(ggml_tensor * x_noise, ggml_tensor * mu, ggml_tensor * spks, ggml_tensor * cond, ggml_tensor * temb, int T) { const auto & c = model.cbx; @@ -199,7 +199,7 @@ ggml_tensor * clip_graph_chatterbox::estimator(ggml_tensor * x_noise, ggml_tenso } // s3 tokenizer encoder: log-mel [T, 128] -> fsq code [8, T / 4], rounded to base 3 tokens on the host -ggml_cgraph * clip_graph_chatterbox::build_s3tok(int T) { +ggml_cgraph * mtmd_graph_chatterbox::build_s3tok(int T) { const auto & c = model.cbx; const int n_head = 20; const int d_head = 64; @@ -261,14 +261,14 @@ ggml_cgraph * clip_graph_chatterbox::build_s3tok(int T) { return gf; } -ggml_cgraph * clip_graph_chatterbox::build() { - if (gen_process == CLIP_GEN_PROCESS_TTS_VOCODE) { +ggml_cgraph * mtmd_graph_chatterbox::build() { + if (gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE) { return build_vocoder(vocode_n_mel, vocode_n_stft); } - if (gen_process == CLIP_GEN_PROCESS_TOKENIZE) { + if (gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) { return build_s3tok(img.nx()); } - if (gen_process != CLIP_GEN_PROCESS_TTS) { + if (gen_process != MTMD_SUBCTX_GEN_PROCESS_TTS) { // load-time buffer sizing path ggml_tensor * inp = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1); ggml_set_name(inp, "inp_stub"); @@ -355,7 +355,7 @@ ggml_cgraph * clip_graph_chatterbox::build() { const bool meanflow = c.time_embed_mixer_w != nullptr; const int n_steps = meanflow ? 2 : 10; - // span points, same schedule as the host side sinusoid fill in clip.cpp + // span points, same schedule as the host side sinusoid fill in mtmd-subcontext.cpp float span[11]; for (int i = 0; i <= n_steps; i++) { const float u = (float) i / n_steps; @@ -454,7 +454,7 @@ static ggml_tensor * cbx_snake(ggml_context * ctx0, ggml_tensor * x, ggml_tensor } // hifigan-snake resblock; kernels with dilations 1/3/5 on convs1, 1 on convs2 -ggml_tensor * clip_graph_chatterbox::hift_resblock(const clip_chatterbox::hift_res & r, ggml_tensor * x) { +ggml_tensor * mtmd_graph_chatterbox::hift_resblock(const mtmd_chatterbox::hift_res & r, ggml_tensor * x) { static const int dil[3] = {1, 3, 5}; for (size_t j = 0; j < r.units.size(); j++) { const auto & u = r.units[j]; @@ -471,7 +471,7 @@ ggml_tensor * clip_graph_chatterbox::hift_resblock(const clip_chatterbox::hift_r } // mel [80, T] + source stft [18, T_stft] -> conv_post output [18, T_stft2] -ggml_cgraph * clip_graph_chatterbox::build_vocoder(int n_mel, int n_stft) { +ggml_cgraph * mtmd_graph_chatterbox::build_vocoder(int n_mel, int n_stft) { const auto & c = model.cbx; ggml_tensor * mel = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, n_mel); diff --git a/tools/mtmd/models/chatterbox-spkenc.cpp b/tools/mtmd/models/chatterbox-spkenc.cpp index afd9f0c29..fc73aa805 100644 --- a/tools/mtmd/models/chatterbox-spkenc.cpp +++ b/tools/mtmd/models/chatterbox-spkenc.cpp @@ -5,7 +5,7 @@ static constexpr float bn_eps = 1e-5f; // per-channel batchnorm on x [C, T], w/b are null on the affine=False variant -ggml_tensor * clip_graph_chatterbox_spkenc::bn1d(const clip_chatterbox::bn & n, ggml_tensor * x) { +ggml_tensor * mtmd_graph_chatterbox_spkenc::bn1d(const mtmd_chatterbox::bn & n, ggml_tensor * x) { ggml_tensor * sd = ggml_sqrt(ctx0, ggml_scale_bias(ctx0, n.var, 1.0f, bn_eps)); if (!n.w) { return ggml_div(ctx0, ggml_sub(ctx0, x, n.mean), sd); @@ -16,7 +16,7 @@ ggml_tensor * clip_graph_chatterbox_spkenc::bn1d(const clip_chatterbox::bn & n, } // batchnorm on a conv2d activation [W=T, H=F, C, 1], stats on ne2 -static ggml_tensor * cbx_bn2d(ggml_context * ctx0, const clip_chatterbox::bn & n, ggml_tensor * x) { +static ggml_tensor * cbx_bn2d(ggml_context * ctx0, const mtmd_chatterbox::bn & n, ggml_tensor * x) { const int C = (int) x->ne[2]; ggml_tensor * mean = ggml_reshape_4d(ctx0, n.mean, 1, 1, C, 1); ggml_tensor * var = ggml_reshape_4d(ctx0, n.var, 1, 1, C, 1); @@ -27,12 +27,12 @@ static ggml_tensor * cbx_bn2d(ggml_context * ctx0, const clip_chatterbox::bn & n return ggml_add(ctx0, ggml_mul(ctx0, x, a), shift); } -ggml_tensor * clip_graph_chatterbox_spkenc::bn2d_relu(const clip_chatterbox::bn & n, ggml_tensor * x) { +ggml_tensor * mtmd_graph_chatterbox_spkenc::bn2d_relu(const mtmd_chatterbox::bn & n, ggml_tensor * x) { return ggml_relu(ctx0, cbx_bn2d(ctx0, n, x)); } // fcm residual 2d block, stride on the frequency axis only -ggml_tensor * clip_graph_chatterbox_spkenc::res2d(const clip_chatterbox::spk_res2d & r, ggml_tensor * x, +ggml_tensor * mtmd_graph_chatterbox_spkenc::res2d(const mtmd_chatterbox::spk_res2d & r, ggml_tensor * x, int stride) { ggml_tensor * cur = ggml_conv_2d(ctx0, r.conv1_w, x, 1, stride, 1, 1, 1, 1); cur = bn2d_relu(r.bn1, cur); @@ -48,7 +48,7 @@ ggml_tensor * clip_graph_chatterbox_spkenc::res2d(const clip_chatterbox::spk_res } // cam dense tdnn layer: bottleneck then context-gated conv; x [C_in, T] -> [growth, T] -ggml_tensor * clip_graph_chatterbox_spkenc::cam_layer(const clip_chatterbox::spk_cam_layer & l, ggml_tensor * x, +ggml_tensor * mtmd_graph_chatterbox_spkenc::cam_layer(const mtmd_chatterbox::spk_cam_layer & l, ggml_tensor * x, int dil, ggml_tensor * segfix) { ggml_tensor * h = ggml_relu(ctx0, bn1d(l.nl1_bn, x)); h = cbx_conv1d(l.linear1_w, nullptr, h, 1, 0, 0); @@ -86,7 +86,7 @@ ggml_tensor * clip_graph_chatterbox_spkenc::cam_layer(const clip_chatterbox::spk return ggml_mul(ctx0, y, m); } -ggml_cgraph * clip_graph_chatterbox_spkenc::build() { +ggml_cgraph * mtmd_graph_chatterbox_spkenc::build() { const auto & c = model.cbx; const int T = img.nx(); @@ -119,7 +119,7 @@ ggml_cgraph * clip_graph_chatterbox_spkenc::build() { cb(x, "spk_tdnn", -1); static const int block_dil[3] = {1, 2, 2}; - const clip_chatterbox::spk_cam_block * blocks[3] = { &c.spk_block1, &c.spk_block2, &c.spk_block3 }; + const mtmd_chatterbox::spk_cam_block * blocks[3] = { &c.spk_block1, &c.spk_block2, &c.spk_block3 }; for (int bi = 1; bi <= 3; bi++) { const auto & blk = *blocks[bi - 1]; for (const auto & l : blk.layers) { diff --git a/tools/mtmd/models/cogvlm.cpp b/tools/mtmd/models/cogvlm.cpp index 44bc88442..aeaa33e4f 100644 --- a/tools/mtmd/models/cogvlm.cpp +++ b/tools/mtmd/models/cogvlm.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_cogvlm::build() { +ggml_cgraph * mtmd_graph_cogvlm::build() { GGML_ASSERT(model.class_embedding != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); diff --git a/tools/mtmd/models/conformer.cpp b/tools/mtmd/models/conformer.cpp index 9463f9f1c..4cd2bee53 100644 --- a/tools/mtmd/models/conformer.cpp +++ b/tools/mtmd/models/conformer.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_conformer::build() { +ggml_cgraph * mtmd_graph_conformer::build() { const int n_frames = img.nx(); const int n_pos = n_frames / 2; const int n_pos_embd = (((((n_frames + 1) / 2) + 1) / 2 + 1) / 2) * 2 - 1; diff --git a/tools/mtmd/models/deepseek4v.cpp b/tools/mtmd/models/deepseek4v.cpp index 06005d52c..e3c0d95ed 100644 --- a/tools/mtmd/models/deepseek4v.cpp +++ b/tools/mtmd/models/deepseek4v.cpp @@ -11,14 +11,14 @@ // // each aligner row ends with a NEWLINE, an odd row count is padded with a full row of PADs // pairs of adjacent rows are interleaved column-wise ("N-layout") -// the mapping is precomputed on CPU as the "layout_idx" input (see set_input in clip.cpp) +// the mapping is precomputed on CPU as the "layout_idx" input (see set_input in mtmd-subcontext.cpp) // // ref: inference/vision.py and inference/image_processor.py in the HF repo // // DeepSeek-V4.1-Flash (deepseek41v) shares the encoder and aligner; its block is plain reading order // [START] ([IMAGE]*n_llm_w [NEWLINE])*n_llm_h [END], no lead pads, no pad row, no IMAGE_PAD tensor -ggml_cgraph * clip_graph_deepseek4v::build() { +ggml_cgraph * mtmd_graph_deepseek4v::build() { const int n_merge = hparams.n_merge; // 2D input positions @@ -27,7 +27,7 @@ ggml_cgraph * clip_graph_deepseek4v::build() { ggml_set_input(positions); int sections[4] = {d_head/4, d_head/4, 0, 0}; - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return ggml_rope_multi(ctx0, cur, positions, nullptr, d_head/2, sections, GGML_ROPE_TYPE_VISION, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); @@ -92,8 +92,8 @@ ggml_cgraph * clip_graph_deepseek4v::build() { } } - const int n_llm_w = CLIP_ALIGN(n_patches_x, n_merge) / n_merge; - const int n_llm_h = CLIP_ALIGN(n_patches_y, n_merge) / n_merge; + const int n_llm_w = MTMD_ALIGN(n_patches_x, n_merge) / n_merge; + const int n_llm_h = MTMD_ALIGN(n_patches_y, n_merge) / n_merge; const int n_out = is_v41 ? dsv41_n_block_tokens(n_llm_w, n_llm_h) : dsv4_get_block_layout(n_llm_w, n_llm_h, img.lead_pad).n_out; GGML_ASSERT(n_grid == n_llm_w * n_llm_h); diff --git a/tools/mtmd/models/deepseekocr.cpp b/tools/mtmd/models/deepseekocr.cpp index b784cdad6..af69c0332 100644 --- a/tools/mtmd/models/deepseekocr.cpp +++ b/tools/mtmd/models/deepseekocr.cpp @@ -105,7 +105,7 @@ static ggml_tensor * conv_2d_f32(ggml_context * ctx0, ggml_tensor * a, ggml_tens } -ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) { +ggml_tensor * mtmd_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) { // Building SAM const int n_embd = hparams.sam_n_embd; const int n_layer = hparams.sam_n_layer; @@ -211,7 +211,7 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) { mask = ggml_add(ctx0, rw, rh); // [B*n_heads, H*W, H, W] mask = ggml_reshape_4d(ctx0, mask, W * H, W * H, n_heads, B); // casting mask to F16 only required when flash-attn is enabled - if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { + if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) { mask = ggml_cast(ctx0, mask, GGML_TYPE_F16); } @@ -264,7 +264,7 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) { return cur; } -ggml_cgraph * clip_graph_deepseekocr::build() { +ggml_cgraph * mtmd_graph_deepseekocr::build() { // patch embedding ggml_tensor * inp_raw = build_inp_raw(); diff --git a/tools/mtmd/models/deepseekocr2.cpp b/tools/mtmd/models/deepseekocr2.cpp index 7088bb97b..fb20e9d1d 100644 --- a/tools/mtmd/models/deepseekocr2.cpp +++ b/tools/mtmd/models/deepseekocr2.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_deepseekocr2::build() { +ggml_cgraph * mtmd_graph_deepseekocr2::build() { GGML_ASSERT(hparams.n_head_kv > 0); GGML_ASSERT(n_head % hparams.n_head_kv == 0); @@ -45,7 +45,7 @@ ggml_cgraph * clip_graph_deepseekocr2::build() { ggml_tensor * inp_pos = ggml_cast(ctx0, ggml_arange(ctx0, 0, seq_len, 1), GGML_TYPE_I32); - auto add_rope = [&](ggml_tensor * x, const clip_layer &) { + auto add_rope = [&](ggml_tensor * x, const mtmd_layer &) { return ggml_rope_ext(ctx0, x, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 131072, 1000000, 1, 0, 1, 0, 0); }; diff --git a/tools/mtmd/models/dots3note.cpp b/tools/mtmd/models/dots3note.cpp index 93c14fc79..c942ea7da 100644 --- a/tools/mtmd/models/dots3note.cpp +++ b/tools/mtmd/models/dots3note.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_dots3note_a::build() { +ggml_cgraph * mtmd_graph_dots3note_a::build() { // inp_raw: [n_frames, n_mel, 1], one 60s chunk, mel frames not padded // the reference impl zero-masks conv inputs beyond the valid length at each stage; // running on exactly the valid frames with the convs' zero padding is equivalent @@ -37,7 +37,7 @@ ggml_cgraph * clip_graph_dots3note_a::build() { ggml_set_input(positions); // partial rotary: first half of each head, NEOX style - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return ggml_rope_ext(ctx0, cur, positions, nullptr, d_head/2, GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); }; diff --git a/tools/mtmd/models/dotsocr.cpp b/tools/mtmd/models/dotsocr.cpp index 92974bb67..dcede11c3 100644 --- a/tools/mtmd/models/dotsocr.cpp +++ b/tools/mtmd/models/dotsocr.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_dotsocr::build() { +ggml_cgraph * mtmd_graph_dotsocr::build() { const int n_pos = n_patches; const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position @@ -11,7 +11,7 @@ ggml_cgraph * clip_graph_dotsocr::build() { ggml_set_name(positions, "positions"); ggml_set_input(positions); - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return ggml_rope_multi( ctx0, cur, positions, nullptr, d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, diff --git a/tools/mtmd/models/drama3-gen.cpp b/tools/mtmd/models/drama3-gen.cpp index de7a4f9e8..e77fcf540 100644 --- a/tools/mtmd/models/drama3-gen.cpp +++ b/tools/mtmd/models/drama3-gen.cpp @@ -7,13 +7,13 @@ static constexpr int DRAMA3_N_ROT = 64; static constexpr float DRAMA3_ROPE_THETA = 1e7f; -std::vector list_drama3_state_slots(const clip_hparams & hparams, const clip_model & model) { +std::vector list_drama3_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { std::vector slots; slots.push_back({"tower_pos", 1, 1, true}); for (size_t il = 0; il < model.layers.size(); il++) { const int64_t n_kv = model.layers[il].k_w->ne[1]; - slots.push_back({"tower_k_" + std::to_string(il), n_kv, clip_graph_drama3_gen::n_pos_max, true}); - slots.push_back({"tower_v_" + std::to_string(il), n_kv, clip_graph_drama3_gen::n_pos_max, true}); + slots.push_back({"tower_k_" + std::to_string(il), n_kv, mtmd_graph_drama3_gen::n_pos_max, true}); + slots.push_back({"tower_v_" + std::to_string(il), n_kv, mtmd_graph_drama3_gen::n_pos_max, true}); } for (auto & slot : list_c2w_state_slots(hparams, model)) { slots.push_back(slot); @@ -22,7 +22,7 @@ std::vector list_drama3_state_slots(const clip_hparams & hparams } // h_out = out_norm(Block(eh_proj([enorm(mask * E[code] + extra); hnorm(h_in)]))) -ggml_tensor * clip_graph_drama3_gen::tower_layer(ggml_tensor * x, int il, ggml_tensor * code, ggml_tensor * mask, ggml_tensor * extra, +ggml_tensor * mtmd_graph_drama3_gen::tower_layer(ggml_tensor * x, int il, ggml_tensor * code, ggml_tensor * mask, ggml_tensor * extra, ggml_tensor * inp_pos, ggml_tensor * k_slot, ggml_tensor * v_slot, int pos) { const auto & layer = model.layers[il]; const auto & tl = model.drama3_layers[il]; @@ -80,7 +80,7 @@ ggml_tensor * clip_graph_drama3_gen::tower_layer(ggml_tensor * x, int il, ggml_t } // one row: backbone hidden + next main code -> other codes of that frame + next backbone input -ggml_cgraph * clip_graph_drama3_gen::build_tower() { +ggml_cgraph * mtmd_graph_drama3_gen::build_tower() { const int n_layer = (int) model.layers.size(); const int n_heads = (int) model.gen_code_head_w->ne[2]; // one per residual codebook const int n_shared = n_layer - n_heads; @@ -141,7 +141,7 @@ ggml_cgraph * clip_graph_drama3_gen::build_tower() { ggml_tensor * extra = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_get_rows(ctx0, cb_all, inp_extra))); extra = ggml_mul(ctx0, ggml_reshape_2d(ctx0, ggml_sum_rows(ctx0, extra), n_embd_cb, 1), extra_on); - clip_graph_qwen3tts_gen::code_gen cg(*this, top_k, top_p); + mtmd_graph_qwen3tts_gen::code_gen cg(*this, top_k, top_p); ggml_tensor * out_codes = cg.cache_set(ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, 1, n_cb), 0, inp_code0); ggml_tensor * code = inp_code0; @@ -183,7 +183,7 @@ ggml_cgraph * clip_graph_drama3_gen::build_tower() { } // frames -> PCM, qwen3tts code2wav blocks without pre_conv and tfm projections -ggml_cgraph * clip_graph_drama3_gen::build_codec() { +ggml_cgraph * mtmd_graph_drama3_gen::build_codec() { const auto & c2w = model.c2w; const int64_t n_cb = model.gen_code_embd_w->ne[2]; @@ -191,7 +191,7 @@ ggml_cgraph * clip_graph_drama3_gen::build_codec() { ggml_set_name(inp_codes, "inp_codes"); ggml_set_input(inp_codes); - clip_graph_qwen3tts_gen::code2wav dec(*this); + mtmd_graph_qwen3tts_gen::code2wav dec(*this); dec.gelu_erf = true; for (const auto & slot : list_c2w_state_slots(hparams, model)) { ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1); @@ -251,6 +251,6 @@ ggml_cgraph * clip_graph_drama3_gen::build_codec() { return gf; } -ggml_cgraph * clip_graph_drama3_gen::build() { - return gen_process == CLIP_GEN_PROCESS_GEN_WAV ? build_codec() : build_tower(); +ggml_cgraph * mtmd_graph_drama3_gen::build() { + return gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV ? build_codec() : build_tower(); } diff --git a/tools/mtmd/models/exaone4_5.cpp b/tools/mtmd/models/exaone4_5.cpp index d3372a1cd..768620e59 100644 --- a/tools/mtmd/models/exaone4_5.cpp +++ b/tools/mtmd/models/exaone4_5.cpp @@ -1,7 +1,7 @@ // similar to qwen2vl, except for GQA attention #include "models.h" -ggml_cgraph * clip_graph_exaone4_5::build() { +ggml_cgraph * mtmd_graph_exaone4_5::build() { GGML_ASSERT(model.patch_bias == nullptr); GGML_ASSERT(model.class_embedding == nullptr); diff --git a/tools/mtmd/models/gemma4a.cpp b/tools/mtmd/models/gemma4a.cpp index f98a8b6fc..f1ac867ca 100644 --- a/tools/mtmd/models/gemma4a.cpp +++ b/tools/mtmd/models/gemma4a.cpp @@ -1,5 +1,5 @@ /** - * Gemma 4 Audio Conformer Encoder (clip_graph_gemma4a) + * Gemma 4 Audio Conformer Encoder (mtmd_graph_gemma4a) * * Architecture: Conformer with dual half-step FFN, full self-attention * with sinusoidal RPE, depthwise light conv, and output projection. @@ -8,7 +8,7 @@ #include "models.h" #include -ggml_cgraph * clip_graph_gemma4a::build() { +ggml_cgraph * mtmd_graph_gemma4a::build() { const float res_weight = 0.5f; const float norm_eps = 1e-6f; @@ -273,7 +273,7 @@ ggml_cgraph * clip_graph_gemma4a::build() { return gf; } -ggml_tensor * clip_graph_gemma4a::build_mm(ggml_tensor * w, ggml_tensor * x) const { +ggml_tensor * mtmd_graph_gemma4a::build_mm(ggml_tensor * w, ggml_tensor * x) const { auto it = model.clamp_info_map.find(w->name); if (it == model.clamp_info_map.end()) { return ggml_mul_mat(ctx0, w, x); diff --git a/tools/mtmd/models/gemma4ua.cpp b/tools/mtmd/models/gemma4ua.cpp index e24bef2ed..1d77b8889 100644 --- a/tools/mtmd/models/gemma4ua.cpp +++ b/tools/mtmd/models/gemma4ua.cpp @@ -1,7 +1,7 @@ #include "models.h" #include -ggml_cgraph * clip_graph_gemma4ua::build() { +ggml_cgraph * mtmd_graph_gemma4ua::build() { ggml_tensor * inp = build_inp_raw(1); auto cur = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3)); diff --git a/tools/mtmd/models/gemma4uv.cpp b/tools/mtmd/models/gemma4uv.cpp index 96031141b..04e139a1b 100644 --- a/tools/mtmd/models/gemma4uv.cpp +++ b/tools/mtmd/models/gemma4uv.cpp @@ -1,7 +1,7 @@ #include "models.h" #include -ggml_cgraph * clip_graph_gemma4uv::build() { +ggml_cgraph * mtmd_graph_gemma4uv::build() { ggml_tensor * inp_raw = build_inp_raw(); // Gemma4UnifiedVisionEmbedder uses default pytorch LayerNorm, not RMSNorm diff --git a/tools/mtmd/models/gemma4v.cpp b/tools/mtmd/models/gemma4v.cpp index 448438947..46733e0f3 100644 --- a/tools/mtmd/models/gemma4v.cpp +++ b/tools/mtmd/models/gemma4v.cpp @@ -1,7 +1,7 @@ #include "models.h" #include -ggml_cgraph * clip_graph_gemma4v::build() { +ggml_cgraph * mtmd_graph_gemma4v::build() { ggml_tensor * inp_raw = build_inp_raw(); // patches = 2 * (patches - 0.5) @@ -43,7 +43,7 @@ ggml_cgraph * clip_graph_gemma4v::build() { } // similar to build_rope_2d, but use neox ordering - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { const int64_t n_dim = cur->ne[0]; // first half, dims [0, n_dim/2) @@ -117,7 +117,7 @@ ggml_cgraph * clip_graph_gemma4v::build() { return gf; } -ggml_tensor * clip_graph_gemma4v::build_mm(ggml_tensor * w, ggml_tensor * x) const { +ggml_tensor * mtmd_graph_gemma4v::build_mm(ggml_tensor * w, ggml_tensor * x) const { // Gemma4ClippableLinear auto it = model.clamp_info_map.find(w->name); diff --git a/tools/mtmd/models/glm4v.cpp b/tools/mtmd/models/glm4v.cpp index 1cbcea942..2b9a30192 100644 --- a/tools/mtmd/models/glm4v.cpp +++ b/tools/mtmd/models/glm4v.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_glm4v::build() { +ggml_cgraph * mtmd_graph_glm4v::build() { GGML_ASSERT(model.patch_bias != nullptr); GGML_ASSERT(model.class_embedding == nullptr); @@ -63,7 +63,7 @@ ggml_cgraph * clip_graph_glm4v::build() { cb(learned_pos_embd, "learned_pos_embd", -1); } - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return ggml_rope_multi( ctx0, cur, positions, nullptr, d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, diff --git a/tools/mtmd/models/granite-speech.cpp b/tools/mtmd/models/granite-speech.cpp index 9725def82..64fb76c41 100644 --- a/tools/mtmd/models/granite-speech.cpp +++ b/tools/mtmd/models/granite-speech.cpp @@ -2,7 +2,7 @@ #include -ggml_cgraph * clip_graph_granite_speech::build() { +ggml_cgraph * mtmd_graph_granite_speech::build() { const int n_frames = img.nx(); const int context_size = hparams.audio_chunk_size; const int ctc_layer = n_layer / 2; diff --git a/tools/mtmd/models/granite4-vision.cpp b/tools/mtmd/models/granite4-vision.cpp index a75f1cee9..390d6edf6 100644 --- a/tools/mtmd/models/granite4-vision.cpp +++ b/tools/mtmd/models/granite4-vision.cpp @@ -1,6 +1,6 @@ #include "models.h" -#include "../clip-impl.h" -#include "../clip-model.h" +#include "../mtmd-impl.h" +#include "../mtmd-model.h" #include #include @@ -9,7 +9,7 @@ #include /* - * Granite Vision 4.1 clip graph + * Granite Vision 4.1 vision graph * * Stage 1a: SigLIP vision tower (N layers, post-norm) * Stage 1b: WindowQFormer blocks (deepstack + spatial) @@ -22,7 +22,7 @@ // --------------------------------------------------------------------------- // split the stacked tiles into the batch axis, then run the usual patch embedding -ggml_tensor * clip_graph_granite4_vision::build_tile_inp() { +ggml_tensor * mtmd_graph_granite4_vision::build_tile_inp() { ggml_tensor * inp_raw = build_inp_raw(); if (n_tiles > 1) { @@ -41,7 +41,7 @@ ggml_tensor * clip_graph_granite4_vision::build_tile_inp() { return inp; } -ggml_tensor * clip_graph_granite4_vision::gather( +ggml_tensor * mtmd_graph_granite4_vision::gather( ggml_tensor * src, const std::string & name, int idx_len) { @@ -52,7 +52,7 @@ ggml_tensor * clip_graph_granite4_vision::gather( return ggml_get_rows(ctx0, src, idx); } -ggml_tensor * clip_graph_granite4_vision::interp_down( +ggml_tensor * mtmd_graph_granite4_vision::interp_down( ggml_tensor * src, int side, int new_side) { @@ -72,7 +72,7 @@ ggml_tensor * clip_graph_granite4_vision::interp_down( // build_block - WindowQFormer block implementation // --------------------------------------------------------------------------- -ggml_tensor * clip_graph_granite4_vision::build_block( +ggml_tensor * mtmd_graph_granite4_vision::build_block( const qf_block & blk, ggml_tensor * h, int bid, @@ -244,7 +244,7 @@ ggml_tensor * clip_graph_granite4_vision::build_block( // Build the K-tiled, base-scaled newline row tensor. // Shape: (n_mmproj_embd, 1) -ggml_tensor * clip_graph_granite4_vision::build_newline_row(ggml_context * ctx0) { +ggml_tensor * mtmd_graph_granite4_vision::build_newline_row(ggml_context * ctx0) { const int K = (int) model.qf_proj_blocks.size(); GGML_ASSERT(K > 0); GGML_ASSERT(n_mmproj_embd % K == 0); @@ -272,7 +272,7 @@ ggml_tensor * clip_graph_granite4_vision::build_newline_row(ggml_context * ctx0) // Assemble [overview, tile(0,0), tile(0,1), ...] into one token sequence: // the overview tokens first, then the tile grid read in raster order with one newline per row. // ref: https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/llava_next/modeling_llava_next.py#L266 -ggml_tensor * clip_graph_granite4_vision::build_anyres_assembly(ggml_tensor * cur, int out_side) { +ggml_tensor * mtmd_graph_granite4_vision::build_anyres_assembly(ggml_tensor * cur, int out_side) { const int n_dim = cur->ne[0]; const int grid_x = anyres.grid_x; const int grid_y = anyres.grid_y; @@ -293,7 +293,7 @@ ggml_tensor * clip_graph_granite4_vision::build_anyres_assembly(ggml_tensor * cu // drop the tokens that only cover the padding added when resizing to the grid int off_x, off_y, w, h; - clip_anyres_unpad(cur_w, cur_h, anyres.orig_nx, anyres.orig_ny, off_x, off_y, w, h); + mtmd_anyres_unpad(cur_w, cur_h, anyres.orig_nx, anyres.orig_ny, off_x, off_y, w, h); if (w != cur_w || h != cur_h) { tiles = ggml_cont(ctx0, ggml_view_3d(ctx0, tiles, n_dim, w, h, tiles->nb[1], tiles->nb[2], @@ -307,7 +307,7 @@ ggml_tensor * clip_graph_granite4_vision::build_anyres_assembly(ggml_tensor * cu return ggml_concat(ctx0, base, tiles, 1); } -ggml_cgraph * clip_graph_granite4_vision::build() { +ggml_cgraph * mtmd_graph_granite4_vision::build() { GGML_ASSERT(model.patch_embeddings_0 != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); GGML_ASSERT(model.class_embedding == nullptr); diff --git a/tools/mtmd/models/hunyuanvl.cpp b/tools/mtmd/models/hunyuanvl.cpp index 2c670979d..b6b3fef58 100644 --- a/tools/mtmd/models/hunyuanvl.cpp +++ b/tools/mtmd/models/hunyuanvl.cpp @@ -1,12 +1,12 @@ #include "models.h" -ggml_cgraph * clip_graph_hunyuanvl::build() { +ggml_cgraph * mtmd_graph_hunyuanvl::build() { const int merge = hparams.n_merge; const int pw = n_patches_x; const int ph = n_patches_y; // position embedding: declared as a graph input, filled on CPU - // by clip_image_batch_encode (see PROJECTOR_TYPE_HUNYUANVL branch there). + // by mtmd_subcontext::encode() (see PROJECTOR_TYPE_HUNYUANVL branch there). ggml_tensor * pos_embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, ph * pw); ggml_set_name(pos_embd, "hunyuanvl_pos_embd"); ggml_set_input(pos_embd); diff --git a/tools/mtmd/models/internvl.cpp b/tools/mtmd/models/internvl.cpp index af8f12cdc..5e0b679c4 100644 --- a/tools/mtmd/models/internvl.cpp +++ b/tools/mtmd/models/internvl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_internvl::build() { +ggml_cgraph * mtmd_graph_internvl::build() { GGML_ASSERT(model.class_embedding != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); diff --git a/tools/mtmd/models/kimik25.cpp b/tools/mtmd/models/kimik25.cpp index cb345f0fc..13baf86d0 100644 --- a/tools/mtmd/models/kimik25.cpp +++ b/tools/mtmd/models/kimik25.cpp @@ -2,10 +2,10 @@ #include #include -// note: this is similar to clip_graph::resize_position_embeddings, major difference is having +// note: this is similar to mtmd_graph::resize_position_embeddings, major difference is having // the w/h in ne[1] and ne[2] instead of assuming with sqrt. Could try storing the tensor in 2D instead // with a w*h? Also the permute is a bit different at (2, 1, 0, 3) instead of (2, 0, 1, 3). -ggml_tensor * clip_graph_kimik25::resize_position_embeddings_3d(uint32_t interpolation_mode) { +ggml_tensor * mtmd_graph_kimik25::resize_position_embeddings_3d(uint32_t interpolation_mode) { ggml_tensor * pos_embd = model.position_embeddings; const int height = img.ny() / patch_size; const int width = img.nx() / patch_size; @@ -31,7 +31,7 @@ ggml_tensor * clip_graph_kimik25::resize_position_embeddings_3d(uint32_t interpo return pos_embd; } -ggml_cgraph * clip_graph_kimik25::build() { +ggml_cgraph * mtmd_graph_kimik25::build() { ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches); ggml_set_name(pos_h, "pos_h"); ggml_set_input(pos_h); @@ -44,7 +44,7 @@ ggml_cgraph * clip_graph_kimik25::build() { // Kimi-K2.5 uses interleaved 2D RoPE pattern natively, but // Q / K are permuted during conversion to use split format. - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { cur = build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false); return cur; }; diff --git a/tools/mtmd/models/kimivl.cpp b/tools/mtmd/models/kimivl.cpp index 0a06f5090..dd8146708 100644 --- a/tools/mtmd/models/kimivl.cpp +++ b/tools/mtmd/models/kimivl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_kimivl::build() { +ggml_cgraph * mtmd_graph_kimivl::build() { // 2D input positions ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches); ggml_set_name(pos_h, "pos_h"); @@ -13,7 +13,7 @@ ggml_cgraph * clip_graph_kimivl::build() { ggml_tensor * learned_pos_embd = resize_position_embeddings(); // build ViT with 2D position embeddings - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { // first half is X axis and second half is Y axis return build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false); }; diff --git a/tools/mtmd/models/lfm3a-gen.cpp b/tools/mtmd/models/lfm3a-gen.cpp index 998605711..f5aedac6f 100644 --- a/tools/mtmd/models/lfm3a-gen.cpp +++ b/tools/mtmd/models/lfm3a-gen.cpp @@ -9,7 +9,7 @@ // GEN_WAV : a window of frames -> polar STFT frames of the liquid-codec decoder, the ISTFT runs on the host // state carried between GEN_WAV calls: rope offset, short-conv taps and attention window of the codec decoder -std::vector list_lfm3a_gen_state_slots(const clip_hparams & hparams, const clip_model & model) { +std::vector list_lfm3a_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { std::vector slots; if (model.c2w.tfm_layers.empty()) { return slots; @@ -29,7 +29,7 @@ std::vector list_lfm3a_gen_state_slots(const clip_hparams & hpar } // pre-norm LFM block, with the LFM3 sandwich norms when the layer has them -ggml_tensor * clip_graph_lfm3a_gen::build_lfm_layer(ggml_tensor * inpL, const clip_layer & layer, float eps, int il, +ggml_tensor * mtmd_graph_lfm3a_gen::build_lfm_layer(ggml_tensor * inpL, const mtmd_layer & layer, float eps, int il, const std::function & op) { ggml_tensor * cur = build_norm(inpL, layer.ln_1_w, nullptr, NORM_TYPE_RMS, eps, il); cur = op(cur); @@ -47,7 +47,7 @@ ggml_tensor * clip_graph_lfm3a_gen::build_lfm_layer(ggml_tensor * inpL, const cl } // GQA with q/k RMSNorm, NEOX rope, an optional cached K/V prefix and the LFM3 per-head sigmoid gate -ggml_tensor * clip_graph_lfm3a_gen::build_lfm_attn(ggml_tensor * cur, const clip_layer & layer, int n_head, int n_head_kv, +ggml_tensor * mtmd_graph_lfm3a_gen::build_lfm_attn(ggml_tensor * cur, const mtmd_layer & layer, int n_head, int n_head_kv, float eps, float rope_theta, int il, ggml_tensor * inp_pos, ggml_tensor * kq_mask, ggml_tensor * k_cache, ggml_tensor * v_cache, ggml_tensor ** k_all, ggml_tensor ** v_all) { @@ -89,7 +89,7 @@ ggml_tensor * clip_graph_lfm3a_gen::build_lfm_attn(ggml_tensor * cur, const clip } // short conv: in_proj -> b * x -> causal depthwise conv over [conv_state | x] -> c * conv -> out_proj -ggml_tensor * clip_graph_lfm3a_gen::build_lfm_conv(ggml_tensor * cur, const clip_layer & layer, int il, +ggml_tensor * mtmd_graph_lfm3a_gen::build_lfm_conv(ggml_tensor * cur, const mtmd_layer & layer, int il, ggml_tensor * conv_state, ggml_tensor ** conv_state_out) { const int64_t n_embd_c = cur->ne[0]; const int64_t n_pos = cur->ne[1]; @@ -110,7 +110,7 @@ ggml_tensor * clip_graph_lfm3a_gen::build_lfm_conv(ggml_tensor * cur, const clip return build_mm(layer.conv_pw2_w, cur); } -ggml_tensor * clip_graph_lfm3a_gen::build_gen_code() { +ggml_tensor * mtmd_graph_lfm3a_gen::build_gen_code() { ggml_tensor * embd_w = model.gen_code_embd_w; // [n_embd, n_vocab, n_codebooks], tied to the output head const int64_t n_cb = embd_w->ne[2]; const int64_t n_vocab = embd_w->ne[1]; @@ -169,7 +169,7 @@ ggml_tensor * clip_graph_lfm3a_gen::build_gen_code() { return fb; } -ggml_tensor * clip_graph_lfm3a_gen::build_gen_wav() { +ggml_tensor * mtmd_graph_lfm3a_gen::build_gen_wav() { const auto & c2w = model.c2w; const int64_t n_cb = model.gen_code_embd_w->ne[2]; const int64_t n_fs = hparams.wav_frame_stack; @@ -201,7 +201,7 @@ ggml_tensor * clip_graph_lfm3a_gen::build_gen_wav() { cur = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, cur, n_embd_dec, 1, n_frames), n_embd_dec, n_fs, n_frames, 1); cur = ggml_reshape_2d(ctx0, cur, n_embd_dec, n_pos); - // positions continue across calls from the "pos" slot, see clip_encode() + // positions continue across calls from the "pos" slot, see mtmd_subcontext::encode() ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos); ggml_set_name(inp_pos, "positions"); ggml_set_input(inp_pos); @@ -250,8 +250,8 @@ ggml_tensor * clip_graph_lfm3a_gen::build_gen_wav() { return cur; } -ggml_cgraph * clip_graph_lfm3a_gen::build() { - ggml_tensor * out = gen_process == CLIP_GEN_PROCESS_GEN_WAV ? build_gen_wav() : build_gen_code(); +ggml_cgraph * mtmd_graph_lfm3a_gen::build() { + ggml_tensor * out = gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV ? build_gen_wav() : build_gen_code(); ggml_build_forward_expand(gf, out); return gf; } diff --git a/tools/mtmd/models/lfm3a.cpp b/tools/mtmd/models/lfm3a.cpp index 4882fc42f..03128a298 100644 --- a/tools/mtmd/models/lfm3a.cpp +++ b/tools/mtmd/models/lfm3a.cpp @@ -2,7 +2,7 @@ // streaming encoder state: base position, conv left context, short-conv taps and attention window // a zero state gives the same result as encoding the whole clip -std::vector list_lfm3a_state_slots(const clip_hparams & hparams, const clip_model & model) { +std::vector list_lfm3a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { std::vector slots; const int64_t n_mel = hparams.n_mel_bins; @@ -27,7 +27,7 @@ std::vector list_lfm3a_state_slots(const clip_hparams & hparams, } // LFM3-Audio encoder: conv downsampler (8x in time) -> causal LFM2 stack -> MLP projector -ggml_cgraph * clip_graph_lfm3a::build() { +ggml_cgraph * mtmd_graph_lfm3a::build() { const norm_type norm_t = NORM_TYPE_RMS; const float rope_theta = hparams.rope_theta; const int64_t prefix = hparams.attn_window_size - 1; diff --git a/tools/mtmd/models/lfm3vl.cpp b/tools/mtmd/models/lfm3vl.cpp index e37390976..6ad64ecb1 100644 --- a/tools/mtmd/models/lfm3vl.cpp +++ b/tools/mtmd/models/lfm3vl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_lfm3vl::build() { +ggml_cgraph * mtmd_graph_lfm3vl::build() { GGML_ASSERT(model.patch_embeddings_0 != nullptr); GGML_ASSERT(model.lfm3vl_embedding_norm_w != nullptr); GGML_ASSERT(model.class_embedding == nullptr); diff --git a/tools/mtmd/models/ling3vl.cpp b/tools/mtmd/models/ling3vl.cpp index fb9eb7c74..c3b8ed968 100644 --- a/tools/mtmd/models/ling3vl.cpp +++ b/tools/mtmd/models/ling3vl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_ling3vl::build() { +ggml_cgraph * mtmd_graph_ling3vl::build() { // same vision tower as qwen3vl, but the merger is norm-only (no fc1/fc2) and // the projector MLP lives at the top level (mm.0 / mm.2) GGML_ASSERT(model.class_embedding == nullptr); @@ -57,7 +57,7 @@ ggml_cgraph * clip_graph_ling3vl::build() { ggml_tensor * inpL = build_vit( inp, n_pos, norm_t, hparams.ffn_op, learned_pos_embd, - [&](ggml_tensor * c, const clip_layer &) { + [&](ggml_tensor * c, const mtmd_layer &) { return ggml_rope_multi( ctx0, c, positions, nullptr, d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1); diff --git a/tools/mtmd/models/llama4.cpp b/tools/mtmd/models/llama4.cpp index 01af54bba..60bfd5e2e 100644 --- a/tools/mtmd/models/llama4.cpp +++ b/tools/mtmd/models/llama4.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_llama4::build() { +ggml_cgraph * mtmd_graph_llama4::build() { GGML_ASSERT(model.class_embedding != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); @@ -31,7 +31,7 @@ ggml_cgraph * clip_graph_llama4::build() { inp = ggml_concat(ctx0, inp, model.class_embedding, 1); // build ViT with 2D position embeddings - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { // first half is X axis and second half is Y axis // ref: https://github.com/huggingface/transformers/blob/40a493c7ed4f19f08eadb0639cf26d49bfa5e180/src/transformers/models/llama4/modeling_llama4.py#L1312 // ref: https://github.com/Blaizzy/mlx-vlm/blob/a57156aa87b33cca6e5ee6cfc14dd4ef8f611be6/mlx_vlm/models/llama4/vision.py#L441 diff --git a/tools/mtmd/models/llava.cpp b/tools/mtmd/models/llava.cpp index 47efe68bd..f4f31ffdb 100644 --- a/tools/mtmd/models/llava.cpp +++ b/tools/mtmd/models/llava.cpp @@ -2,7 +2,7 @@ // this graph is used by llava, granite and glm // due to having embedding_stack (used by granite), we cannot reuse build_vit -ggml_cgraph * clip_graph_llava::build() { +ggml_cgraph * mtmd_graph_llava::build() { const int batch_size = 1; const int n_pos = n_patches + (model.class_embedding ? 1 : 0); diff --git a/tools/mtmd/models/magevl.cpp b/tools/mtmd/models/magevl.cpp index f5370ea9f..393f19aaf 100644 --- a/tools/mtmd/models/magevl.cpp +++ b/tools/mtmd/models/magevl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_magevl::build() { +ggml_cgraph * mtmd_graph_magevl::build() { GGML_ASSERT(hparams.n_merge == 2); const int n_pair = d_head / 2; @@ -14,7 +14,7 @@ ggml_cgraph * clip_graph_magevl::build() { ggml_set_input(positions); // a pair uses 2 angles: even outputs come from one rope, odd outputs from the other - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { ggml_tensor * freqs_a = ggml_view_1d(ctx0, model.rope_freqs, n_pair, 0); ggml_tensor * freqs_b = ggml_view_1d(ctx0, model.rope_freqs, n_pair, ggml_row_size(model.rope_freqs->type, n_pair)); diff --git a/tools/mtmd/models/mimo-audio.cpp b/tools/mtmd/models/mimo-audio.cpp index 0415bb60d..92be15fc9 100644 --- a/tools/mtmd/models/mimo-audio.cpp +++ b/tools/mtmd/models/mimo-audio.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_mimo_audio::build() { +ggml_cgraph * mtmd_graph_mimo_audio::build() { ggml_tensor * inp = build_inp_raw(1); // [n_frames, n_mel, 1] ggml_tensor * cur = ggml_conv_1d_ph(ctx0, model.conv1d_1_w, inp, 1, 1); @@ -43,7 +43,7 @@ ggml_cgraph * clip_graph_mimo_audio::build() { } }; - auto add_pos = [&](ggml_tensor * x, const clip_layer &) { + auto add_pos = [&](ggml_tensor * x, const mtmd_layer &) { return ggml_rope_ext(ctx0, x, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); }; diff --git a/tools/mtmd/models/mimovl.cpp b/tools/mtmd/models/mimovl.cpp index 4843698e6..6f2b8934f 100644 --- a/tools/mtmd/models/mimovl.cpp +++ b/tools/mtmd/models/mimovl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_tensor * clip_graph_mimovl::build_mm(ggml_tensor * w, ggml_tensor * x) const { +ggml_tensor * mtmd_graph_mimovl::build_mm(ggml_tensor * w, ggml_tensor * x) const { ggml_tensor * cur = ggml_mul_mat(ctx0, w, x); ggml_prec_set_acc(cur, GGML_PREC_BF16); return cur; @@ -19,7 +19,7 @@ ggml_tensor * clip_graph_mimovl::build_mm(ggml_tensor * w, ggml_tensor * x) cons // 4. 1D banded sliding window (|q-k| > window_size -> -inf) as a // single 2D mask broadcast across heads. // 5. Per-block MLP biases. -ggml_cgraph * clip_graph_mimovl::build() { +ggml_cgraph * mtmd_graph_mimovl::build() { GGML_ASSERT(model.patch_embeddings_0 != nullptr); GGML_ASSERT(model.patch_embeddings_1 != nullptr); GGML_ASSERT(model.class_embedding == nullptr); diff --git a/tools/mtmd/models/minicpmv.cpp b/tools/mtmd/models/minicpmv.cpp index 0e0fc0381..08ede0dd1 100644 --- a/tools/mtmd/models/minicpmv.cpp +++ b/tools/mtmd/models/minicpmv.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_minicpmv::build() { +ggml_cgraph * mtmd_graph_minicpmv::build() { GGML_ASSERT(model.class_embedding == nullptr); const int n_pos = n_patches; const int n_embd_proj = n_mmproj_embd; @@ -113,7 +113,7 @@ ggml_cgraph * clip_graph_minicpmv::build() { return gf; } -ggml_cgraph * clip_graph_minicpmv4_6::build() { +ggml_cgraph * mtmd_graph_minicpmv4_6::build() { const bool is_4x = hparams.n_merge == 2; const int n_pos = n_patches; const int half_h = n_patches_y / 2; diff --git a/tools/mtmd/models/minimax-m3.cpp b/tools/mtmd/models/minimax-m3.cpp index 256e53105..2ac437bf1 100644 --- a/tools/mtmd/models/minimax-m3.cpp +++ b/tools/mtmd/models/minimax-m3.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_tensor * clip_graph_minimax_m3::apply_rope( +ggml_tensor * mtmd_graph_minimax_m3::apply_rope( ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w) { const int dh = (int) x->ne[0]; const int axd = 2 * ((2 * (dh / 2) / 3) / 2); @@ -20,7 +20,7 @@ ggml_tensor * clip_graph_minimax_m3::apply_rope( return x; } -ggml_cgraph * clip_graph_minimax_m3::build() { +ggml_cgraph * mtmd_graph_minimax_m3::build() { GGML_ASSERT(model.patch_bias == nullptr); GGML_ASSERT(model.class_embedding == nullptr); GGML_ASSERT(model.patch_embeddings_0 && model.patch_embeddings_1); @@ -54,7 +54,7 @@ ggml_cgraph * clip_graph_minimax_m3::build() { ggml_tensor * inpL = build_vit( inp, n_pos, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr, - [&](ggml_tensor * c, const clip_layer &) { + [&](ggml_tensor * c, const mtmd_layer &) { return apply_rope(c, pos_h, pos_w); }); diff --git a/tools/mtmd/models/mobilenetv5.cpp b/tools/mtmd/models/mobilenetv5.cpp index 1c42218d2..e127cb31e 100644 --- a/tools/mtmd/models/mobilenetv5.cpp +++ b/tools/mtmd/models/mobilenetv5.cpp @@ -2,7 +2,7 @@ // Helpers for MobileNetV5 Blocks // RMS Norm 2D - normalizes over channels for each spatial position -ggml_tensor * clip_graph_mobilenetv5::rms_norm_2d(ggml_tensor * inp, ggml_tensor * weight, float eps) { +ggml_tensor * mtmd_graph_mobilenetv5::rms_norm_2d(ggml_tensor * inp, ggml_tensor * weight, float eps) { // inp: [W, H, C, B] ggml_tensor * cur = ggml_permute(ctx0, inp, 2, 1, 0, 3); @@ -20,7 +20,7 @@ ggml_tensor * clip_graph_mobilenetv5::rms_norm_2d(ggml_tensor * inp, ggml_tensor } // Conv2dSame padding - asymmetric SAME padding like PyTorch/TF -ggml_tensor* clip_graph_mobilenetv5::pad_same_2d(ggml_tensor* inp, int kernel_h, int kernel_w, int stride_h, int stride_w, int dilation_h, int dilation_w) { +ggml_tensor* mtmd_graph_mobilenetv5::pad_same_2d(ggml_tensor* inp, int kernel_h, int kernel_w, int stride_h, int stride_w, int dilation_h, int dilation_w) { const int64_t ih = inp->ne[1]; // height const int64_t iw = inp->ne[0]; // width @@ -54,7 +54,7 @@ ggml_tensor* clip_graph_mobilenetv5::pad_same_2d(ggml_tensor* inp, int kernel_h, // Edge Residual Block (Stage 0) -ggml_tensor * clip_graph_mobilenetv5::build_edge_residual(ggml_tensor * inp, const mobilenetv5_block & block, int stride) { +ggml_tensor * mtmd_graph_mobilenetv5::build_edge_residual(ggml_tensor * inp, const mobilenetv5_block & block, int stride) { ggml_tensor * cur = inp; // 1. Expansion Conv (3x3) @@ -88,7 +88,7 @@ ggml_tensor * clip_graph_mobilenetv5::build_edge_residual(ggml_tensor * inp, con } // Universal Inverted Residual Block (Stage 1+) -ggml_tensor * clip_graph_mobilenetv5::build_inverted_residual(ggml_tensor * inp, const mobilenetv5_block & block, int stride) { +ggml_tensor * mtmd_graph_mobilenetv5::build_inverted_residual(ggml_tensor * inp, const mobilenetv5_block & block, int stride) { ggml_tensor * cur = inp; // 1. Depthwise Start (Optional) @@ -149,7 +149,7 @@ ggml_tensor * clip_graph_mobilenetv5::build_inverted_residual(ggml_tensor * inp, } // Attention Block (MQA) -ggml_tensor * clip_graph_mobilenetv5::build_mobilenet_attn(ggml_tensor * inp, const mobilenetv5_block & block) { +ggml_tensor * mtmd_graph_mobilenetv5::build_mobilenet_attn(ggml_tensor * inp, const mobilenetv5_block & block) { ggml_tensor * cur = inp; // Norm @@ -245,7 +245,7 @@ ggml_tensor * clip_graph_mobilenetv5::build_mobilenet_attn(ggml_tensor * inp, co return cur; } -ggml_cgraph * clip_graph_mobilenetv5::build() { +ggml_cgraph * mtmd_graph_mobilenetv5::build() { ggml_tensor * inp = build_inp_raw(); // 1. Stem - Conv2dSame(3, 64, kernel_size=(3, 3), stride=(2, 2)) diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 8487a0042..17bd57d95 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -1,6 +1,6 @@ #pragma once -#include "../clip-graph.h" +#include "../mtmd-graph.h" #include #include @@ -13,32 +13,32 @@ * We encourage human contributors to ensure the quality and reliability of the codebase. */ -struct clip_graph_siglip : clip_graph { - clip_graph_siglip(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_siglip : mtmd_graph { + mtmd_graph_siglip(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_gemma4v : clip_graph { - clip_graph_gemma4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_gemma4v : mtmd_graph { + mtmd_graph_gemma4v(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override; bool support_batch() const override { return true; } }; -struct clip_graph_gemma4uv : clip_graph { - clip_graph_gemma4uv(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_gemma4uv : mtmd_graph { + mtmd_graph_gemma4uv(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_nemotron_transcribe : clip_graph { - const clip_enc_state * state; // nullptr during warmup - clip_graph_nemotron_transcribe(clip_ctx * ctx, const clip_image_f32 & img, const clip_enc_state * state) - : clip_graph(ctx, img), state(state) {} +struct mtmd_graph_nemotron_transcribe : mtmd_graph { + const mtmd_subctx_enc_state * state; // nullptr during warmup + mtmd_graph_nemotron_transcribe(mtmd_subcontext * ctx, const mtmd_image_f32 & img, const mtmd_subctx_enc_state * state) + : mtmd_graph(ctx, img), state(state) {} ggml_cgraph * build() override; ggml_tensor * build_spk_encoder(ggml_tensor * cur, ggml_tensor * pos); // one recording: encodes its windows in turn, carrying the speaker state between them - static bool encode_windows(clip_ctx * ctx, const clip_model & model, clip_encode_params * params); + static bool encode_windows(mtmd_subcontext * ctx, const mtmd_model & model, mtmd_subctx_encode_params * params); // NeMo ParallelExpertEncoder online defaults, in output frames: 40 s windows with 4 s of context on each side static constexpr int window_core = 500; @@ -46,54 +46,54 @@ struct clip_graph_nemotron_transcribe : clip_graph { static constexpr int window_right = 50; }; -struct clip_graph_pixtral : clip_graph { - clip_graph_pixtral(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_pixtral : mtmd_graph { + mtmd_graph_pixtral(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_deepseek4v : clip_graph { - clip_graph_deepseek4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_deepseek4v : mtmd_graph { + mtmd_graph_deepseek4v(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_qwen2vl : clip_graph { - clip_graph_qwen2vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_qwen2vl : mtmd_graph { + mtmd_graph_qwen2vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_qwen3vl : clip_graph_qwen2vl { - clip_graph_qwen3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen2vl(ctx, img) {} +struct mtmd_graph_qwen3vl : mtmd_graph_qwen2vl { + mtmd_graph_qwen3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_qwen2vl(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_magevl : clip_graph { - clip_graph_magevl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_magevl : mtmd_graph { + mtmd_graph_magevl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_sensenova_u1 : clip_graph { - clip_graph_sensenova_u1(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_sensenova_u1 : mtmd_graph { + mtmd_graph_sensenova_u1(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_lfm3vl : clip_graph { - clip_graph_lfm3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_lfm3vl : mtmd_graph { + mtmd_graph_lfm3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_ling3vl : clip_graph_qwen3vl { - clip_graph_ling3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen3vl(ctx, img) {} +struct mtmd_graph_ling3vl : mtmd_graph_qwen3vl { + mtmd_graph_ling3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_qwen3vl(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_minimax_m3 : clip_graph { - clip_graph_minimax_m3(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_minimax_m3 : mtmd_graph { + mtmd_graph_minimax_m3(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * apply_rope(ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w); }; -struct clip_graph_mimovl : clip_graph { - clip_graph_mimovl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_mimovl : mtmd_graph { + mtmd_graph_mimovl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; // Force F32 mat-mul accumulation to avoid F16 overflow in the FFN down-proj // when the mmproj is stored in F16 (the source weights are BF16; downcasting @@ -101,119 +101,119 @@ struct clip_graph_mimovl : clip_graph { ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override; }; -struct clip_graph_step3vl : clip_graph { - clip_graph_step3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_step3vl : mtmd_graph { + mtmd_graph_step3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_youtuvl : clip_graph { - clip_graph_youtuvl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_youtuvl : mtmd_graph { + mtmd_graph_youtuvl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_yasa2 : clip_graph { - clip_graph_yasa2(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_yasa2 : mtmd_graph { + mtmd_graph_yasa2(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * layer_norm_channels(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b, float eps = 1e-6f); ggml_tensor * convnext_grn(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b); }; -struct clip_graph_minicpmv : clip_graph { - clip_graph_minicpmv(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_minicpmv : mtmd_graph { + mtmd_graph_minicpmv(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_minicpmv4_6 : clip_graph { - clip_graph_minicpmv4_6(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_minicpmv4_6 : mtmd_graph { + mtmd_graph_minicpmv4_6(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_internvl : clip_graph { - clip_graph_internvl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_internvl : mtmd_graph { + mtmd_graph_internvl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; bool support_batch() const override { return true; } }; -struct clip_graph_nemotron_v2_vl : clip_graph { - clip_graph_nemotron_v2_vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_nemotron_v2_vl : mtmd_graph { + mtmd_graph_nemotron_v2_vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_nemotron_omni : clip_graph { - clip_graph_nemotron_omni(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_nemotron_omni : mtmd_graph { + mtmd_graph_nemotron_omni(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * build_inp_with_temporal_merge(); }; -struct clip_graph_llama4 : clip_graph { - clip_graph_llama4(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_llama4 : mtmd_graph { + mtmd_graph_llama4(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_kimivl : clip_graph { - clip_graph_kimivl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_kimivl : mtmd_graph { + mtmd_graph_kimivl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_paddleocr : clip_graph { - clip_graph_paddleocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_paddleocr : mtmd_graph { + mtmd_graph_paddleocr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_dotsocr : clip_graph { - clip_graph_dotsocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_dotsocr : mtmd_graph { + mtmd_graph_dotsocr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_dots3note_a : clip_graph { - clip_graph_dots3note_a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_dots3note_a : mtmd_graph { + mtmd_graph_dots3note_a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_cogvlm : clip_graph { - clip_graph_cogvlm(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_cogvlm : mtmd_graph { + mtmd_graph_cogvlm(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_llava : clip_graph { - clip_graph_llava(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_llava : mtmd_graph { + mtmd_graph_llava(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_whisper_enc : clip_graph { - clip_graph_whisper_enc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_whisper_enc : mtmd_graph { + mtmd_graph_whisper_enc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; // causal whisper-like encoder with a sliding window KV cache (voxtral realtime) -struct clip_graph_voxtral_rt : clip_graph { - clip_enc_state * state; // nullptr during warmup, the cache is then a graph input - clip_graph_voxtral_rt(clip_ctx * ctx, const clip_image_f32 & img, clip_enc_state * state) : clip_graph(ctx, img), state(state) {} +struct mtmd_graph_voxtral_rt : mtmd_graph { + mtmd_subctx_enc_state * state; // nullptr during warmup, the cache is then a graph input + mtmd_graph_voxtral_rt(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_enc_state * state) : mtmd_graph(ctx, img), state(state) {} ggml_cgraph * build() override; }; -struct clip_graph_deepseekocr : clip_graph { - clip_graph_deepseekocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_deepseekocr : mtmd_graph { + mtmd_graph_deepseekocr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * build_sam(ggml_tensor * inp); // build the SAM model bool support_batch() const override { return true; } }; -struct clip_graph_deepseekocr2 : clip_graph_deepseekocr { - clip_graph_deepseekocr2(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_deepseekocr(ctx, img) {} +struct mtmd_graph_deepseekocr2 : mtmd_graph_deepseekocr { + mtmd_graph_deepseekocr2(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_deepseekocr(ctx, img) {} ggml_cgraph * build() override; // reuses build_sam() from base bool support_batch() const override { return true; } }; -struct clip_graph_conformer : clip_graph { - clip_graph_conformer(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_conformer : mtmd_graph { + mtmd_graph_conformer(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; // builders shared by the chatterbox gen and spkenc graphs, defined in chatterbox-gen.cpp -struct clip_graph_chatterbox_base : clip_graph { - using clip_graph::clip_graph; +struct mtmd_graph_chatterbox_base : mtmd_graph { + using mtmd_graph::mtmd_graph; protected: ggml_tensor * cbx_linear(ggml_tensor * w, ggml_tensor * b, ggml_tensor * x) const; @@ -223,79 +223,79 @@ struct clip_graph_chatterbox_base : clip_graph { int pad, int dil) const; }; -struct clip_graph_chatterbox_spkenc : clip_graph_chatterbox_base { - clip_graph_chatterbox_spkenc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_chatterbox_base(ctx, img) {} +struct mtmd_graph_chatterbox_spkenc : mtmd_graph_chatterbox_base { + mtmd_graph_chatterbox_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_chatterbox_base(ctx, img) {} ggml_cgraph * build() override; private: - ggml_tensor * bn1d(const clip_chatterbox::bn & n, ggml_tensor * x); - ggml_tensor * bn2d_relu(const clip_chatterbox::bn & n, ggml_tensor * x); - ggml_tensor * res2d(const clip_chatterbox::spk_res2d & r, ggml_tensor * x, int stride); - ggml_tensor * cam_layer(const clip_chatterbox::spk_cam_layer & l, ggml_tensor * x, int dil, ggml_tensor * segfix); + ggml_tensor * bn1d(const mtmd_chatterbox::bn & n, ggml_tensor * x); + ggml_tensor * bn2d_relu(const mtmd_chatterbox::bn & n, ggml_tensor * x); + ggml_tensor * res2d(const mtmd_chatterbox::spk_res2d & r, ggml_tensor * x, int stride); + ggml_tensor * cam_layer(const mtmd_chatterbox::spk_cam_layer & l, ggml_tensor * x, int dil, ggml_tensor * segfix); }; -struct clip_graph_chatterbox : clip_graph_chatterbox_base { - clip_gen_process_type gen_process = CLIP_GEN_PROCESS_GEN_CODE; +struct mtmd_graph_chatterbox : mtmd_graph_chatterbox_base { + mtmd_subctx_gen_process gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; int n_tokens = 0; int n_prompt_mel = 0; int vocode_n_mel = 0; int vocode_n_stft = 0; - clip_graph_chatterbox(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, int n_tokens, + mtmd_graph_chatterbox(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int n_tokens, int n_prompt_mel, int vocode_n_mel, int vocode_n_stft) - : clip_graph_chatterbox_base(ctx, img), gen_process(gen_process), n_tokens(n_tokens), n_prompt_mel(n_prompt_mel), + : mtmd_graph_chatterbox_base(ctx, img), gen_process(gen_process), n_tokens(n_tokens), n_prompt_mel(n_prompt_mel), vocode_n_mel(vocode_n_mel), vocode_n_stft(vocode_n_stft) {} ggml_cgraph * build() override; private: - ggml_tensor * enc_layer(const clip_chatterbox::enc_layer & l, ggml_tensor * x, ggml_tensor * pos, int T); - ggml_tensor * causal_block(const clip_chatterbox::causal_block & b, ggml_tensor * x); - ggml_tensor * resnet(const clip_chatterbox::resnet & r, ggml_tensor * x, ggml_tensor * temb); - ggml_tensor * tfm_block(const clip_chatterbox::tfm_block & b, ggml_tensor * x); + ggml_tensor * enc_layer(const mtmd_chatterbox::enc_layer & l, ggml_tensor * x, ggml_tensor * pos, int T); + ggml_tensor * causal_block(const mtmd_chatterbox::causal_block & b, ggml_tensor * x); + ggml_tensor * resnet(const mtmd_chatterbox::resnet & r, ggml_tensor * x, ggml_tensor * temb); + ggml_tensor * tfm_block(const mtmd_chatterbox::tfm_block & b, ggml_tensor * x); ggml_tensor * estimator(ggml_tensor * x_noise, ggml_tensor * mu, ggml_tensor * spks, ggml_tensor * cond, ggml_tensor * temb, int T); - ggml_tensor * hift_resblock(const clip_chatterbox::hift_res & r, ggml_tensor * x); + ggml_tensor * hift_resblock(const mtmd_chatterbox::hift_res & r, ggml_tensor * x); ggml_cgraph * build_s3tok(int T); ggml_cgraph * build_vocoder(int n_mel, int n_stft); }; -struct clip_graph_lfm3a : clip_graph { - clip_graph_lfm3a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_lfm3a : mtmd_graph { + mtmd_graph_lfm3a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_granite_speech : clip_graph { - clip_graph_granite_speech(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_granite_speech : mtmd_graph { + mtmd_graph_granite_speech(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_muse_spark_audio : clip_graph { - clip_graph_muse_spark_audio(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_muse_spark_audio : mtmd_graph { + mtmd_graph_muse_spark_audio(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_gemma4a : clip_graph { - clip_graph_gemma4a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_gemma4a : mtmd_graph { + mtmd_graph_gemma4a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override; }; -struct clip_graph_gemma4ua : clip_graph { - clip_graph_gemma4ua(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_gemma4ua : mtmd_graph { + mtmd_graph_gemma4ua(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_glm4v : clip_graph { - clip_graph_glm4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_glm4v : mtmd_graph { + mtmd_graph_glm4v(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_hunyuanvl : clip_graph { - clip_graph_hunyuanvl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_hunyuanvl : mtmd_graph { + mtmd_graph_hunyuanvl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_mobilenetv5 : clip_graph { - clip_graph_mobilenetv5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_mobilenetv5 : mtmd_graph { + mtmd_graph_mobilenetv5(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * rms_norm_2d( @@ -327,34 +327,34 @@ struct clip_graph_mobilenetv5 : clip_graph { const mobilenetv5_block & block); }; -struct clip_graph_qwen3a : clip_graph { - clip_graph_qwen3a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_qwen3a : mtmd_graph { + mtmd_graph_qwen3a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_mimo_audio : clip_graph { - clip_graph_mimo_audio(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_mimo_audio : mtmd_graph { + mtmd_graph_mimo_audio(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_qwen3tts_spkenc : clip_graph { - clip_graph_qwen3tts_spkenc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_qwen3tts_spkenc : mtmd_graph { + mtmd_graph_qwen3tts_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const; - ggml_tensor * res2net(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const; - ggml_tensor * se_block(ggml_tensor * x, const clip_layer & layer) const; - ggml_tensor * se_res2net_block(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const; + ggml_tensor * res2net(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const; + ggml_tensor * se_block(ggml_tensor * x, const mtmd_layer & layer) const; + ggml_tensor * se_res2net_block(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const; ggml_tensor * attentive_stats_pool(ggml_tensor * x) const; }; -struct clip_graph_qwen3tts_gen : clip_graph { - clip_graph_qwen3tts_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, int top_k, float top_p) - : clip_graph(ctx, img), gen_process(gen_process), top_k(top_k), top_p(top_p) {} +struct mtmd_graph_qwen3tts_gen : mtmd_graph { + mtmd_graph_qwen3tts_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int top_k, float top_p) + : mtmd_graph(ctx, img), gen_process(gen_process), top_k(top_k), top_p(top_p) {} ggml_cgraph * build() override; // which sub-graph build() constructs, fixed at graph-build time - clip_gen_process_type gen_process; + mtmd_subctx_gen_process gen_process; // sampling params, fixed at graph-build time (GEN_CODE only) int top_k; @@ -364,9 +364,9 @@ struct clip_graph_qwen3tts_gen : clip_graph { // code_gen: backbone hidden state + sampled code0 -> 16 RVQ codes // MTP-style code predictor, one token per codebook // - struct code_gen : clip_graph { - code_gen(const clip_graph & parent, int top_k, float top_p) - : clip_graph(parent), top_k(top_k), top_p(top_p) {} + struct code_gen : mtmd_graph { + code_gen(const mtmd_graph & parent, int top_k, float top_p) + : mtmd_graph(parent), top_k(top_k), top_p(top_p) {} ggml_cgraph * build() override { GGML_ABORT("call prefill()/step() instead"); } int top_k; @@ -381,7 +381,7 @@ struct clip_graph_qwen3tts_gen : clip_graph { ggml_tensor * layer_forward( ggml_tensor * cur, - const clip_layer & layer, + const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, ggml_tensor *& k_cache_layer, @@ -409,8 +409,8 @@ struct clip_graph_qwen3tts_gen : clip_graph { // // code2wav: RVQ codes -> raw PCM (quantizer + pre_conv + pre_transformer + upsample + DAC). // - struct code2wav : clip_graph { - code2wav(const clip_graph & parent) : clip_graph(parent) {} + struct code2wav : mtmd_graph { + code2wav(const mtmd_graph & parent) : mtmd_graph(parent) {} ggml_cgraph * build() override { GGML_ABORT("call decode() instead"); } // state_in: previous call's persisted state, by slot name (see list_c2w_state_slots()) @@ -425,9 +425,9 @@ struct clip_graph_qwen3tts_gen : clip_graph { ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const; ggml_tensor * quant_decode(ggml_tensor * inp_codes) const; - ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, int il) const; - ggml_tensor * convnext_block(ggml_tensor * x, const clip_code2wav::upsample_block & blk, const std::string & state_prefix) const; - ggml_tensor * dac_res_unit(ggml_tensor * x, const clip_code2wav::dac_res & res, int dilation, const std::string & state_name) const; + ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il) const; + ggml_tensor * convnext_block(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk, const std::string & state_prefix) const; + ggml_tensor * dac_res_unit(ggml_tensor * x, const mtmd_code2wav::dac_res & res, int dilation, const std::string & state_name) const; // drama3: erf GELU in ConvNeXt, not the tanh approximation bool gelu_erf = false; @@ -441,8 +441,8 @@ struct clip_graph_qwen3tts_gen : clip_graph { // pocket-tts: SEANet convolution stack, shared by the voice encoder and the mimi decoder. // stateless unless state_in is populated: convs then pad instead of carrying left-context. // -struct clip_graph_pockettts_seanet : clip_graph { - clip_graph_pockettts_seanet(const clip_graph & parent) : clip_graph(parent) {} +struct mtmd_graph_pockettts_seanet : mtmd_graph { + mtmd_graph_pockettts_seanet(const mtmd_graph & parent) : mtmd_graph(parent) {} ggml_cgraph * build() override { GGML_ABORT("call encode()/decode() instead"); } // per-call streaming state, keyed by slot name (see list_pockettts_state_slots) @@ -453,7 +453,7 @@ struct clip_graph_pockettts_seanet : clip_graph { bool pad_replicate = false, const std::string & state_name = "") const; ggml_tensor * conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name = "") const; - ggml_tensor * res_unit(ggml_tensor * x, const clip_seanet::stage & stage, int dilation, + ggml_tensor * res_unit(ggml_tensor * x, const mtmd_seanet::stage & stage, int dilation, const std::string & state_prefix = "") const; // x: [T, C] -> [T / hop, dim] @@ -463,11 +463,11 @@ struct clip_graph_pockettts_seanet : clip_graph { }; // mimi encoder + speaker_proj: reference waveform -> voice conditioning rows -struct clip_graph_pockettts_spkenc : clip_graph { - clip_graph_pockettts_spkenc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_pockettts_spkenc : mtmd_graph { + mtmd_graph_pockettts_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; - ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const; + ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const; }; // @@ -475,18 +475,18 @@ struct clip_graph_pockettts_spkenc : clip_graph { // GEN_CODE = flow-matching decoder + end-of-speech head, one latent per call // GEN_WAV = mimi decoder, a window of latents -> PCM // -struct clip_graph_pockettts_gen : clip_graph { - clip_graph_pockettts_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, int n_step, int n_frames) - : clip_graph(ctx, img), gen_process(gen_process), n_step(n_step), n_frames(n_frames) {} +struct mtmd_graph_pockettts_gen : mtmd_graph { + mtmd_graph_pockettts_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int n_step, int n_frames) + : mtmd_graph(ctx, img), gen_process(gen_process), n_step(n_step), n_frames(n_frames) {} ggml_cgraph * build() override; - clip_gen_process_type gen_process; + mtmd_subctx_gen_process gen_process; int n_step; // lsd_decode steps, fixed at graph-build time int n_frames; // GEN_WAV only: number of latents to decode // AdaLN modulation: x * (1 + scale) + shift ggml_tensor * modulate(ggml_tensor * x, ggml_tensor * shift, ggml_tensor * scale) const; - ggml_tensor * time_embed(const clip_flow_net::time_embd & te, float t) const; + ggml_tensor * time_embed(const mtmd_flow_net::time_embd & te, float t) const; ggml_tensor * flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const; }; @@ -497,30 +497,30 @@ struct c2w_state_slot { int64_t ne1; bool in_place = false; // graph writes the slot directly, no state_in/state_out copy }; -std::vector list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model); +std::vector list_c2w_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); // drama3: tower KV slots, then list_c2w_state_slots() -std::vector list_drama3_state_slots(const clip_hparams & hparams, const clip_model & model); +std::vector list_drama3_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); // same, for the streaming mimi decoder (pocket-tts GEN_WAV) -std::vector list_pockettts_state_slots(const clip_hparams & hparams, const clip_model & model); +std::vector list_pockettts_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); // // drama3 generation: // GEN_CODE = one codebook tower row, each layer attends over the whole sequence (KV in gen state) // GEN_WAV = Fish DAC decoder, frames -> PCM // -struct clip_graph_drama3_gen : clip_graph { - clip_graph_drama3_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, clip_gen_state * gen_state, +struct mtmd_graph_drama3_gen : mtmd_graph { + mtmd_graph_drama3_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, mtmd_subctx_gen_state * gen_state, int n_frames, int top_k, float top_p, float temp) - : clip_graph(ctx, img), gen_process(gen_process), gen_state(gen_state), n_frames(n_frames), top_k(top_k), top_p(top_p), temp(temp) {} + : mtmd_graph(ctx, img), gen_process(gen_process), gen_state(gen_state), n_frames(n_frames), top_k(top_k), top_p(top_p), temp(temp) {} ggml_cgraph * build() override; // tower KV rows, shared by the prompt and the generated frames static constexpr int n_pos_max = 4096; - clip_gen_process_type gen_process; - clip_gen_state * gen_state; // nullptr on buffer reserve + mtmd_subctx_gen_process gen_process; + mtmd_subctx_gen_state * gen_state; // nullptr on buffer reserve int n_frames; // GEN_WAV only int top_k; float top_p; @@ -532,10 +532,10 @@ struct clip_graph_drama3_gen : clip_graph { ggml_tensor * inp_pos, ggml_tensor * k_slot, ggml_tensor * v_slot, int pos); }; -struct clip_graph_soprano : clip_graph { - clip_graph_soprano(clip_ctx * ctx, const clip_image_f32 & img, int n_frames) : clip_graph(ctx, img), n_frames(n_frames) {} +struct mtmd_graph_soprano : mtmd_graph { + mtmd_graph_soprano(mtmd_subcontext * ctx, const mtmd_image_f32 & img, int n_frames) : mtmd_graph(ctx, img), n_frames(n_frames) {} ggml_cgraph * build() override; - static void decode_spectrum(const clip_model & model, const std::vector & spectrum, std::vector & pcm); + static void decode_spectrum(const mtmd_model & model, const std::vector & spectrum, std::vector & pcm); // SopranoDecoder default static constexpr int upscale = 4; int n_frames; @@ -546,54 +546,54 @@ struct clip_graph_soprano : clip_graph { // GEN_CODE = one MaskGIT pass of the depthformer, or only the feedback embedding once all codes are known // GEN_WAV = liquid-codec decoder, a window of frames -> polar STFT frames // -struct clip_graph_lfm3a_gen : clip_graph { - clip_graph_lfm3a_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, bool all_known, int n_frames) - : clip_graph(ctx, img), gen_process(gen_process), all_known(all_known), n_frames(n_frames) {} +struct mtmd_graph_lfm3a_gen : mtmd_graph { + mtmd_graph_lfm3a_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, bool all_known, int n_frames) + : mtmd_graph(ctx, img), gen_process(gen_process), all_known(all_known), n_frames(n_frames) {} ggml_cgraph * build() override; - clip_gen_process_type gen_process; + mtmd_subctx_gen_process gen_process; bool all_known; // GEN_CODE only int n_frames; // GEN_WAV only ggml_tensor * build_gen_code(); ggml_tensor * build_gen_wav(); - ggml_tensor * build_lfm_layer(ggml_tensor * inpL, const clip_layer & layer, float eps, int il, + ggml_tensor * build_lfm_layer(ggml_tensor * inpL, const mtmd_layer & layer, float eps, int il, const std::function & op); - ggml_tensor * build_lfm_attn(ggml_tensor * cur, const clip_layer & layer, int n_head, int n_head_kv, + ggml_tensor * build_lfm_attn(ggml_tensor * cur, const mtmd_layer & layer, int n_head, int n_head_kv, float eps, float rope_theta, int il, ggml_tensor * inp_pos, ggml_tensor * kq_mask, ggml_tensor * k_cache, ggml_tensor * v_cache, ggml_tensor ** k_all, ggml_tensor ** v_all); - ggml_tensor * build_lfm_conv(ggml_tensor * cur, const clip_layer & layer, int il, + ggml_tensor * build_lfm_conv(ggml_tensor * cur, const mtmd_layer & layer, int il, ggml_tensor * conv_state, ggml_tensor ** conv_state_out); }; // state slots of the streaming codec decoder (lfm3a_gen GEN_WAV) -std::vector list_lfm3a_gen_state_slots(const clip_hparams & hparams, const clip_model & model); +std::vector list_lfm3a_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); // state slots of the streaming LFM3-Audio encoder (lfm3a) -std::vector list_lfm3a_state_slots(const clip_hparams & hparams, const clip_model & model); +std::vector list_lfm3a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); -struct clip_graph_kimik25 : clip_graph { - clip_graph_kimik25(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_kimik25 : mtmd_graph { + mtmd_graph_kimik25(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode); }; -struct clip_graph_parakeet : clip_graph { - clip_graph_parakeet(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_parakeet : mtmd_graph { + mtmd_graph_parakeet(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_exaone4_5 : clip_graph { - clip_graph_exaone4_5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_exaone4_5 : mtmd_graph { + mtmd_graph_exaone4_5(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_granite4_vision : clip_graph { - clip_graph_granite4_vision(clip_ctx * ctx, const clip_image_f32 & img) - : clip_graph(ctx, img), +struct mtmd_graph_granite4_vision : mtmd_graph { + mtmd_graph_granite4_vision(mtmd_subcontext * ctx, const mtmd_image_f32 & img) + : mtmd_graph(ctx, img), anyres(img.anyres), n_tiles(img.ny() / img.nx()), tile_side(img.nx() / patch_size) {} @@ -602,7 +602,7 @@ struct clip_graph_granite4_vision : clip_graph { private: // the input image is a stack of tiles on the Y axis: [overview, tile(0,0), tile(0,1), ...] - const clip_image_f32::anyres_info anyres; + const mtmd_image_f32::anyres_info anyres; const int n_tiles; const int tile_side; // patches per tile side @@ -617,7 +617,7 @@ private: ggml_tensor * build_anyres_assembly(ggml_tensor * cur, int out_side); }; -struct clip_graph_muse_glimmer : clip_graph { - clip_graph_muse_glimmer(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_muse_glimmer : mtmd_graph { + mtmd_graph_muse_glimmer(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; diff --git a/tools/mtmd/models/muse-glimmer.cpp b/tools/mtmd/models/muse-glimmer.cpp index a397c1b22..869954f99 100644 --- a/tools/mtmd/models/muse-glimmer.cpp +++ b/tools/mtmd/models/muse-glimmer.cpp @@ -5,13 +5,13 @@ // adapter MLP + LLM's vision_projection. // // Several quantities are precomputed on host and fed as named graph inputs (filled in -// clip.cpp set_input, PROJECTOR_TYPE_MUSE_GLIMMER branch): +// mtmd-subcontext.cpp set_input, PROJECTOR_TYPE_MUSE_GLIMMER branch): // muse_glimmer_pos_w/_h [n_tok] i32 : 1-indexed RoPE positions (sparse-permuted order) // muse_glimmer_sp_perm [n_tok] i32 : window grouping permutation (applied after ln_pre) // muse_glimmer_inv_perm [n_tok] i32 : inverse of sp_perm (applied after blocks) // muse_glimmer_ds_perm [n_tok] i32 : pixel-shuffle gather (original order) // attn_mask [n_tok, n_tok] f32 (f16 with flash attn) : block-diagonal window mask (sparse layers) -ggml_cgraph * clip_graph_muse_glimmer::build() { +ggml_cgraph * mtmd_graph_muse_glimmer::build() { const int ds = hparams.n_merge; // downsample factor (2) const int sf = hparams.muse_glimmer_sparse_factor; // 4 const int n_tok = n_patches; @@ -50,7 +50,7 @@ ggml_cgraph * clip_graph_muse_glimmer::build() { } // 2D RoPE: first half of head_dim uses width pos, second half uses height pos - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return build_rope_2d(ctx0, cur, pos_w, pos_h, rope_base, false); }; diff --git a/tools/mtmd/models/muse-spark-audio.cpp b/tools/mtmd/models/muse-spark-audio.cpp index d5340d5f9..20e770ef8 100644 --- a/tools/mtmd/models/muse-spark-audio.cpp +++ b/tools/mtmd/models/muse-spark-audio.cpp @@ -2,7 +2,7 @@ // Muse Spark audio tower: a causal conv pyramid over the raw waveform, no attention and no spectrogram // each stage folds `patch` samples into the channel dim, projects, then runs `depth` residual blocks (depthwise causal conv, RMS norm, GELU MLP) -ggml_cgraph * clip_graph_muse_spark_audio::build() { +ggml_cgraph * mtmd_graph_muse_spark_audio::build() { const int kernel = hparams.audio_conv_kernel_size; const int n_stage = (int) hparams.audio_patch_sizes.size(); diff --git a/tools/mtmd/models/nemotron-omni.cpp b/tools/mtmd/models/nemotron-omni.cpp index c4b1bff40..fc0f2b2ce 100644 --- a/tools/mtmd/models/nemotron-omni.cpp +++ b/tools/mtmd/models/nemotron-omni.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_tensor * clip_graph_nemotron_omni::build_inp_with_temporal_merge() { +ggml_tensor * mtmd_graph_nemotron_omni::build_inp_with_temporal_merge() { if (n_batch == 1 && !img_batch->is_video) { return build_inp(); } @@ -22,7 +22,7 @@ ggml_tensor * clip_graph_nemotron_omni::build_inp_with_temporal_merge() { return ggml_cont(ctx0, ggml_transpose(ctx0, inp)); } -ggml_cgraph * clip_graph_nemotron_omni::build() { +ggml_cgraph * mtmd_graph_nemotron_omni::build() { GGML_ASSERT(model.class_embedding != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); diff --git a/tools/mtmd/models/nemotron-transcribe.cpp b/tools/mtmd/models/nemotron-transcribe.cpp index 2ce20c81f..b8fe37749 100644 --- a/tools/mtmd/models/nemotron-transcribe.cpp +++ b/tools/mtmd/models/nemotron-transcribe.cpp @@ -27,7 +27,7 @@ static constexpr float SPK_WEAK_BOOST = 2.0f; // _boost_topk_scores, scale_fac static constexpr float SPK_SPEECH_THR = 0.5f; // _disable_low_scores: a speaker's frames at or below it are not cached // the speaker expert has its own layers, width and head count, which build_vit does not take; full rotary -ggml_tensor * clip_graph_nemotron_transcribe::build_spk_encoder(ggml_tensor * cur, ggml_tensor * pos) { +ggml_tensor * mtmd_graph_nemotron_transcribe::build_spk_encoder(ggml_tensor * cur, ggml_tensor * pos) { const int n_head = hparams.spk_n_head; const int64_t n_embd = cur->ne[0]; const int64_t n_pos = cur->ne[1]; @@ -57,7 +57,7 @@ ggml_tensor * clip_graph_nemotron_transcribe::build_spk_encoder(ggml_tensor * cu return build_norm(cur, model.spk_post_ln_w, model.spk_post_ln_b, NORM_TYPE_NORMAL, eps, -1); } -ggml_cgraph * clip_graph_nemotron_transcribe::build() { +ggml_cgraph * mtmd_graph_nemotron_transcribe::build() { const int sub = hparams.subsampling_factor; const int64_t n_mel = hparams.n_mel_bins; const int64_t n_mel_frames = img.nx(); @@ -90,7 +90,7 @@ ggml_cgraph * clip_graph_nemotron_transcribe::build() { } ggml_tensor * asr_pos = ggml_view_1d(ctx0, pos, n_pos, 0); - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return ggml_rope_ext(ctx0, cur, asr_pos, nullptr, hparams.audio_rope_dims, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); }; @@ -231,10 +231,10 @@ struct spk_stream { }; // every update has a full window, which overflows the FIFO and the cache (NeMo's update period of 300 never applies) -static_assert(clip_graph_nemotron_transcribe::window_core - SPK_FIFO_LEN > SPK_CACHE_LEN); +static_assert(mtmd_graph_nemotron_transcribe::window_core - SPK_FIFO_LEN > SPK_CACHE_LEN); // NeMo SortformerModules.streaming_update: core rows join the FIFO, its overflow moves to the cache, which is compressed -static void update_spk_state(spk_stream & ss, const clip_enc_state & st, const clip_model & model, int64_t core_begin, +static void update_spk_state(spk_stream & ss, const mtmd_subctx_enc_state & st, const mtmd_model & model, int64_t core_begin, int64_t n_core) { const int64_t d = model.hparams.spk_n_embd; const int64_t n_spk = model.spk_head_fc2_w->ne[1]; @@ -257,13 +257,13 @@ static void update_spk_state(spk_stream & ss, const clip_enc_state & st, const c } // NeMo ParallelExpertEncoder._forward_online: windows with left/right context, one graph each, the speaker state carried -bool clip_graph_nemotron_transcribe::encode_windows(clip_ctx * ctx, const clip_model & model, clip_encode_params * params) { +bool mtmd_graph_nemotron_transcribe::encode_windows(mtmd_subcontext * ctx, const mtmd_model & model, mtmd_subctx_encode_params * params) { GGML_ASSERT(params->imgs->entries.size() == 1 && params->out_embd != nullptr); - const clip_image_f32 & rec = params->imgs->entries[0]; + const mtmd_image_f32 & rec = params->imgs->entries[0]; const auto & buf = rec.get_ro_buf(); const int64_t n_frames = rec.nx(); const int64_t n_rows = rec.ny(); - const int64_t n_embd = clip_n_mmproj_embd(ctx); + const int64_t n_embd = ctx->model.n_mmproj_embd(); const int sub = model.hparams.subsampling_factor; const int64_t core = (int64_t) window_core * sub; const int64_t left = (int64_t) window_left * sub; @@ -272,7 +272,7 @@ bool clip_graph_nemotron_transcribe::encode_windows(clip_ctx * ctx, const clip_m GGML_ASSERT((int64_t) buf.size() == n_frames * n_rows); auto & out = *params->out_embd; - GGML_ASSERT(out.size() == (size_t) n_embd * clip_n_output_tokens(ctx, &rec)); + GGML_ASSERT(out.size() == (size_t) n_embd * ctx->model.n_output_tokens(&rec)); // the ASR expert's input: per-feature normalization over the whole recording (NeMo normalize_batch, per_feature) std::vector mean(n_rows), denom(n_rows); @@ -292,7 +292,7 @@ bool clip_graph_nemotron_transcribe::encode_windows(clip_ctx * ctx, const clip_m } spk_stream ss; - clip_enc_state st; + mtmd_subctx_enc_state st; size_t n_out = 0; for (int64_t w = 0; w < n_win; w++) { const int64_t core_begin = w * core; @@ -310,18 +310,18 @@ bool clip_graph_nemotron_transcribe::encode_windows(clip_ctx * ctx, const clip_m st.mel_norm[r * win_len + t] = (float) ((src[t] - mean[r]) / denom[r]); } } - clip_image_f32_batch batch; + mtmd_image_f32_batch batch; batch.is_audio = true; batch.entries.emplace_back(); batch.entries[0].set_size({(int) win_len, (int) n_rows}, false, true); batch.entries[0].cpy_buf(data); - std::vector embd((size_t) n_embd * clip_n_output_tokens(ctx, &batch.entries[0])); - clip_encode_params wp = *params; + std::vector embd((size_t) n_embd * ctx->model.n_output_tokens(&batch.entries[0])); + mtmd_subctx_encode_params wp = *params; wp.imgs = &batch; wp.out_embd = &embd; wp.enc_state = &st; - if (!clip_encode(ctx, &wp)) { + if (!ctx->encode(&wp)) { return false; } diff --git a/tools/mtmd/models/nemotron-v2-vl.cpp b/tools/mtmd/models/nemotron-v2-vl.cpp index 03094be1b..6e644ea97 100644 --- a/tools/mtmd/models/nemotron-v2-vl.cpp +++ b/tools/mtmd/models/nemotron-v2-vl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_nemotron_v2_vl::build() { +ggml_cgraph * mtmd_graph_nemotron_v2_vl::build() { GGML_ASSERT(model.class_embedding != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); diff --git a/tools/mtmd/models/paddleocr.cpp b/tools/mtmd/models/paddleocr.cpp index 0e6b3f4b6..1ad186935 100644 --- a/tools/mtmd/models/paddleocr.cpp +++ b/tools/mtmd/models/paddleocr.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_paddleocr::build() { +ggml_cgraph * mtmd_graph_paddleocr::build() { GGML_ASSERT(img.nx() % (patch_size * model.hparams.n_merge) == 0); GGML_ASSERT(img.ny() % (patch_size * model.hparams.n_merge) == 0); @@ -13,7 +13,7 @@ ggml_cgraph * clip_graph_paddleocr::build() { ggml_set_name(positions, "positions"); ggml_set_input(positions); - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return ggml_rope_multi( ctx0, cur, positions, nullptr, d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, diff --git a/tools/mtmd/models/parakeet.cpp b/tools/mtmd/models/parakeet.cpp index a6d7f0739..4a2377b4c 100644 --- a/tools/mtmd/models/parakeet.cpp +++ b/tools/mtmd/models/parakeet.cpp @@ -4,7 +4,7 @@ static constexpr int PARAKEET_LOCAL_ATTN_THRESHOLD = 8192; static constexpr int PARAKEET_LOCAL_ATTN_WINDOW = 128; // conv subsampling + conformer encoder -ggml_cgraph * clip_graph_parakeet::build() { +ggml_cgraph * mtmd_graph_parakeet::build() { // Conv subsampling ggml_tensor * inp = build_inp_raw(1); diff --git a/tools/mtmd/models/pixtral.cpp b/tools/mtmd/models/pixtral.cpp index edfae0825..6d3f01de0 100644 --- a/tools/mtmd/models/pixtral.cpp +++ b/tools/mtmd/models/pixtral.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_pixtral::build() { +ggml_cgraph * mtmd_graph_pixtral::build() { const int n_merge = hparams.n_merge; // 2D input positions @@ -12,7 +12,7 @@ ggml_cgraph * clip_graph_pixtral::build() { ggml_set_name(pos_w, "pos_w"); ggml_set_input(pos_w); - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return build_rope_2d(ctx0, cur, pos_h, pos_w, hparams.rope_theta, true); }; diff --git a/tools/mtmd/models/pockettts-gen.cpp b/tools/mtmd/models/pockettts-gen.cpp index 662feff7f..c5f340656 100644 --- a/tools/mtmd/models/pockettts-gen.cpp +++ b/tools/mtmd/models/pockettts-gen.cpp @@ -9,13 +9,13 @@ // // there is no codebook anywhere, "codes" in the mtmd API are continuous features here -ggml_tensor * clip_graph_pockettts_gen::modulate(ggml_tensor * x, ggml_tensor * shift, ggml_tensor * scale) const { +ggml_tensor * mtmd_graph_pockettts_gen::modulate(ggml_tensor * x, ggml_tensor * shift, ggml_tensor * scale) const { ggml_tensor * cur = ggml_mul(ctx0, x, ggml_scale_bias(ctx0, scale, 1.0f, 1.0f)); return ggml_add(ctx0, cur, shift); } // see TimestepEmbedder in the reference -ggml_tensor * clip_graph_pockettts_gen::time_embed(const clip_flow_net::time_embd & te, float t) const { +ggml_tensor * mtmd_graph_pockettts_gen::time_embed(const mtmd_flow_net::time_embd & te, float t) const { // t is a graph-build constant, so the cos/sin table can be folded into a scaled copy ggml_tensor * args = ggml_scale(ctx0, te.freqs, t); ggml_tensor * emb = ggml_concat(ctx0, ggml_cos(ctx0, args), ggml_sin(ctx0, args), 0); @@ -42,7 +42,7 @@ ggml_tensor * clip_graph_pockettts_gen::time_embed(const clip_flow_net::time_emb } // one velocity evaluation: v(cond, s, t, x) -ggml_tensor * clip_graph_pockettts_gen::flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const { +ggml_tensor * mtmd_graph_pockettts_gen::flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const { const auto & flow = model.flow; ggml_tensor * cur = build_mm(flow.input_proj_w, x); @@ -99,7 +99,7 @@ ggml_tensor * clip_graph_pockettts_gen::flow_forward(ggml_tensor * cond, ggml_te // state carried between GEN_WAV calls: rope offset, per-layer KV window, conv left context // and the transposed-conv overlap tails -std::vector list_pockettts_state_slots(const clip_hparams & hparams, const clip_model & model) { +std::vector list_pockettts_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { std::vector slots; if (model.gen_upsample_w == nullptr) { return slots; // not a pocket-tts decoder @@ -136,8 +136,8 @@ std::vector list_pockettts_state_slots(const clip_hparams & hpar return slots; } -ggml_cgraph * clip_graph_pockettts_gen::build() { - if (gen_process == CLIP_GEN_PROCESS_GEN_CODE) { +ggml_cgraph * mtmd_graph_pockettts_gen::build() { + if (gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE) { // the backbone hidden state arrives as the single batch entry ggml_tensor * h_state = build_inp_raw(1); h_state = ggml_reshape_2d(ctx0, h_state, n_mmproj_embd, 1); @@ -188,7 +188,7 @@ ggml_cgraph * clip_graph_pockettts_gen::build() { cur = build_mm(model.gen_quant_out_w, cur); cb(cur, "quant_out", -1); - clip_graph_pockettts_seanet seanet(*this); + mtmd_graph_pockettts_seanet seanet(*this); for (const auto & slot : list_pockettts_state_slots(hparams, model)) { ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1); ggml_set_name(t, ("state_in_" + slot.name).c_str()); @@ -225,7 +225,7 @@ ggml_cgraph * clip_graph_pockettts_gen::build() { keep = ggml_mul(ctx0, keep, ggml_step(ctx0, ggml_scale_bias(ctx0, ggml_add(ctx0, pos_k, base), 1.0f, 0.5f - (float) prefix))); ggml_tensor * kq_mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), n_kv, n_pos, 1, 1); - if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { + if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) { kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F16); } diff --git a/tools/mtmd/models/pockettts-seanet.cpp b/tools/mtmd/models/pockettts-seanet.cpp index c47207f56..4480aac9d 100644 --- a/tools/mtmd/models/pockettts-seanet.cpp +++ b/tools/mtmd/models/pockettts-seanet.cpp @@ -11,7 +11,7 @@ static int64_t div_ceil(int64_t a, int64_t b) { // x: [T, IC], w: [K, IC, OC] -> [T / stride, OC] // the convs are causal, so the whole K - stride padding goes on the left -ggml_tensor * clip_graph_pockettts_seanet::conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation, +ggml_tensor * mtmd_graph_pockettts_seanet::conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation, bool pad_replicate, const std::string & state_name) const { const int64_t k_size = (w->ne[0] - 1) * dilation + 1; const int64_t p_total = k_size - stride; @@ -48,7 +48,7 @@ ggml_tensor * clip_graph_pockettts_seanet::conv1d(ggml_tensor * x, ggml_tensor * // x: [T, IC], w: [K, OC/groups, IC] -> [T * stride, OC] // the K - stride overlap tail belongs to the next call: added to its head when streaming, else dropped -ggml_tensor * clip_graph_pockettts_seanet::conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, +ggml_tensor * mtmd_graph_pockettts_seanet::conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const { const int64_t K = w->ne[0]; const int64_t T = x->ne[0]; @@ -99,7 +99,7 @@ ggml_tensor * clip_graph_pockettts_seanet::conv_transpose1d(ggml_tensor * x, ggm return out; } -ggml_tensor * clip_graph_pockettts_seanet::res_unit(ggml_tensor * x, const clip_seanet::stage & stage, int dilation, +ggml_tensor * mtmd_graph_pockettts_seanet::res_unit(ggml_tensor * x, const mtmd_seanet::stage & stage, int dilation, const std::string & state_prefix) const { ggml_tensor * h = ggml_elu(ctx0, x); h = conv1d(h, stage.res_conv1_w, stage.res_conv1_b, 1, dilation, false, state_prefix); @@ -109,7 +109,7 @@ ggml_tensor * clip_graph_pockettts_seanet::res_unit(ggml_tensor * x, const clip_ return ggml_add(ctx0, x, h); } -ggml_tensor * clip_graph_pockettts_seanet::encode(ggml_tensor * x) const { +ggml_tensor * mtmd_graph_pockettts_seanet::encode(ggml_tensor * x) const { const auto & seanet = model.seanet; ggml_tensor * cur = conv1d(x, seanet.conv_in_w, seanet.conv_in_b, 1, 1); @@ -132,7 +132,7 @@ ggml_tensor * clip_graph_pockettts_seanet::encode(ggml_tensor * x) const { return cur; } -ggml_tensor * clip_graph_pockettts_seanet::decode(ggml_tensor * x) const { +ggml_tensor * mtmd_graph_pockettts_seanet::decode(ggml_tensor * x) const { const auto & seanet = model.seanet; const bool stream = !state_in.empty(); diff --git a/tools/mtmd/models/pockettts-spkenc.cpp b/tools/mtmd/models/pockettts-spkenc.cpp index 280f2a97c..f01727ad5 100644 --- a/tools/mtmd/models/pockettts-spkenc.cpp +++ b/tools/mtmd/models/pockettts-spkenc.cpp @@ -4,7 +4,7 @@ // mimi encoder (SEANet + transformer + downsample), then flow_lm.speaker_proj_weight // pre-norm block with layer scale on both residual paths, see mimi_transformer.py -ggml_tensor * clip_graph_pockettts_spkenc::tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const { +ggml_tensor * mtmd_graph_pockettts_spkenc::tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const { ggml_tensor * inp = cur; cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il); @@ -36,12 +36,12 @@ ggml_tensor * clip_graph_pockettts_spkenc::tfm_layer_forward(ggml_tensor * cur, return cur; } -ggml_cgraph * clip_graph_pockettts_spkenc::build() { +ggml_cgraph * mtmd_graph_pockettts_spkenc::build() { // the preprocessor hands over the waveform as a single-row "mel", already [n_samples, 1] ggml_tensor * inp_raw = build_inp_raw(1); ggml_tensor * cur = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], inp_raw->ne[1]); - clip_graph_pockettts_seanet seanet(*this); + mtmd_graph_pockettts_seanet seanet(*this); cur = seanet.encode(cur); cb(cur, "mimi_enc", -1); diff --git a/tools/mtmd/models/qwen2vl.cpp b/tools/mtmd/models/qwen2vl.cpp index 7516aa23a..8ecd59036 100644 --- a/tools/mtmd/models/qwen2vl.cpp +++ b/tools/mtmd/models/qwen2vl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_qwen2vl::build() { +ggml_cgraph * mtmd_graph_qwen2vl::build() { GGML_ASSERT(model.patch_bias == nullptr); GGML_ASSERT(model.class_embedding == nullptr); diff --git a/tools/mtmd/models/qwen3a.cpp b/tools/mtmd/models/qwen3a.cpp index 4de96955d..0ed6b9a8f 100644 --- a/tools/mtmd/models/qwen3a.cpp +++ b/tools/mtmd/models/qwen3a.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_qwen3a::build() { +ggml_cgraph * mtmd_graph_qwen3a::build() { // Ref implementation: https://github.com/QwenLM/Qwen3-ASR/blob/main/qwen_asr/core/transformers_backend/modeling_qwen3_asr.py // inp_raw: [n_frames, n_mel, 1] (nx=n_frames, ny=n_mel) diff --git a/tools/mtmd/models/qwen3tts-gen.cpp b/tools/mtmd/models/qwen3tts-gen.cpp index ab619b237..59958fc4c 100644 --- a/tools/mtmd/models/qwen3tts-gen.cpp +++ b/tools/mtmd/models/qwen3tts-gen.cpp @@ -3,7 +3,7 @@ #include // on-device sampling: top-k, top-p, then a random draw -ggml_tensor * clip_graph_qwen3tts_gen::code_gen::do_sampling(ggml_tensor * logits, ggml_tensor * inp_rand) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::do_sampling(ggml_tensor * logits, ggml_tensor * inp_rand) const { logits = ggml_reshape_1d(ctx0, logits, ggml_nelements(logits)); const int64_t n_vocab = logits->ne[0]; @@ -68,7 +68,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::do_sampling(ggml_tensor * logit } // returns a new cache with row row_idx set to value -ggml_tensor * clip_graph_qwen3tts_gen::code_gen::cache_set(ggml_tensor * cache, int row_idx, ggml_tensor * value) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::cache_set(ggml_tensor * cache, int row_idx, ggml_tensor * value) const { const int64_t n_embd = cache->ne[0]; const int64_t n_cache = cache->ne[1]; GGML_ASSERT(row_idx >= 0 && row_idx < n_cache); @@ -95,7 +95,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::cache_set(ggml_tensor * cache, } // builds a const i32 with no host upload: view a tensor, zero it via scale, add value, cast to i32 -ggml_tensor * clip_graph_qwen3tts_gen::code_gen::const_i32(ggml_tensor * anchor, float value) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::const_i32(ggml_tensor * anchor, float value) const { ggml_tensor * v = ggml_view_1d(ctx0, anchor, 1, 0); if (v->type != GGML_TYPE_F32) { v = ggml_cast(ctx0, v, GGML_TYPE_F32); @@ -104,20 +104,20 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::const_i32(ggml_tensor * anchor, } // causal keep-mask row for a query at position pos, window size n_kv_pad -ggml_tensor * clip_graph_qwen3tts_gen::code_gen::causal_mask_row(int64_t n_kv_pad, int pos) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::causal_mask_row(int64_t n_kv_pad, int pos) const { ggml_tensor * ones = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv_pad, n_kv_pad), 1.0f); ggml_tensor * keep = ggml_tri(ctx0, ones, GGML_TRI_TYPE_LOWER_DIAG); ggml_tensor * row = ggml_view_1d(ctx0, keep, n_kv_pad, (size_t) pos * keep->nb[1]); ggml_tensor * mask = ggml_log(ctx0, row); // 0 = keep, -inf = masked mask = ggml_reshape_4d(ctx0, mask, n_kv_pad, 1, 1, 1); - if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { + if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) { mask = ggml_cast(ctx0, mask, GGML_TYPE_F16); } return mask; } // talker hidden size -> predictor hidden size (small_to_mtp_projection) -ggml_tensor * clip_graph_qwen3tts_gen::code_gen::project_in(ggml_tensor * cur) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::project_in(ggml_tensor * cur) const { if (!model.gen_code_proj_in_w) { return cur; } @@ -129,9 +129,9 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::project_in(ggml_tensor * cur) c } // one transformer layer at position pos; writes k/v into k_cache_layer/v_cache_layer at row pos -ggml_tensor * clip_graph_qwen3tts_gen::code_gen::layer_forward( +ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::layer_forward( ggml_tensor * cur, - const clip_layer & layer, + const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, ggml_tensor *& k_cache_layer, @@ -192,7 +192,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::layer_forward( // position 0: hidden bridge, seeds the k/v cache, no sampling // position 1: embed(code0), sample with lm_head[0], write out_code_cache[1] -void clip_graph_qwen3tts_gen::code_gen::prefill( +void mtmd_graph_qwen3tts_gen::code_gen::prefill( std::vector & k_cache, std::vector & v_cache, ggml_tensor *& out_code_cache, @@ -237,7 +237,7 @@ void clip_graph_qwen3tts_gen::code_gen::prefill( // - write new kv at cache row g+1, sample with lm_head[g] // - write result to out_code_cache[g+1] // step_idx must be in [1, n_acoustic - 1] -ggml_tensor * clip_graph_qwen3tts_gen::code_gen::step( +ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::step( std::vector & k_cache, std::vector & v_cache, ggml_tensor * out_code_cache, @@ -290,7 +290,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::step( // causal conv1d, stride 1: prepend persisted left-context instead of zero-padding, then a plain conv // x: [T, IC] (T-first). w: [K, IC, OC]. state_name empty means K == 1 (no left-context). returns [T, OC] -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation, const std::string & state_name) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::causal_conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation, const std::string & state_name) const { const int K = (int) w->ne[0]; const int pad = (K - 1) * dilation; @@ -314,7 +314,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d(ggml_tensor * x, // causal depthwise conv1d, stride 1, dilation 1, kernel from w's shape. // x: [T, C]. w: [K, 1, C]. returns [T, C]. see causal_conv1d for the state contract. -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name) const { const int K = (int) w->ne[0]; const int pad = K - 1; @@ -338,7 +338,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d_dw(ggml_tensor * // causal ConvTranspose1d, the (kernel - stride) overlap tail is kept as state for the next call // x: [T, IC], w: [K, OC, IC]. state_name empty means K == stride (no overlap). returns [T * stride, OC] -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const { const int K = (int) w->ne[0]; const int OC = (int) w->ne[1]; const int trim = K - stride; @@ -372,7 +372,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv_transpose1d(ggml_te // SnakeBeta activation: y = x + sin(alpha*x)^2 * inv_beta (alpha/inv_beta folded via exp/reciprocal at conversion time) // x: [T, C]. alpha/beta: [C], broadcasts over T -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const { ggml_tensor * a = ggml_reshape_2d(ctx0, alpha, 1, alpha->ne[0]); ggml_tensor * b = ggml_reshape_2d(ctx0, beta, 1, beta->ne[0]); @@ -388,7 +388,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::snake(ggml_tensor * x, ggml_ten // RVQ codebook decode: T frames of 16 codes -> 512-dim hidden (C-first, [512, T]) // codebook 0 (semantic) and 1..15 (acoustic) sum within their group, project separately, then add -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_codes) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_codes) const { const auto & c2w = model.c2w; const int64_t T = inp_codes->ne[0]; @@ -418,7 +418,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_ // one pre_transformer layer over a batch of N = sliding_window new frames // attention runs over [(W-1)-frame prefix from the last batch] + [N new frames] // RoPE positions come from a persisted counter, so phases line up across batches -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, int il) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il) const { const int n_head = hparams.wav_tfm_n_head; const int n_head_kv = hparams.wav_tfm_n_head_kv; const int64_t d_head = layer.q_w->ne[1] / n_head; @@ -485,7 +485,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * keep = ggml_mul(ctx0, keep, warm); ggml_tensor * mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), total_kv, N, 1, 1); // 0 = keep, -inf = masked - if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { + if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) { mask = ggml_cast(ctx0, mask, GGML_TYPE_F16); } @@ -515,7 +515,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * // dwconv -> LayerNorm -> pwconv1 -> GELU -> pwconv2 -> layer scale -> residual // x: [T, C] T-first; LayerNorm/pwconv need C on ne0, so this transposes in and back out -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::convnext_block(ggml_tensor * x, const clip_code2wav::upsample_block & blk, const std::string & state_prefix) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::convnext_block(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk, const std::string & state_prefix) const { ggml_tensor * residual = x; ggml_tensor * h = causal_conv1d_dw(x, blk.dwconv_w, blk.dwconv_b, state_prefix + "_dwconv"); // [T, C] @@ -538,7 +538,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::convnext_block(ggml_tensor * x, // SnakeBeta -> dilated causal conv (k=7) -> SnakeBeta -> pointwise causal conv (k=1) -> residual. // x: [T, C]. returns [T, C]. -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::dac_res_unit(ggml_tensor * x, const clip_code2wav::dac_res & res, int dilation, const std::string & state_name) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::dac_res_unit(ggml_tensor * x, const mtmd_code2wav::dac_res & res, int dilation, const std::string & state_name) const { ggml_tensor * residual = x; ggml_tensor * h = snake(x, res.act1_alpha, res.act1_beta); h = causal_conv1d(h, res.conv1_w, res.conv1_b, dilation, state_name); @@ -548,7 +548,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::dac_res_unit(ggml_tensor * x, c } // RVQ codes -> raw PCM for a batch of N = sliding_window frames -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes) const { const auto & c2w = model.c2w; // 1. quantizer decode: N frames of 16 codes -> [512, N] (C-first) @@ -613,7 +613,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes) // code2wav's persisted state buffers: RoPE position counter, K/V per pre_transformer layer, // left-context/tail per stateful conv. shape lookup only, no graph needed -std::vector list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model) { +std::vector list_c2w_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { const auto & c2w = model.c2w; std::vector slots; @@ -663,20 +663,20 @@ std::vector list_c2w_state_slots(const clip_hparams & hparams, c // both sub-graphs are always built, so the topology stays constant // ggml_build_forward_select() then picks the one that actually runs -ggml_cgraph * clip_graph_qwen3tts_gen::build() { +ggml_cgraph * mtmd_graph_qwen3tts_gen::build() { GGML_ASSERT(n_batch == 1); // this module only ever processes one frame at a time int idx; switch (gen_process) { - case CLIP_GEN_PROCESS_GEN_CODE: idx = 0; break; - case CLIP_GEN_PROCESS_GEN_WAV: idx = 1; break; + case MTMD_SUBCTX_GEN_PROCESS_GEN_CODE: idx = 0; break; + case MTMD_SUBCTX_GEN_PROCESS_GEN_WAV: idx = 1; break; default: GGML_ABORT("unknown gen_process"); } - // ---- CLIP_GEN_PROCESS_GEN_CODE: backbone hidden state -> 16 RVQ codes + next-step embd ---- + // ---- MTMD_SUBCTX_GEN_PROCESS_GEN_CODE: backbone hidden state -> 16 RVQ codes + next-step embd ---- // not build_inp_raw(), a GEN_WAV call's `img` has no hidden-state data ggml_tensor * h_state = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd); - ggml_set_name(h_state, "inp_raw"); // must keep this exact name, clip_encode() sets it by name + ggml_set_name(h_state, "inp_raw"); // must keep this exact name, mtmd_subcontext::encode() sets it by name ggml_set_input(h_state); ggml_tensor * code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); @@ -740,7 +740,7 @@ ggml_cgraph * clip_graph_qwen3tts_gen::build() { out_embd = ggml_reshape_2d(ctx0, out_embd, out_embd->ne[0], 1); cb(out_embd, "gen_audio_out", -1); - // ---- CLIP_GEN_PROCESS_GEN_WAV: 16 RVQ codes -> raw PCM ---- + // ---- MTMD_SUBCTX_GEN_PROCESS_GEN_WAV: 16 RVQ codes -> raw PCM ---- const int n_frames = hparams.wav_tfm_swa; // frames per batch, == the attention window ggml_tensor * inp_codes = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_frames, n_codes); @@ -764,7 +764,7 @@ ggml_cgraph * clip_graph_qwen3tts_gen::build() { ggml_set_output(slot.second); } - // out_embd goes last, clip_encode() reads it back via ggml_graph_node(gf, -1) + // out_embd goes last, mtmd_subcontext::encode() reads it back via ggml_graph_node(gf, -1) ggml_tensor * outs[2]; outs[0] = out_codes; outs[1] = out_audio; ggml_build_forward_select(gf, outs, 2, idx); diff --git a/tools/mtmd/models/qwen3tts-spkenc.cpp b/tools/mtmd/models/qwen3tts-spkenc.cpp index ac6cafd9a..1717eac6e 100644 --- a/tools/mtmd/models/qwen3tts-spkenc.cpp +++ b/tools/mtmd/models/qwen3tts-spkenc.cpp @@ -5,7 +5,7 @@ static constexpr int SPK_DILATIONS[3] = { 2, 3, 4 }; // enc_dilations[1..3] // conv1d, kernel K, padding "same" (reflect), dilation d // x: [C, T] (ne[0]=C, ne[1]=T) -> [out_c, T] -ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const { +ggml_tensor * mtmd_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const { const int K = (int) w->ne[0]; const int IC = (int) w->ne[1]; const int OC = (int) w->ne[2]; @@ -36,7 +36,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tens // Res2Net: split channel axis into `scale` chunks, chain dilated conv1d branches // x: [C, T] -> [C, T] -ggml_tensor * clip_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const { +ggml_tensor * mtmd_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const { const int64_t C = x->ne[0]; const int64_t T = x->ne[1]; const int64_t Cs = C / scale; @@ -70,7 +70,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const clip_la } // squeeze-and-excitation gate. x: [C, T] -> [C, T] -ggml_tensor * clip_graph_qwen3tts_spkenc::se_block(ggml_tensor * x, const clip_layer & layer) const { +ggml_tensor * mtmd_graph_qwen3tts_spkenc::se_block(ggml_tensor * x, const mtmd_layer & layer) const { // temporal mean, keepdim: transpose so T is on ne[0], reduce, transpose back ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, C] ggml_tensor * mean = ggml_mean(ctx0, x_t); // [1, C] @@ -85,7 +85,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::se_block(ggml_tensor * x, const clip_l } // tdnn1 -> res2net -> tdnn2 -> se, plus residual. x: [C, T] -> [C, T] -ggml_tensor * clip_graph_qwen3tts_spkenc::se_res2net_block(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const { +ggml_tensor * mtmd_graph_qwen3tts_spkenc::se_res2net_block(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const { ggml_tensor * residual = x; ggml_tensor * h = conv1d_same(x, layer.conv_pw1_w, layer.conv_pw1_b, 1); // tdnn1 h = ggml_relu(ctx0, h); @@ -97,7 +97,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::se_res2net_block(ggml_tensor * x, cons } // attentive statistics pooling. x: [C, T] -> [2*C, 1] -ggml_tensor * clip_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x) const { +ggml_tensor * mtmd_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x) const { const int64_t T = x->ne[1]; // mean over T: [C, 1] @@ -151,7 +151,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x) return ggml_concat(ctx0, w_mean, w_std, 0); // [2C, 1] } -ggml_cgraph * clip_graph_qwen3tts_spkenc::build() { +ggml_cgraph * mtmd_graph_qwen3tts_spkenc::build() { // inp_raw: [T, n_mel, 1, 1], from mtmd_audio_preprocessor_qwen3tts_spk ggml_tensor * inp = build_inp_raw(1); inp = ggml_reshape_2d(ctx0, inp, inp->ne[0], inp->ne[1]); diff --git a/tools/mtmd/models/qwen3vl.cpp b/tools/mtmd/models/qwen3vl.cpp index 2e49dcd01..ada7a129d 100644 --- a/tools/mtmd/models/qwen3vl.cpp +++ b/tools/mtmd/models/qwen3vl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_qwen3vl::build() { +ggml_cgraph * mtmd_graph_qwen3vl::build() { GGML_ASSERT(model.patch_bias != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); GGML_ASSERT(model.class_embedding == nullptr); diff --git a/tools/mtmd/models/sensenova-u1.cpp b/tools/mtmd/models/sensenova-u1.cpp index 57875c6ed..8e685d74c 100644 --- a/tools/mtmd/models/sensenova-u1.cpp +++ b/tools/mtmd/models/sensenova-u1.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_sensenova_u1::build() { +ggml_cgraph * mtmd_graph_sensenova_u1::build() { GGML_ASSERT(model.patch_embeddings_0 != nullptr); GGML_ASSERT(model.patch_bias != nullptr); GGML_ASSERT(model.mm_0_w != nullptr); diff --git a/tools/mtmd/models/siglip.cpp b/tools/mtmd/models/siglip.cpp index 8cc901a28..84387f86d 100644 --- a/tools/mtmd/models/siglip.cpp +++ b/tools/mtmd/models/siglip.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_siglip::build() { +ggml_cgraph * mtmd_graph_siglip::build() { ggml_tensor * inp = build_inp(); ggml_tensor * learned_pos_embd = model.position_embeddings; diff --git a/tools/mtmd/models/soprano.cpp b/tools/mtmd/models/soprano.cpp index bd16c6912..43f34f86f 100644 --- a/tools/mtmd/models/soprano.cpp +++ b/tools/mtmd/models/soprano.cpp @@ -2,7 +2,7 @@ #include "../mtmd-audio.h" // Soprano Vocos: normalized Qwen3 hidden states -> STFT coefficients. -ggml_cgraph * clip_graph_soprano::build() { +ggml_cgraph * mtmd_graph_soprano::build() { const auto & v = model.vocos; const int n_inp = hparams.projection_dim; const int n_up = upscale * (n_frames - 1) + 1; @@ -59,7 +59,7 @@ ggml_cgraph * clip_graph_soprano::build() { return gf; } -void clip_graph_soprano::decode_spectrum(const clip_model & model, const std::vector & spectrum, std::vector & pcm) { +void mtmd_graph_soprano::decode_spectrum(const mtmd_model & model, const std::vector & spectrum, std::vector & pcm) { const int n_frame_vals = model.vocos.output_w->ne[1]; const int n_fft = n_frame_vals - 2; const int hop_length = model.hparams.audio_hop_len; diff --git a/tools/mtmd/models/step3vl.cpp b/tools/mtmd/models/step3vl.cpp index 5142b0bba..db0245a89 100644 --- a/tools/mtmd/models/step3vl.cpp +++ b/tools/mtmd/models/step3vl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_step3vl::build() { +ggml_cgraph * mtmd_graph_step3vl::build() { GGML_ASSERT(model.class_embedding == nullptr); GGML_ASSERT(model.patch_embeddings_0 != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); @@ -18,7 +18,7 @@ ggml_cgraph * clip_graph_step3vl::build() { ggml_tensor * inp = build_inp(); ggml_tensor * learned_pos_embd = resize_position_embeddings(); - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false); }; diff --git a/tools/mtmd/models/voxtral-rt.cpp b/tools/mtmd/models/voxtral-rt.cpp index 4a01512d2..3f768d6a2 100644 --- a/tools/mtmd/models/voxtral-rt.cpp +++ b/tools/mtmd/models/voxtral-rt.cpp @@ -2,7 +2,7 @@ // the input holds 2 context mel frames followed by the new frames, so the causal conv stem // sees the same left context as on the full spectrogram -ggml_cgraph * clip_graph_voxtral_rt::build() { +ggml_cgraph * mtmd_graph_voxtral_rt::build() { const int n_frames_in = img.nx(); const int n_new = (n_frames_in - 2) / 2; // encoder frames, the conv stem has stride 2 const int n_window = hparams.attn_window_size; diff --git a/tools/mtmd/models/whisper-enc.cpp b/tools/mtmd/models/whisper-enc.cpp index 49d5dd5ad..6225b707c 100644 --- a/tools/mtmd/models/whisper-enc.cpp +++ b/tools/mtmd/models/whisper-enc.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_whisper_enc::build() { +ggml_cgraph * mtmd_graph_whisper_enc::build() { const int n_frames = img.nx(); const int n_pos = n_frames / 2; GGML_ASSERT(model.position_embeddings->ne[1] >= n_pos); diff --git a/tools/mtmd/models/yasa2.cpp b/tools/mtmd/models/yasa2.cpp index e8cd3dacb..07fa84e06 100644 --- a/tools/mtmd/models/yasa2.cpp +++ b/tools/mtmd/models/yasa2.cpp @@ -25,7 +25,7 @@ static ggml_tensor * mul_channel_weight( return ggml_mul(ctx0, x_whcb, w4); } -ggml_tensor * clip_graph_yasa2::layer_norm_channels(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b, float eps) { +ggml_tensor * mtmd_graph_yasa2::layer_norm_channels(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b, float eps) { // Match HF ConvNextLayerNorm(channels_first): // u = mean_c(x), s = mean_c((x-u)^2), x = (x-u)/sqrt(s+eps) // cast back to input dtype before affine. @@ -48,7 +48,7 @@ ggml_tensor * clip_graph_yasa2::layer_norm_channels(ggml_tensor * inp, ggml_tens return xhat; } -ggml_tensor * clip_graph_yasa2::convnext_grn(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b) { +ggml_tensor * mtmd_graph_yasa2::convnext_grn(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b) { // Exact ConvNeXtV2 GRN: // Gx = ||x||_2 over spatial dims (W,H), Nx = Gx / (mean_c(Gx) + eps) // y = w * (x * Nx) + b + x @@ -78,7 +78,7 @@ ggml_tensor * clip_graph_yasa2::convnext_grn(ggml_tensor * inp, ggml_tensor * w, return ggml_add(ctx0, inp, xnx); } -ggml_cgraph * clip_graph_yasa2::build() { +ggml_cgraph * mtmd_graph_yasa2::build() { ggml_tensor * cur = build_inp_raw(); // Patch embedding Conv2d(kernel=4, stride=4) diff --git a/tools/mtmd/models/youtuvl.cpp b/tools/mtmd/models/youtuvl.cpp index cda456c66..8cc714358 100644 --- a/tools/mtmd/models/youtuvl.cpp +++ b/tools/mtmd/models/youtuvl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_youtuvl::build() { +ggml_cgraph * mtmd_graph_youtuvl::build() { GGML_ASSERT(model.class_embedding == nullptr); const int batch_size = 1; const bool use_window_attn = !hparams.wa_layer_indexes.empty(); diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index 60087b16d..fcf94445a 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -1,4 +1,5 @@ #include "mtmd-audio.h" +#include "mtmd-subcontext.h" #include #include @@ -269,12 +270,12 @@ static void ifft(const mtmd_audio_cache & cache, float * in, int N, float * out) fft_impl(cache, in, N, out); } -static void mtmd_audio_fill_entries(std::vector & mels, bool shape_only, std::vector & entries) { +static void mtmd_audio_fill_entries(std::vector & mels, bool shape_only, std::vector & entries) { entries.clear(); for (auto & mel : mels) { GGML_ASSERT(mel.n_len <= INT32_MAX && mel.n_len >= 0); GGML_ASSERT(mel.n_mel <= INT32_MAX && mel.n_mel >= 0); - clip_image_f32 entry; + mtmd_image_f32 entry; entry.set_size({(int) mel.n_len, (int) mel.n_mel}, shape_only || mel.data.empty(), /* is_audio */ true); if (!entry.is_placeholder()) { entry.cpy_buf(mel.data); @@ -430,7 +431,7 @@ static bool log_mel_spectrogram( } out.data.resize(total_size); - if (!shape_only && !clip_audio_log_mel(params, samples_padded.data(), hann, out.n_len, out.data.data())) { + if (!shape_only && !mtmd_audio_log_mel(params, samples_padded.data(), hann, out.n_len, out.data.data())) { return false; } @@ -547,7 +548,7 @@ bool mtmd_audio_preprocessor_whisper::process(const float * samp return false; } - // because the cgraph in clip.cpp only accepts 3000 frames each, we need to split the mel + // because the cgraph only accepts 3000 frames each, we need to split the mel // we always expect the mel to have 3000 silent frames at the end if (DEBUG) { printf("output: n_mel = %d, n_len = %d\n", (int) out_full.n_mel, (int) out_full.n_len); @@ -2002,10 +2003,10 @@ bool mtmd_audio_preprocessor_pockettts::process(const float * sa } // the mimi transformer mask is dense, so cost is quadratic in the reference length - const int64_t max_samples = (int64_t) clip_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate; + const int64_t max_samples = (int64_t) mtmd_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate; if ((int64_t) n_samples > max_samples) { LOG_WRN("%s: speaker reference is %.1f s, truncating to the first %d s\n", __func__, - (double) n_samples / hparams.audio_sample_rate, clip_hparams::pockettts_max_spk_seconds); + (double) n_samples / hparams.audio_sample_rate, mtmd_hparams::pockettts_max_spk_seconds); n_samples = (size_t) max_samples; } diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index c0751b5e6..ba71913e0 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -2,8 +2,7 @@ #include "ggml.h" #include "ggml-backend.h" -#include "clip-model.h" -#include "clip-impl.h" +#include "mtmd-model.h" #include #include @@ -77,7 +76,7 @@ struct mtmd_audio_mel_config { }; struct mtmd_audio_preproc_out { - std::vector entries; // one per chunk, empty on failure + std::vector entries; // one per chunk, empty on failure std::vector samples; // input of compute(), empty if entries already hold the data bool needs_compute() const { @@ -86,13 +85,12 @@ struct mtmd_audio_preproc_out { }; struct mtmd_audio_preprocessor { - const clip_hparams & hparams; + const mtmd_hparams & hparams; mtmd_audio_mel_config mel_config; // filled by initialize() of the preprocessors that use the log-mel graph - mtmd_audio_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) { - mel_config.sched = clip_get_sched(ctx); - } - mtmd_audio_preprocessor(const clip_hparams & hparams, ggml_backend_sched_t sched): hparams(hparams) { + // mel_config.sched must be set before the first compute(), see mtmd_context::init_audio() + mtmd_audio_preprocessor(const mtmd_model * model): hparams(model->hparams) {} + mtmd_audio_preprocessor(const mtmd_hparams & hparams, ggml_backend_sched_t sched): hparams(hparams) { mel_config.sched = sched; } @@ -115,7 +113,7 @@ struct mtmd_audio_preprocessor { }; struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_whisper(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_whisper(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; @@ -124,14 +122,14 @@ struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor { }; struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_muse_spark_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_muse_spark_audio(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override {} bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; }; struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} - mtmd_audio_preprocessor_conformer(const clip_hparams & hparams, ggml_backend_sched_t sched) : mtmd_audio_preprocessor(hparams, sched) {} + mtmd_audio_preprocessor_conformer(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} + mtmd_audio_preprocessor_conformer(const mtmd_hparams & hparams, ggml_backend_sched_t sched) : mtmd_audio_preprocessor(hparams, sched) {} void initialize() override; bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; @@ -146,7 +144,7 @@ struct mtmd_audio_preprocessor_d1omni : mtmd_audio_preprocessor_conformer { }; struct mtmd_audio_preprocessor_lfm3a : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_lfm3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_lfm3a(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; @@ -155,7 +153,7 @@ struct mtmd_audio_preprocessor_lfm3a : mtmd_audio_preprocessor { }; struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_granite_speech(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_granite_speech(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; @@ -164,7 +162,7 @@ struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor { }; struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_gemma4a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_gemma4a(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; @@ -173,13 +171,13 @@ struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor { }; struct mtmd_audio_preprocessor_gemma4ua : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_gemma4ua(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_gemma4ua(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; }; struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_qwen3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_qwen3a(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; @@ -188,7 +186,7 @@ struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor { }; struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_dots3note(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_dots3note(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; @@ -197,7 +195,7 @@ struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor { }; struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_mimo_audio(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; @@ -206,7 +204,7 @@ struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor { }; struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_qwen3tts_spk(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_qwen3tts_spk(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; @@ -216,7 +214,7 @@ struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor { // mimi convolves the waveform directly, so this only pads it to a whole number of frames struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_pockettts(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_pockettts(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override {} bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; }; @@ -229,7 +227,7 @@ struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor { // 4: voice encoder power mel [n_frames x 40] (frame major, trimmed, partial grid) // turbo loudness-normalizes the clip to -27 LUFS and caps the t3 clip at 15 s instead of 6 s struct mtmd_audio_preprocessor_chatterbox_ref : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_chatterbox_ref(const clip_ctx * ctx, bool is_mtl) : mtmd_audio_preprocessor(ctx), is_mtl(is_mtl) {} + mtmd_audio_preprocessor_chatterbox_ref(const mtmd_model * model, bool is_mtl) : mtmd_audio_preprocessor(model), is_mtl(is_mtl) {} void initialize() override; bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; @@ -242,7 +240,7 @@ struct mtmd_audio_preprocessor_chatterbox_ref : mtmd_audio_preprocessor { }; struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_parakeet(clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) { } + mtmd_audio_preprocessor_parakeet(const mtmd_model * model) : mtmd_audio_preprocessor(model) { } void initialize() override; bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; @@ -255,7 +253,7 @@ struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor { // one chunk per recording: the raw log-mel of its valid frames struct mtmd_audio_preprocessor_nemotron_transcribe : mtmd_audio_preprocessor_parakeet { - mtmd_audio_preprocessor_nemotron_transcribe(clip_ctx * ctx) : mtmd_audio_preprocessor_parakeet(ctx) { } + mtmd_audio_preprocessor_nemotron_transcribe(const mtmd_model * model) : mtmd_audio_preprocessor_parakeet(model) { } bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; }; diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp index ba156f1c5..4534527fc 100644 --- a/tools/mtmd/mtmd-cli.cpp +++ b/tools/mtmd/mtmd-cli.cpp @@ -153,7 +153,7 @@ struct mtmd_cli_context { } void init_vision_context(common_params & params) { - const char * clip_path = params.mmproj.path.c_str(); + const char * mmproj_path = params.mmproj.path.c_str(); mtmd_context_params mparams = mtmd_context_params_default(); mparams.use_gpu = params.mmproj_use_gpu; mparams.device = params.mmproj_device; @@ -166,7 +166,7 @@ struct mtmd_cli_context { { // non-causal models need the whole image in one ubatch const int n_ubatch = llama_n_ubatch(lctx); - auto mem = mtmd_get_memory_usage(clip_path, mparams); + auto mem = mtmd_get_memory_usage(mmproj_path, mparams); if (mem.use_non_causal && mem.image_max_tokens > n_ubatch) { LOG_WRN("%s: cap image_max_tokens (original=%d) to n_ubatch (%d) because model needs non-causal attention on image\n", __func__, mem.image_max_tokens, n_ubatch); LOG_WRN("%s: increase n_ubatch (-ub) to increase vision token budget\n", __func__); @@ -178,9 +178,9 @@ struct mtmd_cli_context { mparams.cb_eval_user_data = &cb_data; mparams.cb_eval = common_debug_cb_eval; } - ctx_vision.reset(mtmd_init_from_file(clip_path, model, mparams)); + ctx_vision.reset(mtmd_init_from_file(mmproj_path, model, mparams)); if (!ctx_vision.get()) { - LOG_ERR("Failed to load vision model from %s\n", clip_path); + LOG_ERR("Failed to load vision model from %s\n", mmproj_path); exit(1); } diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp new file mode 100644 index 000000000..3242bddbb --- /dev/null +++ b/tools/mtmd/mtmd-graph.cpp @@ -0,0 +1,1064 @@ +#include "mtmd-graph.h" +#include "models/models.h" + +#include "ggml.h" +#include "ggml-cpp.h" +#include "ggml-backend.h" + +#include +#include +#include +#include +#include + +// +// mtmd_graph +// + +mtmd_graph::mtmd_graph(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : + model(ctx->model), + hparams(model.hparams), + proj_type(ctx->model.proj_type), + img(img), + patch_size(hparams.patch_size), + n_patches_x(img.nx() / patch_size), + n_patches_y(img.ny() / patch_size), + n_patches(n_patches_x * n_patches_y), + n_embd(hparams.n_embd), + n_head(hparams.n_head), + n_head_kv(hparams.n_head_kv), + d_head(hparams.n_embd_head > 0 ? hparams.n_embd_head : (n_head > 0 ? n_embd / n_head : 0)), + n_layer(hparams.n_layer), + n_mmproj_embd(ctx->model.n_mmproj_embd()), + eps(hparams.eps), + kq_scale(d_head > 0 ? 1.0f / sqrtf((float)d_head) : 0.0f), + flash_attn_type(ctx->flash_attn_type) { + struct ggml_init_params params = { + /*.mem_size =*/ ctx->buf_compute_meta.size(), + /*.mem_buffer =*/ ctx->buf_compute_meta.data(), + /*.no_alloc =*/ true, + }; + ctx0_ptr.reset(ggml_init(params)); + ctx0 = ctx0_ptr.get(); + gf = ggml_new_graph_custom(ctx0, ctx->max_nodes, false); +} + +mtmd_graph::mtmd_graph(const mtmd_graph & parent) : + model(parent.model), + hparams(parent.hparams), + proj_type(parent.proj_type), + img(parent.img), + patch_size(parent.patch_size), + n_patches_x(parent.n_patches_x), + n_patches_y(parent.n_patches_y), + n_patches(parent.n_patches), + n_embd(parent.n_embd), + n_head(parent.n_head), + n_head_kv(parent.n_head_kv), + d_head(parent.d_head), + n_layer(parent.n_layer), + n_mmproj_embd(parent.n_mmproj_embd), + eps(parent.eps), + kq_scale(parent.kq_scale), + flash_attn_type(parent.flash_attn_type) { + // reuse from parent + ctx0 = parent.ctx0; + gf = parent.gf; +} + +ggml_tensor * mtmd_graph::build_mm(ggml_tensor * w, ggml_tensor * x) const { + return ggml_mul_mat(ctx0, w, x); +} + +void mtmd_graph::cb(ggml_tensor * cur, const char * name, int il) const { + if (il >= 0) { + ggml_format_name(cur, "%s-%d", name, il); + } else { + ggml_set_name(cur, name); + } +} + +// siglip2 naflex +ggml_tensor * mtmd_graph::resize_position_embeddings(uint32_t interpolation_mode) { + ggml_tensor * pos_embd = model.position_embeddings; + const int height = img.ny() / patch_size; + const int width = img.nx() / patch_size; + const uint32_t mode = interpolation_mode; + const int n_per_side = (int)std::sqrt(pos_embd->ne[1]); + + GGML_ASSERT(pos_embd); + + if (height == n_per_side && width == n_per_side) { + return pos_embd; + } + + pos_embd = ggml_reshape_3d(ctx0, pos_embd, n_embd, n_per_side, n_per_side); // -> (n_embd, n_per_side, n_per_side) + pos_embd = ggml_permute(ctx0, pos_embd, 2, 0, 1, 3); // -> (n_per_side, n_per_side, n_embd) + pos_embd = ggml_interpolate(ctx0, pos_embd, width, height, n_embd, 1, mode); // -> (width, height, n_embd) + pos_embd = ggml_permute(ctx0, pos_embd, 1, 2, 0, 3); // -> (n_embd, width, height) + pos_embd = ggml_cont_2d(ctx0, pos_embd, n_embd, width * height); // -> (n_embd, width * height) + + return pos_embd; +} + +// build vision transformer (ViT) cgraph +// this function should cover most of the models +// if your model has specific features, you should probably duplicate this function +ggml_tensor * mtmd_graph::build_vit( + ggml_tensor * inp, + int64_t n_pos, + norm_type norm_t, + ffn_op_type ffn_t, + ggml_tensor * learned_pos_embd, + std::function add_pos, + const build_vit_opts & opts + ) { + // batch dim: inp is [n_embd, n_pos, B] + const int64_t B = inp->ne[2]; + + if (learned_pos_embd) { + inp = ggml_add(ctx0, inp, learned_pos_embd); + cb(inp, "pos_embed", -1); + } + + // flatten batch; unflatten again in attention + inp = ggml_reshape_2d(ctx0, inp, n_embd, n_pos * B); + + ggml_tensor * inpL = inp; + + // pre-layernorm + if (model.pre_ln_w) { + inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, norm_t, eps, -1); + cb(inpL, "pre_ln", -1); + } + + // loop over layers + for (int il = 0; il < n_layer; il++) { + auto & layer = model.layers[il]; + ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states + + ggml_tensor * attn_mask = opts.attn_mask; + if (opts.attn_mask_layers.size() > (size_t) il) { + attn_mask = opts.attn_mask_layers[il]; + } + + // layernorm1 + cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il); + cb(cur, "layer_inp_normed", il); + + // self-attention + { + ggml_tensor * Qcur = nullptr; + ggml_tensor * Kcur = nullptr; + ggml_tensor * Vcur = nullptr; + if (layer.qkv_w != nullptr) { + // fused qkv + cur = build_mm(layer.qkv_w, cur); + if (layer.qkv_b != nullptr) { + cur = ggml_add(ctx0, cur, layer.qkv_b); + } + + // Q/K/V as [d_head, n_head, n_pos, B], the batch stride is cur->nb[1]*n_pos. + Qcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B, + /* nb1 */ ggml_row_size(cur->type, d_head), + /* nb2 */ cur->nb[1], + /* nb3 */ cur->nb[1] * n_pos, + /* offset */ 0); + + Kcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B, + /* nb1 */ ggml_row_size(cur->type, d_head), + /* nb2 */ cur->nb[1], + /* nb3 */ cur->nb[1] * n_pos, + /* offset */ ggml_row_size(cur->type, n_head * d_head)); + + Vcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B, + /* nb1 */ ggml_row_size(cur->type, d_head), + /* nb2 */ cur->nb[1], + /* nb3 */ cur->nb[1] * n_pos, + /* offset */ ggml_row_size(cur->type, 2 * n_head * d_head)); + + if (layer.q_norm) { + GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]); + Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il); + cb(Qcur, "Qcur_norm", il); + } + + if (layer.k_norm) { + GGML_ASSERT(layer.k_norm->ne[0] == Kcur->ne[0]); + Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il); + cb(Kcur, "Kcur_norm", il); + } + + } else { + // separate q, k, v + Qcur = build_mm(layer.q_w, cur); + if (layer.q_b) { + Qcur = ggml_add(ctx0, Qcur, layer.q_b); + } + + Kcur = build_mm(layer.k_w, cur); + if (layer.k_b) { + Kcur = ggml_add(ctx0, Kcur, layer.k_b); + } + + Vcur = build_mm(layer.v_w, cur); + if (layer.v_b) { + Vcur = ggml_add(ctx0, Vcur, layer.v_b); + } + + // if true, norm must be applied after reshaping to (d_head, n_head, n_pos) + bool norm_per_head = layer.q_norm && layer.q_norm->ne[0] == d_head; + + if (!norm_per_head) { + if (layer.q_norm) { + Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il); + cb(Qcur, "Qcur_norm", il); + } + if (layer.k_norm) { + Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il); + cb(Kcur, "Kcur_norm", il); + } + } + + Qcur = ggml_reshape_4d(ctx0, Qcur, d_head, n_head, n_pos, B); + Kcur = ggml_reshape_4d(ctx0, Kcur, d_head, n_head_kv, n_pos, B); + Vcur = ggml_reshape_4d(ctx0, Vcur, d_head, n_head_kv, n_pos, B); + + if (norm_per_head) { + if (layer.q_norm) { + Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il); + cb(Qcur, "Qcur_norm_per_head", il); + } + if (layer.k_norm) { + Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il); + cb(Kcur, "Kcur_norm_per_head", il); + } + } + } + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + if (add_pos) { + Qcur = add_pos(Qcur, layer); + Kcur = add_pos(Kcur, layer); + cb(Qcur, "Qcur_pos", il); + cb(Kcur, "Kcur_pos", il); + } + + if (proj_type == PROJECTOR_TYPE_GEMMA4V) { + Vcur = ggml_rms_norm(ctx0, Vcur, eps); + cb(Vcur, "Vcur_normed", il); + } + + // build_attn returns a flat 2D [n_embd, n_pos*B] + cur = build_attn(layer.o_w, layer.o_b, + Qcur, Kcur, Vcur, attn_mask, kq_scale, il); + cb(cur, "attn_out", il); + } + + if (layer.ls_1_w) { + cur = ggml_mul(ctx0, cur, layer.ls_1_w); + cb(cur, "attn_out_scaled", il); + } + + if (layer.attn_post_norm_w) { + cur = build_norm(cur, layer.attn_post_norm_w, nullptr, norm_t, eps, il); + cb(cur, "attn_post_normed", il); + } + + // re-add the layer input, e.g., residual + cur = ggml_add(ctx0, cur, inpL); + + inpL = cur; // inpL = residual, cur = hidden_states + + if (opts.callback_layer_out) { + opts.callback_layer_out(cur, il); + } + + cb(cur, "ffn_inp", il); + + // layernorm2 (pre-ffn norm) + cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il); + cb(cur, "ffn_inp_normed", il); + + // ffn + cur = layer.ff_gate_exps_w + ? build_moe_ffn(cur, layer, ffn_t, il) + : build_ffn(cur, + layer.ff_up_w, layer.ff_up_b, + layer.ff_gate_w, layer.ff_gate_b, + layer.ff_down_w, layer.ff_down_b, + ffn_t, il); + + cb(cur, "ffn_out", il); + + if (layer.ff_post_norm_w) { + cur = build_norm(cur, layer.ff_post_norm_w, nullptr, norm_t, eps, il); + cb(cur, "ffn_post_normed", il); + } + + if (layer.ls_2_w) { + cur = ggml_mul(ctx0, cur, layer.ls_2_w); + cb(cur, "ffn_out_scaled", il); + } + + // residual 2 + cur = ggml_add(ctx0, inpL, cur); + cb(cur, "layer_out", il); + + if (layer.ls_out_w) { + cur = ggml_mul(ctx0, cur, layer.ls_out_w); + cb(cur, "layer_out_scaled", il); + } + + inpL = cur; + } + + if (model.audio_has_avgpool()) { + ggml_tensor * cur = inpL; + cur = ggml_transpose(ctx0, cur); + cur = ggml_cont(ctx0, cur); + cur = ggml_pool_1d(ctx0, cur, GGML_OP_POOL_AVG, 2, 2, 0); + cur = ggml_transpose(ctx0, cur); + cur = ggml_cont(ctx0, cur); + inpL = cur; + } + + // post-layernorm + if (model.post_ln_w && !opts.skip_post_ln) { + inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, norm_t, eps, -1); + } + + // restore the batch dim + GGML_ASSERT(inpL->ne[1] % B == 0); + inpL = ggml_reshape_3d(ctx0, inpL, n_embd, inpL->ne[1] / B, B); + return inpL; +} + +// build the input after conv2d (inp_raw --> patches) +// returns tensor with shape [n_embd, n_patches] +ggml_tensor * mtmd_graph::build_inp() { + ggml_tensor * inp_raw = build_inp_raw(); + ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1); + inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, n_batch); + inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); + if (model.patch_bias) { + inp = ggml_add(ctx0, inp, model.patch_bias); + cb(inp, "patch_bias", -1); + } + return inp; +} + +ggml_tensor * mtmd_graph::build_inp_raw(int channels) { + ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, img.nx(), img.ny(), channels, n_batch); + ggml_set_name(inp_raw, "inp_raw"); + ggml_set_input(inp_raw); + return inp_raw; +} + +// qwen-vl style temporal merge: Conv3D(kt=2) split into 2 Conv2D +ggml_tensor * mtmd_graph::build_inp_temporal_merge() { + ggml_tensor * inp_raw = build_inp_raw(); + + GGML_ASSERT(img.nx() % (patch_size * 2) == 0); + GGML_ASSERT(img.ny() % (patch_size * 2) == 0); + + const size_t nb1 = ggml_row_size(inp_raw->type, img.nx()); + const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny()); + + if (n_batch == 1) { + // still image input + return ggml_add(ctx0, + ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1), + ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1)); + } else if (n_batch == 2) { + // 2 frames input (video input) + ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw, + img.nx(), img.ny(), 3, nb1, nb2, 0); + ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw, + img.nx(), img.ny(), 3, nb1, nb2, + nb2 * 3); // move to the second frame + return ggml_add(ctx0, + ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1), + ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1)); + } else { + GGML_ASSERT(false && "n_batch > 2 is not supported"); + } +} + +ggml_tensor * mtmd_graph::build_inp_attn_mask(int64_t n_kv, int64_t n_q, int idx) { + const ggml_type type = flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED ? GGML_TYPE_F16 : GGML_TYPE_F32; + ggml_tensor * mask = ggml_new_tensor_2d(ctx0, type, n_kv, n_q); + ggml_set_name(mask, get_attn_mask_name(idx).c_str()); + ggml_set_input(mask); + return mask; +} + +ggml_tensor * mtmd_graph::build_norm( + ggml_tensor * cur, + ggml_tensor * mw, + ggml_tensor * mb, + norm_type type, + float norm_eps, + int il) const { + + cur = type == NORM_TYPE_RMS + ? ggml_rms_norm(ctx0, cur, norm_eps) + : ggml_norm(ctx0, cur, norm_eps); + + if (mw) { + cur = ggml_mul(ctx0, cur, mw); + cb(cur, "norm_w", il); + } + + if (mb) { + cur = ggml_add(ctx0, cur, mb); + cb(cur, "norm_b", il); + } + + return cur; +} + +ggml_tensor * mtmd_graph::build_ffn( + ggml_tensor * cur, + ggml_tensor * up, + ggml_tensor * up_b, + ggml_tensor * gate, + ggml_tensor * gate_b, + ggml_tensor * down, + ggml_tensor * down_b, + ffn_op_type type_op, + int il) const { + + ggml_tensor * tmp = up ? build_mm(up, cur) : cur; + cb(tmp, "ffn_up", il); + + if (up_b) { + tmp = ggml_add(ctx0, tmp, up_b); + cb(tmp, "ffn_up_b", il); + } + + if (gate) { + cur = build_mm(gate, cur); + cb(cur, "ffn_gate", il); + + if (gate_b) { + cur = ggml_add(ctx0, cur, gate_b); + cb(cur, "ffn_gate_b", il); + } + } else { + cur = tmp; + } + + // we only support parallel ffn for now + switch (type_op) { + case FFN_SILU: + if (gate) { + cur = hparams.swiglu_clamp > 0.0f + ? ggml_swiglu_clamp(ctx0, cur, tmp, hparams.swiglu_clamp) + : ggml_swiglu_split(ctx0, cur, tmp); + cb(cur, "ffn_swiglu", il); + } else { + cur = ggml_silu(ctx0, cur); + cb(cur, "ffn_silu", il); + } break; + case FFN_GELU: + if (gate) { + cur = ggml_geglu_split(ctx0, cur, tmp); + cb(cur, "ffn_geglu", il); + } else { + cur = ggml_gelu(ctx0, cur); + cb(cur, "ffn_gelu", il); + } break; + case FFN_GELU_ERF: + if (gate) { + cur = ggml_geglu_erf_split(ctx0, cur, tmp); + cb(cur, "ffn_geglu_erf", il); + } else { + cur = ggml_gelu_erf(ctx0, cur); + cb(cur, "ffn_gelu_erf", il); + } break; + case FFN_GELU_QUICK: + if (gate) { + cur = ggml_geglu_quick_split(ctx0, cur, tmp); + cb(cur, "ffn_geglu_quick", il); + } else { + cur = ggml_gelu_quick(ctx0, cur); + cb(cur, "ffn_gelu_quick", il); + } break; + case FFN_RELU_SQR: + { + cur = ggml_relu(ctx0, cur); + cur = ggml_sqr(ctx0, cur); + cb(cur, "ffn_relu_sqr", il); + } break; + } + + if (down) { + cur = build_mm(down, cur); + } + + if (down_b) { + cb(cur, "ffn_down", il); + } + + if (down_b) { + cur = ggml_add(ctx0, cur, down_b); + } + + return cur; +} + +// MoE FFN with sigmoid router and normalized top-k weights (dots3note vision) +// the router runs in fp32; exp_probs_b only affects expert selection, not the weights +ggml_tensor * mtmd_graph::build_moe_ffn(ggml_tensor * cur, const mtmd_layer & layer, ffn_op_type type_op, int il) const { + const int64_t n_tokens = cur->ne[1]; + const int64_t n_expert = layer.ff_gate_exps_w->ne[2]; + const int64_t n_expert_used = std::min((int64_t) hparams.n_expert_used, n_expert); + GGML_ASSERT(n_expert_used > 0); + GGML_ASSERT(type_op == FFN_SILU); + + ggml_tensor * probs = ggml_sigmoid(ctx0, build_mm(layer.ff_gate_inp_w, cur)); // [n_expert, n_tokens] + cb(probs, "ffn_moe_probs", il); + + ggml_tensor * sel = layer.ff_exp_probs_b + ? ggml_add(ctx0, probs, layer.ff_exp_probs_b) + : probs; + ggml_tensor * selected = ggml_top_k(ctx0, sel, n_expert_used); // [n_expert_used, n_tokens] + + ggml_tensor * weights = ggml_get_rows(ctx0, + ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens), selected); + weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens); + weights = ggml_div(ctx0, weights, ggml_sum_rows(ctx0, weights)); + weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens); + cb(weights, "ffn_moe_weights", il); + + cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], 1, n_tokens); + ggml_tensor * gate = ggml_mul_mat_id(ctx0, layer.ff_gate_exps_w, cur, selected); // [n_ff, n_expert_used, n_tokens] + ggml_tensor * up = ggml_mul_mat_id(ctx0, layer.ff_up_exps_w, cur, selected); + cur = ggml_mul(ctx0, ggml_silu(ctx0, gate), up); + cur = ggml_mul_mat_id(ctx0, layer.ff_down_exps_w, cur, selected); // [n_embd, n_expert_used, n_tokens] + cur = ggml_mul(ctx0, cur, weights); + + // sum over the selected experts + ggml_tensor * out = nullptr; + for (int64_t i = 0; i < n_expert_used; i++) { + ggml_tensor * v = ggml_view_2d(ctx0, cur, cur->ne[0], n_tokens, cur->nb[2], i * cur->nb[1]); + out = out ? ggml_add(ctx0, out, v) : v; + } + if (n_expert_used == 1) { + out = ggml_cont(ctx0, out); + } + cb(out, "ffn_moe_out", il); + return out; +} + +ggml_tensor * mtmd_graph::build_attn( + ggml_tensor * wo, + ggml_tensor * wo_b, + ggml_tensor * q_cur, + ggml_tensor * k_cur, + ggml_tensor * v_cur, + ggml_tensor * kq_mask, + float kq_scale, + int il, + ggml_tensor * sinks) const { + // these nodes are added to the graph together so that they are not reordered + // by doing so, the number of splits in the graph is reduced + // the order is fixed without the compute flag, so an unselected branch stays out of the compute set + ggml_build_forward_order(gf, q_cur); + ggml_build_forward_order(gf, k_cur); + ggml_build_forward_order(gf, v_cur); + + ggml_tensor * q = ggml_permute(ctx0, q_cur, 0, 2, 1, 3); + //cb(q, "q", il); + + ggml_tensor * k = ggml_permute(ctx0, k_cur, 0, 2, 1, 3); + //cb(k, "k", il); + + ggml_tensor * cur; + + if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) { + ggml_tensor * v = ggml_permute(ctx0, v_cur, 0, 2, 1, 3); + + k = ggml_cast(ctx0, k, GGML_TYPE_F16); + v = ggml_cast(ctx0, v, GGML_TYPE_F16); + // mask must be f16 here, use build_inp_attn_mask() + GGML_ASSERT(!kq_mask || kq_mask->type == GGML_TYPE_F16); + + cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f); + ggml_prec_set_acc(cur, GGML_PREC_F32); + if (sinks != nullptr) { + ggml_flash_attn_ext_add_sinks(cur, sinks); + } + + cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]); + + } else { + ggml_tensor * v = ggml_permute(ctx0, v_cur, 1, 2, 0, 3); + v = ggml_cont(ctx0, v); + + ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); + // TODO: this most likely requires GGML_PREC_BF16, but will enable it later to avoid scope-creep + // ggml_prec_set_acc(kq, GGML_PREC_BF16); + + kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, 0.0f); + if (sinks != nullptr) { + ggml_soft_max_add_sinks(kq, sinks); + } + + ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq); + cur = ggml_permute(ctx0, kqv, 0, 2, 1, 3); + cur = ggml_cont_2d(ctx0, cur, cur->ne[0] * cur->ne[1], cur->ne[2] * cur->ne[3]); + } + + cb(cur, "kqv_out", il); + + if (wo) { + cur = build_mm(wo, cur); + } + + if (wo_b) { + cur = ggml_add(ctx0, cur, wo_b); + } + + return cur; +} + +// implementation of the 2D RoPE without adding a new op in ggml +ggml_tensor * mtmd_graph::build_rope_2d( + ggml_context * ctx0, + ggml_tensor * cur, + ggml_tensor * pos_a, // first half + ggml_tensor * pos_b, // second half + const float freq_base, + const bool interleave_freq +) { + const int64_t n_dim = cur->ne[0]; + + // for example, if we have cur tensor of shape (n_dim=8, n_head, n_pos) + // we will have a list of 4 inv_freq: 1e-0, 1e-1, 1e-2, 1e-3 + // first half of cur will use 1e-0, 1e-2 (even) + // second half of cur will use 1e-1, 1e-3 (odd) + // the trick here is to rotate just half of n_dim, so inv_freq will automatically be even + // ^ don't ask me why, it's math! -2(2i) / n_dim == -2i / (n_dim/2) + // then for the second half, we use freq_scale to shift the inv_freq + // ^ why? replace (2i) with (2i+1) in the above equation + const float freq_scale_odd = interleave_freq + ? std::pow(freq_base, (float)-2/n_dim) + : 1.0; + + // first half, dims [0, n_dim/2) + cur = ggml_rope_ext( + ctx0, + cur, + pos_a, // positions + nullptr, // freq factors + n_dim/2, // n_dims + 0, 0, freq_base, + 1.0f, 0.0f, 1.0f, 0.0f, 0.0f + ); + + // second half, dims [n_dim/2, n_dim) + cur = ggml_rope_ext( + ctx0, + cur, + pos_b, // positions + nullptr, // freq factors + n_dim/2, // n_dims + 0, 0, freq_base, + freq_scale_odd, + 0.0f, 1.0f, 0.0f, 0.0f + ); + cur = ggml_rope_set_offset(cur, n_dim/2); + + return cur; +} + +// Generic function to stack frames for audio processing +// Abstracts out the StackAudioFrames logic used by ultravox +ggml_tensor * mtmd_graph::build_stack(ggml_tensor * cur, int32_t stack_factor, int32_t n_embed) { + if (stack_factor <= 1) { + return cur; + } + + int64_t total_elements = ggml_nelements(cur); + int64_t stride = n_embed * stack_factor; + + // Calculate padded length + int64_t padded_len = GGML_PAD(total_elements, stride); + int64_t pad = padded_len - total_elements; + + if (pad > 0) { + // Pad the tensor to make it divisible by stride + cur = ggml_view_1d(ctx0, cur, total_elements, 0); + cur = ggml_pad(ctx0, cur, pad, 0, 0, 0); + } + + // Reshape to [stride, padded_len / stride] + cur = ggml_view_2d(ctx0, cur, stride, padded_len / stride, + ggml_row_size(cur->type, stride), 0); + return cur; +} + +// aka pixel_shuffle / pixel_unshuffle / patch_merger (Kimi-VL) +// support dynamic resolution +ggml_tensor * mtmd_graph::build_suffix(ggml_tensor * cur) { + for (int idx : mtmd_suffix_rows(img.suffix_type)) { + GGML_ASSERT(model.tok_embd_sep && idx < model.tok_embd_sep->ne[1]); + ggml_tensor * row = ggml_view_2d(ctx0, model.tok_embd_sep, model.tok_embd_sep->ne[0], 1, + model.tok_embd_sep->nb[1], idx * model.tok_embd_sep->nb[1]); + cur = ggml_concat(ctx0, cur, ggml_cast(ctx0, row, cur->type), 1); + } + return cur; +} + +ggml_tensor * mtmd_graph::build_patch_merge_permute(ggml_tensor * cur, int scale_factor) { + GGML_ASSERT(scale_factor > 1); + + const int n_embd = cur->ne[0]; + int width = img.nx() / patch_size; + int height = img.ny() / patch_size; + + // pad width and height to factor + const int64_t pad_width = MTMD_ALIGN(width, scale_factor) - width; + const int64_t pad_height = MTMD_ALIGN(height, scale_factor) - height; + cur = ggml_reshape_3d(ctx0, cur, n_embd, width, height); + if (pad_width || pad_height) { + cur = ggml_pad(ctx0, cur, 0, pad_width, pad_height, 0); + width += pad_width; + height += pad_height; + } + + // unshuffle h + cur = ggml_reshape_3d(ctx0, cur, n_embd * scale_factor, width / scale_factor, height); + cur = ggml_permute(ctx0, cur, 0, 2, 1, 3); + + // unshuffle w + cur = ggml_cont_3d(ctx0, cur, n_embd * scale_factor * scale_factor, height / scale_factor, width / scale_factor); + cur = ggml_permute(ctx0, cur, 0, 2, 1, 3); + + cur = ggml_cont_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]); + cb(cur, "pixel_shuffle", -1); + + return cur; +} + +std::unique_ptr mtmd_get_graph_builder(mtmd_subcontext * ctx, const mtmd_image_f32_batch & imgs, + const mtmd_subctx_encode_params * params) { + const mtmd_image_f32 & img = imgs.entries[0]; + std::unique_ptr builder; + + switch (ctx->model.proj_type) { + case PROJECTOR_TYPE_GEMMA3: + case PROJECTOR_TYPE_IDEFICS3: + case PROJECTOR_TYPE_COHERE2V: + case PROJECTOR_TYPE_LFM2: + case PROJECTOR_TYPE_D1OMNI_V: + case PROJECTOR_TYPE_JANUS_PRO: + case PROJECTOR_TYPE_PHI4: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_GEMMA3NV: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_GEMMA4V: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_GEMMA4UV: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_PIXTRAL: + case PROJECTOR_TYPE_LIGHTONOCR: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: // same ViT + merger; pyramid MoE is handled by build_vit + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_QWEN2VL: + case PROJECTOR_TYPE_QWEN25VL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_QWEN3VL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MAGEVL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_SENSENOVA_U1: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_LFM3VL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_LING3VL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_EXAONE4_5: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MIMOVL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MUSE_GLIMMER: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_STEP3VL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MINICPMV: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MINICPMV4_6: + case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_INTERNVL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_NEMOTRON_V2_VL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_NEMOTRON_OMNI: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_LLAMA4: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_VOXTRAL_RT: + { + builder = std::make_unique(ctx, img, params ? params->enc_state : nullptr); + } break; + case PROJECTOR_TYPE_ULTRAVOX: + case PROJECTOR_TYPE_VOXTRAL: + case PROJECTOR_TYPE_QWEN2A: + case PROJECTOR_TYPE_GLMA: + case PROJECTOR_TYPE_MERALION: + case PROJECTOR_TYPE_MUSIC_FLAMINGO: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_KIMIVL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_PADDLEOCR: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_KIMIK25: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_DEEPSEEK4V: + case PROJECTOR_TYPE_DEEPSEEK41V: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_COGVLM: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_HUNYUANVL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MLP: + case PROJECTOR_TYPE_MLP_NORM: + case PROJECTOR_TYPE_LDP: + case PROJECTOR_TYPE_LDPV2: + case PROJECTOR_TYPE_GLM_EDGE: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_DEEPSEEKOCR: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_DEEPSEEKOCR2: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_LFM2A: + case PROJECTOR_TYPE_D1OMNI_A: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_LFM3A: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_GEMMA4A: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_GEMMA4UA: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_GRANITE_SPEECH: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_GLM4V: + case PROJECTOR_TYPE_GLM5V: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_QWEN3A: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MIMO_AUDIO: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_POCKETTTS_SPKENC: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_POCKETTTS_GEN: + { + const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; + const int n_step = ctx->model.hparams.flow_n_step; + const int64_t n_latent = ctx->model.gen_input_lin_w->ne[0]; + GGML_ASSERT(n_step > 0); + GGML_ASSERT(n_latent > 0); + // "inp_feats" takes the caller's buffer as-is, the graph must consume all of it + if (params && params->feats) { + GGML_ASSERT(params->feats->size() % (size_t) n_latent == 0); + GGML_ASSERT(params->feats->size() >= (size_t) n_latent); + } + const int n_frames = params && params->feats ? (int) (params->feats->size() / n_latent) : 1; + builder = std::make_unique(ctx, img, gen_process, n_step, n_frames); + } break; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_CHATTERBOX: + { + const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; + int n_tokens = params && params->codes ? (int) params->codes->size() : 0; + int n_prompt_mel = 0; + if (n_tokens > 0) { + GGML_ASSERT(params->ref); + n_tokens += (int) params->ref->tokens.size(); + n_prompt_mel = (int) (params->ref->feat.size() / 80); + } + const int vnm = params ? params->vocode_n_mel : 0; + const int vns = params ? params->vocode_n_stft : 0; + builder = std::make_unique(ctx, img, gen_process, n_tokens, n_prompt_mel, vnm, vns); + } break; + case PROJECTOR_TYPE_LFM3A_GEN: + { + const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; + const int64_t n_cb = ctx->model.gen_code_embd_w->ne[2]; + bool all_known = false; + int n_frames = 1; + if (params && params->codes) { + GGML_ASSERT(params->codes->size() % (size_t) n_cb == 0); + if (gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { + n_frames = (int) (params->codes->size() / n_cb); + GGML_ASSERT(n_frames > 0); + } else { + // -1 marks a masked codebook, none masked means only the feedback embedding is needed + all_known = std::all_of(params->codes->begin(), params->codes->end(), [](int32_t c) { return c >= 0; }); + } + } + builder = std::make_unique(ctx, img, gen_process, all_known, n_frames); + } break; + case PROJECTOR_TYPE_DRAMA3_GEN: + { + const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; + const int64_t n_cb = ctx->model.gen_code_embd_w->ne[2]; + const int n_frames = params && params->codes ? (int) (params->codes->size() / n_cb) : 1; + builder = std::make_unique(ctx, img, gen_process, params ? params->gen_state : nullptr, n_frames, + params ? params->top_k : 30, params ? params->top_p : 0.9f, + params ? params->temp : 1.0f); + } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; + const int top_k = params ? params->top_k : 50; + const float top_p = params ? params->top_p : 1.0f; + builder = std::make_unique(ctx, img, gen_process, top_k, top_p); + } break; + case PROJECTOR_TYPE_SOPRANO: + { + const int n_frames = params && params->feats ? (int) (params->feats->size() / ctx->model.hparams.projection_dim) : 2; + builder = std::make_unique(ctx, img, n_frames); + } break; + case PROJECTOR_TYPE_YOUTUVL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_YASA2: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_PARAKEET: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: + { + builder = std::make_unique(ctx, img, params ? params->enc_state : nullptr); + } break; + case PROJECTOR_TYPE_GRANITE4_VISION: + { + builder = std::make_unique(ctx, img); + } break; + default: + GGML_ABORT("missing cgraph builder"); + } + + builder->img_batch = &imgs; + + // TODO [QWEN_VIDEO]: improve this in the future + builder->n_batch = imgs.entries.size(); + + return builder; +} diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/mtmd-graph.h similarity index 84% rename from tools/mtmd/clip-graph.h rename to tools/mtmd/mtmd-graph.h index a0669d1a0..41765d03f 100644 --- a/tools/mtmd/clip-graph.h +++ b/tools/mtmd/mtmd-graph.h @@ -2,9 +2,9 @@ #include "ggml.h" #include "ggml-cpp.h" -#include "clip.h" -#include "clip-impl.h" -#include "clip-model.h" +#include "mtmd-subcontext.h" +#include "mtmd-impl.h" +#include "mtmd-model.h" #include #include @@ -23,13 +23,13 @@ struct build_vit_opts { bool skip_post_ln = false; }; -struct clip_graph { - const clip_model & model; - const clip_hparams & hparams; +struct mtmd_graph { + const mtmd_model & model; + const mtmd_hparams & hparams; projector_type proj_type; - const clip_image_f32 & img; // for backward compat - const clip_image_f32_batch * img_batch = nullptr; + const mtmd_image_f32 & img; // for backward compat + const mtmd_image_f32_batch * img_batch = nullptr; const int patch_size; const int n_patches_x; @@ -43,7 +43,7 @@ struct clip_graph { const int n_mmproj_embd; const float eps; float kq_scale; // TODO: maybe move this to hparams - const clip_flash_attn_type flash_attn_type; + const mtmd_flash_attn_type flash_attn_type; // TODO [QWEN_VIDEO]: improve this in the future int n_batch = 1; @@ -52,12 +52,12 @@ struct clip_graph { ggml_context * ctx0; ggml_cgraph * gf; - clip_graph(clip_ctx * ctx, const clip_image_f32 & img); + mtmd_graph(mtmd_subcontext * ctx, const mtmd_image_f32 & img); // build sub-graph, reuse buf from parent - clip_graph(const clip_graph & parent); + mtmd_graph(const mtmd_graph & parent); - virtual ~clip_graph() = default; + virtual ~mtmd_graph() = default; virtual ggml_cgraph * build() = 0; // wrapper around ggml_mul_mat, allow hooking (e.g. LoRA, clamping) depending on the model @@ -74,7 +74,7 @@ struct clip_graph { // void cb(ggml_tensor * cur0, const char * name, int il) const; - const clip_image_f32 & get_img(size_t idx) const { + const mtmd_image_f32 & get_img(size_t idx) const { GGML_ASSERT(img_batch); GGML_ASSERT(idx < img_batch->entries.size()); return img_batch->entries[idx]; @@ -92,7 +92,7 @@ struct clip_graph { norm_type norm_t, ffn_op_type ffn_t, ggml_tensor * learned_pos_embd, - std::function add_pos, + std::function add_pos, const build_vit_opts & opts = {}); // build the input after conv2d (inp_raw --> patches) @@ -130,7 +130,7 @@ struct clip_graph { ggml_tensor * build_moe_ffn( ggml_tensor * cur, - const clip_layer & layer, + const mtmd_layer & layer, ffn_op_type type_op, int il) const; @@ -174,3 +174,11 @@ struct clip_graph { // append the separators of img.suffix_type after the image tokens ggml_tensor * build_suffix(ggml_tensor * cur); }; + +static inline std::string get_attn_mask_name(int idx) { + return idx == 0 ? "attn_mask" : "attn_mask_" + std::to_string(idx); +} + +// create the graph builder of the subcontext's projector, params is only needed by encode() +std::unique_ptr mtmd_get_graph_builder(mtmd_subcontext * ctx, const mtmd_image_f32_batch & imgs, + const mtmd_subctx_encode_params * params = nullptr); diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index d776ed1e8..acdee5d12 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -494,7 +494,7 @@ private: llama_token tts_eos = LLAMA_TOKEN_NULL; std::vector tok_embd; // whole token embedding matrix, n_vocab * n_embd - // must match hparams.wav_tfm_swa hardcoded in clip.cpp + // must match hparams.wav_tfm_swa hardcoded in mtmd-model.cpp size_t window_frames = 72; // per-generation state, cleared by reset() @@ -2291,7 +2291,7 @@ private: // repetition-aware sampling: if a semantic draw repeats one of the last ras_window main tokens, draw again with fixed settings static constexpr size_t ras_window = 16; static constexpr size_t n_cb = 10; - // codebook tower KV rows (clip_graph_drama3_gen::n_pos_max): one per prompt row but the last, one per frame + // codebook tower KV rows (mtmd_graph_drama3_gen::n_pos_max): one per prompt row but the last, one per frame static constexpr size_t n_tower_rows = 4096; // frames per codec call when streaming (0.33 s) static constexpr size_t stream_frames = 7; diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 8c0f433e3..b0543aa43 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -4,8 +4,8 @@ #include #include -void mtmd_image_preproc_out::append(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized) { - clip_image_f32 dst; +void mtmd_image_preproc_out::append(const mtmd_hparams & hparams, const mtmd_image_u8 & img, bool normalized) { + mtmd_image_f32 dst; dst.from_u8(img); if (normalized) { dst.normalize(hparams.image_mean, hparams.image_std); @@ -13,20 +13,20 @@ void mtmd_image_preproc_out::append(const clip_hparams & hparams, const clip_ima entries.push_back(std::move(dst)); } -void mtmd_image_preproc_out::append(const clip_hparams & hparams, const std::vector & imgs, bool normalized) { +void mtmd_image_preproc_out::append(const mtmd_hparams & hparams, const std::vector & imgs, bool normalized) { for (const auto & img : imgs) { append(hparams, img, normalized); } } -void mtmd_image_preproc_out::append(const clip_hparams & hparams, clip_image_f32 & img, bool normalized) { +void mtmd_image_preproc_out::append(const mtmd_hparams & hparams, mtmd_image_f32 & img, bool normalized) { if (normalized) { img.normalize(hparams.image_mean, hparams.image_std); } entries.push_back(std::move(img)); } -void mtmd_image_preproc_out::append_overview(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized) { +void mtmd_image_preproc_out::append_overview(const mtmd_hparams & hparams, const mtmd_image_u8 & img, bool normalized) { overview.from_u8(img); if (normalized) { overview.normalize(hparams.image_mean, hparams.image_std); @@ -37,9 +37,9 @@ void mtmd_image_preproc_out::append_overview(const clip_hparams & hparams, const // in the future, we can have HW acceleration by allowing this struct to access 3rd party lib like imagick or opencv struct img_tool { static void resize( - const clip_image_u8 & src, - clip_image_u8 & dst, - const clip_image_size & target_resolution, + const mtmd_image_u8 & src, + mtmd_image_u8 & dst, + const mtmd_image_size & target_resolution, resize_algo algo, pad_style padding = PAD_CEIL, std::array pad_color = {0, 0, 0}) { @@ -61,7 +61,7 @@ struct img_tool { resize_pillow(src, dst, target_resolution.width, target_resolution.height, algo); } else { // resize with padding - clip_image_u8 resized_image; + mtmd_image_u8 resized_image; float scale_w = static_cast(target_resolution.width) / src.get_size().width; float scale_h = static_cast(target_resolution.height) / src.get_size().height; float scale = std::min(scale_w, scale_h); @@ -92,7 +92,7 @@ struct img_tool { } } - static void crop(const clip_image_u8 & image, clip_image_u8 & dst, int x, int y, int w, int h) { + static void crop(const mtmd_image_u8 & image, mtmd_image_u8 & dst, int x, int y, int w, int h) { GGML_ASSERT(x >= 0 && y >= 0 && w > 0 && h > 0); GGML_ASSERT(x + w <= image.get_size().width && y + h <= image.get_size().height); dst.set_size({w, h}, image.is_placeholder()); @@ -120,7 +120,7 @@ struct img_tool { // calculate the size of the **resized** image, while preserving the aspect ratio and // aligning to the nearest multiple of align_size ("smart_resize" in transformers code) - static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const calc_size_opt & opts) { + static mtmd_image_size calc_size_preserved_ratio(const mtmd_image_size & inp_size, const calc_size_opt & opts) { GGML_ASSERT(opts.align_size > 0); const int width = inp_size.width; const int height = inp_size.height; @@ -161,7 +161,7 @@ struct img_tool { } // draw src image into dst image at offset (offset_x, offset_y) - static void composite(clip_image_u8 & dst, const clip_image_u8 & src, int offset_x, int offset_y) { + static void composite(mtmd_image_u8 & dst, const mtmd_image_u8 & src, int offset_x, int offset_y) { if (src.is_placeholder()) { // no-op for placeholder image return; @@ -183,7 +183,7 @@ struct img_tool { } // fill the image with a solid color - static void fill(clip_image_u8 & img, const std::array & color) { + static void fill(mtmd_image_u8 & img, const std::array & color) { if (img.is_placeholder()) { // no-op for placeholder image return; @@ -208,8 +208,8 @@ private: // // RESIZE_ALGO_FLAG_TORCH is the same, except for the int16 weights of torchvision (aten/src/ATen/native/cpu/UpSampleKernel.cpp) static bool resize_pillow( - const clip_image_u8 & img, - clip_image_u8 & dst, + const mtmd_image_u8 & img, + mtmd_image_u8 & dst, int target_width, int target_height, resize_algo algo) { @@ -516,8 +516,8 @@ private: // mtmd_image_preprocessor_llava_uhd // -mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const clip_image_u8 & img) const { - const clip_image_size original_size = img.get_size(); +mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const mtmd_image_u8 & img) const { + const mtmd_image_size original_size = img.get_size(); auto const inst = get_slice_instructions(original_size); auto sliced = slice_image(img, inst); @@ -530,7 +530,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const clip_ return output; } -mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_llava_uhd::get_slice_instructions(const clip_image_size & original_size) const { +mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_llava_uhd::get_slice_instructions(const mtmd_image_size & original_size) const { mtmd_image_preprocessor_llava_uhd::slice_instructions res; const int patch_size = get_slice_align(); const int slice_size = hparams.image_size; @@ -542,9 +542,9 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_ll if (!has_slices) { // skip slicing logic - res.overview_size = clip_image_size{slice_size, slice_size}; - res.refined_size = clip_image_size{0, 0}; - res.grid_size = clip_image_size{0, 0}; + res.overview_size = mtmd_image_size{slice_size, slice_size}; + res.refined_size = mtmd_image_size{0, 0}; + res.grid_size = mtmd_image_size{0, 0}; return res; } @@ -554,9 +554,9 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_ll auto refine_size = select_best_resolution( original_size, hparams.image_res_candidates); - res.overview_size = clip_image_size{slice_size, slice_size}; + res.overview_size = mtmd_image_size{slice_size, slice_size}; res.refined_size = refine_size; - res.grid_size = clip_image_size{0, 0}; + res.grid_size = mtmd_image_size{0, 0}; LOG_DBG("%s: using pinpoints for slicing\n", __func__); LOG_DBG("%s: original size: %d x %d, overview size: %d x %d, refined size: %d x %d\n", @@ -633,7 +633,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_ll return res; } -mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uhd::slice_image(const clip_image_u8 & img, const mtmd_image_preprocessor_llava_uhd::slice_instructions & inst) const { +mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uhd::slice_image(const mtmd_image_u8 & img, const mtmd_image_preprocessor_llava_uhd::slice_instructions & inst) const { slice_output output; // resize to overview size @@ -646,7 +646,7 @@ mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uh } // resize to refined size - clip_image_u8 refined_img; + mtmd_image_u8 refined_img; img_tool::resize(img, refined_img, inst.refined_size, hparams.image_resize_algo_rf, hparams.image_pad_rf, hparams.image_pad_color_rf); @@ -657,7 +657,7 @@ mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uh int w = slice.size.width; int h = slice.size.height; - clip_image_u8 img_slice; + mtmd_image_u8 img_slice; img_tool::crop(refined_img, img_slice, x, y, w, h); output.slices.push_back(std::move(img_slice)); } @@ -665,7 +665,7 @@ mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uh return output; } -clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale) const { +mtmd_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const mtmd_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale) const { int width = original_size.width; int height = original_size.height; if ((width * height > scale_resolution * scale_resolution) || allow_upscale) { @@ -673,28 +673,28 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const clip_im height = static_cast(scale_resolution / std::sqrt(r)); width = static_cast(height * r); } - clip_image_size res; + mtmd_image_size res; res.width = ensure_divide(width, patch_size); res.height = ensure_divide(height, patch_size); return res; } -clip_image_size mtmd_image_preprocessor_llava_uhd::resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max) const { +mtmd_image_size mtmd_image_preprocessor_llava_uhd::resize_maintain_aspect_ratio(const mtmd_image_size & orig, const mtmd_image_size & target_max) const { float scale_width = static_cast(target_max.width) / orig.width; float scale_height = static_cast(target_max.height) / orig.height; float scale = std::min(scale_width, scale_height); - return clip_image_size{ + return mtmd_image_size{ static_cast(orig.width * scale), static_cast(orig.height * scale), }; } -clip_image_size mtmd_image_preprocessor_llava_uhd::select_best_resolution(const clip_image_size & original_size, const std::vector & possible_resolutions) const { - clip_image_size best_fit; +mtmd_image_size mtmd_image_preprocessor_llava_uhd::select_best_resolution(const mtmd_image_size & original_size, const std::vector & possible_resolutions) const { + mtmd_image_size best_fit; int min_wasted_area = std::numeric_limits::max(); int max_effective_resolution = 0; - for (const clip_image_size & candidate : possible_resolutions) { + for (const mtmd_image_size & candidate : possible_resolutions) { auto target_size = resize_maintain_aspect_ratio(original_size, candidate); int effective_resolution = std::min( target_size.width * target_size.height, @@ -717,7 +717,7 @@ int mtmd_image_preprocessor_llava_uhd::ensure_divide(int length, int patch_size) return std::max(align_round(static_cast(length) / patch_size) * patch_size, patch_size); } -clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale) const { +mtmd_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const mtmd_image_size & original_size, const mtmd_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale) const { int width = original_size.width; int height = original_size.height; int grid_x = grid.width; @@ -726,7 +726,7 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_im int refine_width = ensure_divide(width, grid_x); int refine_height = ensure_divide(height, grid_y); - clip_image_size grid_size; + mtmd_image_size grid_size; grid_size.width = refine_width / grid_x; grid_size.height = refine_height / grid_y; @@ -734,13 +734,13 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_im int best_grid_width = best_grid_size.width; int best_grid_height = best_grid_size.height; - clip_image_size refine_size; + mtmd_image_size refine_size; refine_size.width = best_grid_width * grid_x; refine_size.height = best_grid_height * grid_y; return refine_size; } -clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const { +mtmd_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const { std::vector candidate_split_grids_nums; for (int i : {multiple - 1, multiple, multiple + 1}) { if (i == 1 || i > max_slice_nums) { @@ -749,18 +749,18 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_s candidate_split_grids_nums.push_back(i); } - std::vector candidate_grids; + std::vector candidate_grids; for (int split_grids_nums : candidate_split_grids_nums) { int m = 1; while (m <= split_grids_nums) { if (split_grids_nums % m == 0) { - candidate_grids.push_back(clip_image_size{m, split_grids_nums / m}); + candidate_grids.push_back(mtmd_image_size{m, split_grids_nums / m}); } ++m; } } - clip_image_size best_grid{1, 1}; + mtmd_image_size best_grid{1, 1}; float min_error = std::numeric_limits::infinity(); for (const auto& grid : candidate_grids) { float error = std::abs(log_ratio - std::log(1.0 * grid.width / grid.height)); @@ -776,8 +776,8 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_s // mtmd_image_preprocessor_fixed_size // -mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip_image_u8 & img) const { - clip_image_u8 resized_image; +mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const mtmd_image_u8 & img) const { + mtmd_image_u8 resized_image; int sz = hparams.image_size; img_tool::resize(img, resized_image, {sz, sz}, hparams.image_resize_algo, @@ -792,7 +792,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip // mtmd_image_preprocessor_dyn_size // -clip_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const clip_hparams & hparams, const clip_image_size & original_size) { +mtmd_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const mtmd_hparams & hparams, const mtmd_image_size & original_size) { GGML_ASSERT(hparams.image_min_pixels > 0 && hparams.image_max_pixels > 0); // the original pixtral model doesn't have n_merge const int cur_merge = hparams.n_merge; @@ -807,8 +807,8 @@ clip_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const clip_hpa }); } -mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const { - clip_image_u8 resized_image; +mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const mtmd_image_u8 & img) const { + mtmd_image_u8 resized_image; img_tool::resize(img, resized_image, target_size(img.get_size()), hparams.image_resize_algo, hparams.image_resize_pad, @@ -823,7 +823,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_i // // rounds before the pixel checks (an edge may round to 0) and computes in double, as transformers does -clip_image_size mtmd_image_preprocessor_qwen3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels, +mtmd_image_size mtmd_image_preprocessor_qwen3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels, int n_frames, int temporal_factor) { double h = height; double w = width; @@ -850,13 +850,13 @@ clip_image_size mtmd_image_preprocessor_qwen3vl::smart_resize(int width, int hei return {std::max(factor, w_bar), std::max(factor, h_bar)}; } -bool mtmd_image_preprocessor_qwen3vl::get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const { +bool mtmd_image_preprocessor_qwen3vl::get_video_frame_size(int n_frames, int width, int height, mtmd_image_size & out) const { // Qwen3VLVideoProcessor (and Lfm3VlVideoProcessor) size: shortest_edge 4096, longest_edge 25165824, temporal_patch_size 2 out = smart_resize(width, height, hparams.patch_size * hparams.n_merge, 4096, 25165824, n_frames, 2); return true; } -mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::preprocess_video_frame(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::preprocess_video_frame(const mtmd_image_u8 & img) const { const int f = hparams.patch_size * hparams.n_merge; const auto size = img.get_size(); return resize_to(img, { @@ -865,8 +865,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::preprocess_video_frame(c }); } -mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::resize_to(const clip_image_u8 & img, const clip_image_size & size) const { - clip_image_u8 resized_image; +mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::resize_to(const mtmd_image_u8 & img, const mtmd_image_size & size) const { + mtmd_image_u8 resized_image; img_tool::resize(img, resized_image, size, hparams.image_resize_algo, hparams.image_resize_pad, @@ -880,7 +880,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::resize_to(const clip_ima // mtmd_image_preprocessor_lfm3vl // -clip_image_size mtmd_image_preprocessor_lfm3vl::target_size(const clip_image_size & original_size) const { +mtmd_image_size mtmd_image_preprocessor_lfm3vl::target_size(const mtmd_image_size & original_size) const { return smart_resize(original_size.width, original_size.height, hparams.patch_size * hparams.n_merge, hparams.image_min_pixels, hparams.image_max_pixels); } @@ -891,7 +891,7 @@ clip_image_size mtmd_image_preprocessor_lfm3vl::target_size(const clip_image_siz // The canvas is ceil-aligned to patch_size*n_merge and fitted to the token budget. // Only rescaled to meet the budget and sits top-left, with black padding on the right and bottom -mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const mtmd_image_u8 & img) const { GGML_ASSERT(hparams.image_min_pixels > 0 && hparams.image_max_pixels > 0); const int64_t factor = hparams.patch_size * hparams.n_merge; @@ -941,11 +941,11 @@ mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const clip_imag const int content_h = (int) std::max(1, std::min(canvas_h, (int64_t) std::floor(height * scale))); const int content_w = (int) std::max(1, std::min(canvas_w, (int64_t) std::floor(width * scale))); - clip_image_u8 content; - img_tool::resize(img, content, clip_image_size{content_w, content_h}, hparams.image_resize_algo, PAD_NONE); + mtmd_image_u8 content; + img_tool::resize(img, content, mtmd_image_size{content_w, content_h}, hparams.image_resize_algo, PAD_NONE); - clip_image_u8 canvas; - canvas.set_size(clip_image_size{(int) canvas_w, (int) canvas_h}, img.is_placeholder()); + mtmd_image_u8 canvas; + canvas.set_size(mtmd_image_size{(int) canvas_w, (int) canvas_h}, img.is_placeholder()); img_tool::fill(canvas, {0, 0, 0}); img_tool::composite(canvas, content, 0, 0); @@ -958,9 +958,9 @@ mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const clip_imag // mtmd_image_preprocessor_longest_edge // -clip_image_size mtmd_image_preprocessor_longest_edge::get_target_size(const clip_hparams & hparams, const clip_image_size & original_size) { +mtmd_image_size mtmd_image_preprocessor_longest_edge::get_target_size(const mtmd_hparams & hparams, const mtmd_image_size & original_size) { GGML_ASSERT(hparams.image_longest_edge > 0); - clip_image_size size = original_size; + mtmd_image_size size = original_size; if (hparams.image_floor_before_align) { const double ratio = double(std::max(size.width, size.height)) / hparams.image_longest_edge; if (ratio > 1.0) { @@ -985,9 +985,9 @@ clip_image_size mtmd_image_preprocessor_longest_edge::get_target_size(const clip }); } -mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const clip_image_u8 & img) const { - clip_image_u8 resized_image; - const clip_image_size target_size = get_target_size(hparams, img.get_size()); +mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const mtmd_image_u8 & img) const { + mtmd_image_u8 resized_image; + const mtmd_image_size target_size = get_target_size(hparams, img.get_size()); img_tool::resize(img, resized_image, target_size, hparams.image_resize_algo, hparams.image_resize_pad, @@ -1001,15 +1001,15 @@ mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const cl // mtmd_image_preprocessor_minicpmv // -mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_minicpmv::get_slice_instructions(const clip_image_size & original_size) const { +mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_minicpmv::get_slice_instructions(const mtmd_image_size & original_size) const { // overview only for small images, unlike generic llava-uhd which slices once one side exceeds scale resolution const int slice_size = hparams.image_size; const float ratio = (float) original_size.width * original_size.height / (slice_size * slice_size); if (ratio <= 1.0f) { mtmd_image_preprocessor_llava_uhd::slice_instructions inst; inst.overview_size = get_best_resize(original_size, slice_size, get_slice_align(), true); - inst.refined_size = clip_image_size{0, 0}; - inst.grid_size = clip_image_size{0, 0}; + inst.refined_size = mtmd_image_size{0, 0}; + inst.grid_size = mtmd_image_size{0, 0}; return inst; } return mtmd_image_preprocessor_llava_uhd::get_slice_instructions(original_size); @@ -1019,7 +1019,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_mi // mtmd_image_preprocessor_lfm2 // -mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const mtmd_image_u8 & img) const { auto const inst = get_slice_instructions(img.get_size()); if (!inst.slices.empty()) { return mtmd_image_preprocessor_llava_uhd::preprocess(img); @@ -1034,8 +1034,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const clip_image } bool mtmd_image_preprocessor_lfm2::should_tile( - const clip_hparams & hparams, - const clip_image_size & original_size) { + const mtmd_hparams & hparams, + const mtmd_image_size & original_size) { const int align_size = hparams.patch_size * hparams.n_merge; const auto round_by_factor = [align_size](float x) { @@ -1052,9 +1052,9 @@ bool mtmd_image_preprocessor_lfm2::should_tile( } // same as Lfm2VlImageProcessor.smart_resize: Python's round half to even, double precision -clip_image_size mtmd_image_preprocessor_lfm2::smart_resize( - const clip_hparams & hparams, - const clip_image_size & original_size) { +mtmd_image_size mtmd_image_preprocessor_lfm2::smart_resize( + const mtmd_hparams & hparams, + const mtmd_image_size & original_size) { const int align_size = hparams.patch_size * hparams.n_merge; const double height = original_size.height; const double width = original_size.width; @@ -1075,7 +1075,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::smart_resize( return {w_bar, h_bar}; } -mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) const { +mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const mtmd_image_size & original_size) const { mtmd_image_preprocessor_llava_uhd::slice_instructions inst; inst.overview_size = smart_resize(hparams, original_size); @@ -1083,15 +1083,15 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf const int tile_size = hparams.preproc_tile_size; if (!needs_tiling) { - inst.refined_size = clip_image_size{0, 0}; - inst.grid_size = clip_image_size{0, 0}; + inst.refined_size = mtmd_image_size{0, 0}; + inst.grid_size = mtmd_image_size{0, 0}; return inst; } - const clip_image_size grid = get_grid_layout(hparams, original_size.height, original_size.width); + const mtmd_image_size grid = get_grid_layout(hparams, original_size.height, original_size.width); inst.grid_size = grid; - inst.refined_size = clip_image_size{tile_size * grid.width, tile_size * grid.height}; + inst.refined_size = mtmd_image_size{tile_size * grid.width, tile_size * grid.height}; LOG_DBG("%s: original size: %d x %d, overview size: %d x %d, refined size: %d x %d, grid size: %d x %d\n", __func__, @@ -1105,7 +1105,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf mtmd_image_preprocessor_llava_uhd::slice_coordinates slice; slice.x = col * tile_size; slice.y = row * tile_size; - slice.size = clip_image_size{tile_size, tile_size}; + slice.size = mtmd_image_size{tile_size, tile_size}; inst.slices.push_back(slice); LOG_DBG("%s: slice %d: x=%d, y=%d, size=%d x %d\n", __func__, (int)inst.slices.size() - 1, @@ -1117,13 +1117,13 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf } // double precision as in Python, float breaks exact ties such as 720/1600 between 2/4 and 2/5 -clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio( - const clip_hparams & hparams, +mtmd_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio( + const mtmd_hparams & hparams, double aspect_ratio, - const std::vector & target_ratios, + const std::vector & target_ratios, int width, int height) { double best_ratio_diff = std::numeric_limits::max(); - clip_image_size best_ratio = {1, 1}; + mtmd_image_size best_ratio = {1, 1}; const double area = static_cast(width) * height; for (const auto & ratio : target_ratios) { @@ -1143,10 +1143,10 @@ clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio( return best_ratio; } -std::vector mtmd_image_preprocessor_lfm2::get_target_ratios(const clip_hparams & hparams) { +std::vector mtmd_image_preprocessor_lfm2::get_target_ratios(const mtmd_hparams & hparams) { const int min_tiles = hparams.preproc_min_tiles; const int max_tiles = hparams.preproc_max_tiles; - std::vector ratios; + std::vector ratios; for (int n = min_tiles; n <= max_tiles; n++) { for (int w = 1; w <= n; w++) { for (int h = 1; h <= n; h++) { @@ -1165,13 +1165,13 @@ std::vector mtmd_image_preprocessor_lfm2::get_target_ratios(con } } } - std::sort(ratios.begin(), ratios.end(), [](const clip_image_size & a, const clip_image_size & b) { + std::sort(ratios.begin(), ratios.end(), [](const mtmd_image_size & a, const mtmd_image_size & b) { return a.width * a.height < b.width * b.height; }); return ratios; } -clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(const clip_hparams & hparams, int height, int width) { +mtmd_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(const mtmd_hparams & hparams, int height, int width) { const double aspect_ratio = static_cast(width) / height; const auto ratios = get_target_ratios(hparams); return find_closest_aspect_ratio(hparams, aspect_ratio, ratios, width, height); @@ -1181,7 +1181,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(const clip_hparams // mtmd_image_preprocessor_idefics3 // -mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const mtmd_image_u8 & img) const { // The refined size has two steps: // 1. Resize w/ aspect-ratio preserving such that the longer side is // the preprocessor longest size @@ -1189,7 +1189,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i // multiples of image_size (always rounding up) // // CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737 - const clip_image_size original_size = img.get_size(); + const mtmd_image_size original_size = img.get_size(); // old gguf files have no preprocessor longest size, custom token limits also need the generic size below if (hparams.image_longest_edge > 0 && hparams.image_min_pixels <= 0 && hparams.image_max_pixels <= 0) { @@ -1197,7 +1197,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i const int longest_edge = hparams.image_longest_edge; const double aspect_ratio = (double) original_size.width / original_size.height; - clip_image_size resized_size; + mtmd_image_size resized_size; if (original_size.width >= original_size.height) { resized_size.width = longest_edge; resized_size.height = (int) (longest_edge / aspect_ratio); @@ -1210,21 +1210,21 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i const int grid_x = (resized_size.width + tile_size - 1) / tile_size; const int grid_y = (resized_size.height + tile_size - 1) / tile_size; - const clip_image_size refined_size = clip_image_size{grid_x * tile_size, grid_y * tile_size}; + const mtmd_image_size refined_size = mtmd_image_size{grid_x * tile_size, grid_y * tile_size}; - clip_image_u8 resized_img; + mtmd_image_u8 resized_img; img_tool::resize(img, resized_img, resized_size, hparams.image_resize_algo, PAD_NONE); - clip_image_u8 refined_img; + mtmd_image_u8 refined_img; img_tool::resize(resized_img, refined_img, refined_size, hparams.image_resize_algo, PAD_NONE); - clip_image_u8 overview; + mtmd_image_u8 overview; img_tool::resize(refined_img, overview, {tile_size, tile_size}, hparams.image_resize_algo, PAD_NONE); - std::vector slices; + std::vector slices; for (int y = 0; y < grid_y; y++) { for (int x = 0; x < grid_x; x++) { - clip_image_u8 slice; + mtmd_image_u8 slice; img_tool::crop(refined_img, slice, x * tile_size, y * tile_size, tile_size, tile_size); slices.push_back(std::move(slice)); } @@ -1240,7 +1240,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i return output; } - const clip_image_size refined_size = img_tool::calc_size_preserved_ratio( + const mtmd_image_size refined_size = img_tool::calc_size_preserved_ratio( original_size, { hparams.image_size, std::max(0, hparams.image_min_pixels), std::max(0, hparams.image_max_pixels), hparams.image_longest_edge }); // LOG_INF("%s: original size: %d x %d, refined size: %d x %d\n", @@ -1248,9 +1248,9 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i // refined_size.width, refined_size.height); mtmd_image_preprocessor_llava_uhd::slice_instructions instructions; - instructions.overview_size = clip_image_size{hparams.image_size, hparams.image_size}; + instructions.overview_size = mtmd_image_size{hparams.image_size, hparams.image_size}; instructions.refined_size = refined_size; - instructions.grid_size = clip_image_size{ + instructions.grid_size = mtmd_image_size{ static_cast(std::ceil(static_cast(refined_size.width) / hparams.image_size)), static_cast(std::ceil(static_cast(refined_size.height) / hparams.image_size)), }; @@ -1260,7 +1260,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i instructions.slices.push_back(mtmd_image_preprocessor_llava_uhd::slice_coordinates{ /* x */x, /* y */y, - /* size */clip_image_size{ + /* size */mtmd_image_size{ std::min(hparams.image_size, refined_size.width - x), std::min(hparams.image_size, refined_size.height - y) } @@ -1281,7 +1281,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i // mtmd_image_preprocessor_cohere2v // -mtmd_image_preproc_out mtmd_image_preprocessor_cohere2v::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_cohere2v::preprocess(const mtmd_image_u8 & img) const { const auto inst = get_slice_instructions(img.get_size()); auto sliced = slice_image(img, inst); @@ -1298,21 +1298,21 @@ mtmd_image_preproc_out mtmd_image_preprocessor_cohere2v::preprocess(const clip_i return output; } -mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_cohere2v::get_slice_instructions(const clip_image_size & original_size) const { +mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_cohere2v::get_slice_instructions(const mtmd_image_size & original_size) const { const int tile = hparams.image_size; // pick the grid with the least upscale; if all grids need downscale, pick the one with the least downscale // grids are visited by tile count, then by width, same order as HF for ties double best_down = -1.0; double best_up = std::numeric_limits::max(); - clip_image_size grid_down = { 1, 1 }; - clip_image_size grid_up = { 0, 0 }; + mtmd_image_size grid_down = { 1, 1 }; + mtmd_image_size grid_up = { 0, 0 }; for (int n = 1; n <= hparams.preproc_max_tiles; n++) { for (int w = 1; w <= n; w++) { if (n % w != 0) { continue; } - const clip_image_size g = { w, n / w }; + const mtmd_image_size g = { w, n / w }; const double scale = std::min( (double) (g.width * tile) / original_size.width, (double) (g.height * tile) / original_size.height); @@ -1327,7 +1327,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_co } } } - const clip_image_size grid = grid_up.width > 0 ? grid_up : grid_down; + const mtmd_image_size grid = grid_up.width > 0 ? grid_up : grid_down; slice_instructions inst; inst.overview_size = { tile, tile }; @@ -1347,9 +1347,9 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_co // mtmd_image_preprocessor_internvl // -mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const mtmd_image_u8 & img) const { GGML_ASSERT(!hparams.image_res_candidates.empty()); - const clip_image_size original_size = img.get_size(); + const mtmd_image_size original_size = img.get_size(); auto const inst = get_slice_instructions(original_size); auto sliced = slice_image(img, inst); @@ -1366,8 +1366,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const clip_i // mtmd_image_preprocessor_deepseekocr // -std::vector mtmd_image_preprocessor_deepseekocr::get_target_ratios() const { - std::vector ratios; +std::vector mtmd_image_preprocessor_deepseekocr::get_target_ratios() const { + std::vector ratios; for (int n = min_tiles; n <= max_tiles; n++) { for (int w = 1; w <= n; w++) { for (int h = 1; h <= n; h++) { @@ -1387,19 +1387,19 @@ std::vector mtmd_image_preprocessor_deepseekocr::get_target_rat } } } - std::sort(ratios.begin(), ratios.end(), [](const clip_image_size & a, const clip_image_size & b) { + std::sort(ratios.begin(), ratios.end(), [](const mtmd_image_size & a, const mtmd_image_size & b) { return a.width * a.height < b.width * b.height; }); return ratios; } -clip_image_size mtmd_image_preprocessor_deepseekocr::find_closest_aspect_ratio( +mtmd_image_size mtmd_image_preprocessor_deepseekocr::find_closest_aspect_ratio( float aspect_ratio, - const std::vector & target_ratios, + const std::vector & target_ratios, int width, int height) const { float best_ratio_diff = std::numeric_limits::max(); - clip_image_size best_ratio = { 1, 1 }; + mtmd_image_size best_ratio = { 1, 1 }; const float area = static_cast(width * height); for (const auto & ratio : target_ratios) { @@ -1482,7 +1482,7 @@ void mtmd_image_preprocessor_deepseek4v::safe_resize(int height, int width, int } // ref: load_image() -mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const mtmd_image_u8 & img) const { mtmd_image_preproc_out out; const int p = hparams.patch_size; @@ -1490,7 +1490,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip const int max_n_token = hparams.dsv4_max_n_token; const int max_wh = hparams.dsv4_max_wh_ratio; - const clip_image_size orig = img.get_size(); + const mtmd_image_size orig = img.get_size(); int width = orig.width; int height = orig.height; if (max_wh > 0 && width > height * max_wh) { @@ -1502,11 +1502,11 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip width = (int) (width * up); height = (int) (height * up); } - int best_width = CLIP_ALIGN(width, p); - int best_height = CLIP_ALIGN(height, p); + int best_width = MTMD_ALIGN(width, p); + int best_height = MTMD_ALIGN(height, p); safe_resize(height, width, best_height, best_width, p, r, max_n_token); - clip_image_u8 resized; + mtmd_image_u8 resized; if (max_wh > 0 && orig.width >= max_wh * orig.height) { // extreme aspect ratio: plain stretch resize, no padding img_tool::resize(img, resized, {best_width, best_height}, hparams.image_resize_algo, PAD_NONE); @@ -1540,8 +1540,8 @@ void mtmd_image_preprocessor_deepseek41v::plan_grid(int width, int height, int p width = (int) (width * ratio); height = (int) (height * ratio); } - best_width = CLIP_ALIGN(width, p); - best_height = CLIP_ALIGN(height, p); + best_width = MTMD_ALIGN(width, p); + best_height = MTMD_ALIGN(height, p); if (n_tokens(best_height, best_width) > max_n_token) { // solve_resize_ratio: largest aspect-preserving size whose token block fits const double ratio = (double) height / width; @@ -1563,11 +1563,11 @@ void mtmd_image_preprocessor_deepseek41v::plan_grid(int width, int height, int p } } -mtmd_image_preproc_out mtmd_image_preprocessor_deepseek41v::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_deepseek41v::preprocess(const mtmd_image_u8 & img) const { mtmd_image_preproc_out out; const int p = hparams.patch_size; - const clip_image_size orig = img.get_size(); + const mtmd_image_size orig = img.get_size(); int best_width = 0, best_height = 0; plan_grid(orig.width, orig.height, p, hparams.n_merge, hparams.dsv4_max_n_token, hparams.image_min_pixels, best_width, best_height); @@ -1585,10 +1585,10 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek41v::preprocess(const cli new_w = round_half_even((double) orig.width / orig.height * best_height); } } - clip_image_u8 resized; + mtmd_image_u8 resized; img_tool::resize(img, resized, {new_w, new_h}, hparams.image_resize_algo, PAD_NONE); - clip_image_u8 padded; + mtmd_image_u8 padded; if (new_w == best_width && new_h == best_height) { padded = std::move(resized); } else { @@ -1614,14 +1614,14 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek41v::preprocess(const cli return out; } -mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const mtmd_image_u8 & img) const { mtmd_image_preproc_out output; int grid_w = 0; int grid_h = 0; const auto img_size = img.get_size(); // global view: aspect-preserving fit-and-pad to base_size - clip_image_u8 padded; + mtmd_image_u8 padded; img_tool::resize(img, padded, { base_size, base_size }, RESIZE_ALGO_BICUBIC, @@ -1634,12 +1634,12 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const cli if (img_size.width > tile_size || img_size.height > tile_size) { const float aspect_ratio = static_cast(img_size.width) / img_size.height; const auto target_ratios = get_target_ratios(); - const clip_image_size grid = + const mtmd_image_size grid = find_closest_aspect_ratio(aspect_ratio, target_ratios, img_size.width, img_size.height); grid_w = grid.width; grid_h = grid.height; - clip_image_u8 refined; + mtmd_image_u8 refined; img_tool::resize(img, refined, { tile_size * grid_w, tile_size * grid_h }, RESIZE_ALGO_BICUBIC, PAD_NONE); @@ -1648,7 +1648,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const cli // concat all tiles in this row into a single image, along the H axis // output image size: w = tile_size, h = tile_size * grid_w // this is to ensure the whole row is always processed together - clip_image_u8 row_img; + mtmd_image_u8 row_img; row_img.set_size({tile_size, tile_size * grid_w}, false); for (int col = 0; col < grid_w; col++) { for (int py = 0; py < tile_size; py++) { @@ -1661,7 +1661,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const cli output.append(hparams, row_img, true); } else { for (int col = 0; col < grid_w; col++) { - clip_image_u8 tile; + mtmd_image_u8 tile; img_tool::crop(refined, tile, col * tile_size, row * tile_size, tile_size, tile_size); output.append(hparams, tile, true); } @@ -1685,8 +1685,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const cli // void mtmd_image_preprocessor_step3vl::img_u8_resize_bilinear_to_f32( - const clip_image_u8 & src, - clip_image_f32 & dst, + const mtmd_image_u8 & src, + mtmd_image_f32 & dst, int target_width, int target_height, const float mean[3], @@ -1745,11 +1745,11 @@ void mtmd_image_preprocessor_step3vl::img_u8_resize_bilinear_to_f32( dst.cpy_buf(local_buf); } -int mtmd_image_preprocessor_step3vl::get_image_longest_edge(const clip_hparams & params) { +int mtmd_image_preprocessor_step3vl::get_image_longest_edge(const mtmd_hparams & params) { return params.image_longest_edge > 0 ? params.image_longest_edge : default_image_longest_edge; } -int mtmd_image_preprocessor_step3vl::determine_window_size(const clip_hparams & params, int longer, int shorter) { +int mtmd_image_preprocessor_step3vl::determine_window_size(const mtmd_hparams & params, int longer, int shorter) { const int image_size = params.image_size; const int crop_size = default_image_crop_size; const float aspect_ratio = static_cast(longer) / shorter; @@ -1791,15 +1791,15 @@ std::vector mtmd_image_preprocessor_step3vl::calc_grid(int length, int wind return starts; } -clip_image_u8 mtmd_image_preprocessor_step3vl::prepare_image(const clip_image_u8 & img, const clip_hparams & params) { - clip_image_u8 resized = img; +mtmd_image_u8 mtmd_image_preprocessor_step3vl::prepare_image(const mtmd_image_u8 & img, const mtmd_hparams & params) { + mtmd_image_u8 resized = img; const auto img_size = img.get_size(); const float aspect_ratio = img_size.height > 0 ? static_cast(img_size.width) / img_size.height : 1.0f; if (std::min(img_size.width, img_size.height) < 32 && (aspect_ratio > wide_aspect_ratio_limit || aspect_ratio < 1.0f / wide_aspect_ratio_limit)) { const int square_size = std::max(img_size.width, img_size.height); - clip_image_u8 padded; + mtmd_image_u8 padded; padded.set_size({square_size, square_size}, false); img_tool::fill(padded, {0, 0, 0}); img_tool::composite(padded, img, 0, 0); @@ -1810,11 +1810,11 @@ clip_image_u8 mtmd_image_preprocessor_step3vl::prepare_image(const clip_image_u8 const auto resized_size = resized.get_size(); if (std::max(resized_size.width, resized_size.height) > max_image_size) { const float scale = static_cast(max_image_size) / std::max(resized_size.width, resized_size.height); - const clip_image_size new_size = { + const mtmd_image_size new_size = { std::max(1, static_cast(std::floor(resized_size.width * scale))), std::max(1, static_cast(std::floor(resized_size.height * scale))), }; - clip_image_u8 scaled; + mtmd_image_u8 scaled; img_tool::resize(resized, scaled, new_size, RESIZE_ALGO_BILINEAR, PAD_NONE); resized = std::move(scaled); } @@ -1822,8 +1822,8 @@ clip_image_u8 mtmd_image_preprocessor_step3vl::prepare_image(const clip_image_u8 return resized; } -clip_image_u8 mtmd_image_preprocessor_step3vl::crop_with_black_padding(const clip_image_u8 & image, int x, int y, int w, int h) { - clip_image_u8 dst; +mtmd_image_u8 mtmd_image_preprocessor_step3vl::crop_with_black_padding(const mtmd_image_u8 & image, int x, int y, int w, int h) { + mtmd_image_u8 dst; dst.set_size({w, h}, false); img_tool::fill(dst, {0, 0, 0}); @@ -1850,8 +1850,8 @@ clip_image_u8 mtmd_image_preprocessor_step3vl::crop_with_black_padding(const cli } mtmd_image_preprocessor_step3vl::slice_instructions mtmd_image_preprocessor_step3vl::build_slice_instructions( - const clip_hparams & params, - const clip_image_size & prepared_size) { + const mtmd_hparams & params, + const mtmd_image_size & prepared_size) { slice_instructions instructions; instructions.overview_size = prepared_size; @@ -1860,18 +1860,18 @@ mtmd_image_preprocessor_step3vl::slice_instructions mtmd_image_preprocessor_step std::max(prepared_size.width, prepared_size.height), std::min(prepared_size.width, prepared_size.height)); if (window_size <= 0) { - instructions.refined_size = clip_image_size{0, 0}; - instructions.grid_size = clip_image_size{0, 0}; + instructions.refined_size = mtmd_image_size{0, 0}; + instructions.grid_size = mtmd_image_size{0, 0}; return instructions; } const int crop_width = calc_crop_extent(prepared_size.width, window_size); const int crop_height = calc_crop_extent(prepared_size.height, window_size); - instructions.refined_size = clip_image_size{crop_width, crop_height}; + instructions.refined_size = mtmd_image_size{crop_width, crop_height}; const auto xs = calc_grid(crop_width, window_size); const auto ys = calc_grid(crop_height, window_size); - instructions.grid_size = clip_image_size{ + instructions.grid_size = mtmd_image_size{ static_cast(xs.size()), static_cast(ys.size()), }; @@ -1881,7 +1881,7 @@ mtmd_image_preprocessor_step3vl::slice_instructions mtmd_image_preprocessor_step instructions.slices.push_back(slice_coordinates{ /* x */ x, /* y */ y, - /* size */ clip_image_size{window_size, window_size}, + /* size */ mtmd_image_size{window_size, window_size}, }); } } @@ -1889,8 +1889,8 @@ mtmd_image_preprocessor_step3vl::slice_instructions mtmd_image_preprocessor_step return instructions; } -mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_image_u8 & img) const { - clip_image_u8 prepared = prepare_image(img, hparams); +mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const mtmd_image_u8 & img) const { + mtmd_image_u8 prepared = prepare_image(img, hparams); const auto instructions = build_slice_instructions(hparams, prepared.get_size()); mtmd_image_preproc_out output; @@ -1909,10 +1909,10 @@ mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_im return output; } - clip_image_u8 img_for_crop = prepared; + mtmd_image_u8 img_for_crop = prepared; const auto prepared_size = prepared.get_size(); if (instructions.refined_size.width != prepared_size.width || instructions.refined_size.height != prepared_size.height) { - clip_image_u8 refined; + mtmd_image_u8 refined; img_tool::resize(prepared, refined, instructions.refined_size, RESIZE_ALGO_BILINEAR, PAD_NONE); img_for_crop = std::move(refined); } @@ -1920,9 +1920,9 @@ mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_im const int crop_size = default_image_crop_size; for (const auto & slice : instructions.slices) { // If the requested patch extends past the source image, pad the out-of-bounds area with black. - clip_image_u8 patch = crop_with_black_padding(img_for_crop, slice.x, slice.y, slice.size.width, slice.size.height); + mtmd_image_u8 patch = crop_with_black_padding(img_for_crop, slice.x, slice.y, slice.size.width, slice.size.height); - clip_image_f32 patch_f32; + mtmd_image_f32 patch_f32; img_u8_resize_bilinear_to_f32( patch, patch_f32, @@ -1943,7 +1943,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_im // mtmd_image_preprocessor_youtuvl // -mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const mtmd_image_u8 & img) const { const int patch_size = hparams.patch_size; // typically 16 const int merge_size = hparams.n_merge; // typically 2 const int align_size = patch_size * merge_size; // 32 @@ -1981,10 +1981,10 @@ mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_im } } - clip_image_size new_size = {target_width, target_height}; + mtmd_image_size new_size = {target_width, target_height}; // Resize the image - clip_image_u8 resized; + mtmd_image_u8 resized; img_tool::resize(img, resized, new_size, hparams.image_resize_algo, hparams.image_resize_pad); mtmd_image_preproc_out output; @@ -1992,33 +1992,33 @@ mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_im return output; } -mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const mtmd_image_u8 & img) const { GGML_ASSERT(!hparams.image_res_candidates.empty()); - const clip_image_size orig_size = img.get_size(); + const mtmd_image_size orig_size = img.get_size(); const int tile_size = hparams.image_size; GGML_ASSERT(tile_size > 0); // llava-next always encodes an overview plus a grid of tiles, even for small images - const clip_image_size refined_size = select_best_resolution(orig_size, hparams.image_res_candidates); + const mtmd_image_size refined_size = select_best_resolution(orig_size, hparams.image_res_candidates); const int grid_x = refined_size.width / tile_size; const int grid_y = refined_size.height / tile_size; // the tiles are stacked on the Y axis, a big grid overflows the stacked image height GGML_ASSERT(grid_x >= 0 && grid_x <= 1024 && grid_y >= 0 && grid_y <= 1024); - clip_image_u8 overview; + mtmd_image_u8 overview; img_tool::resize(img, overview, {tile_size, tile_size}, hparams.image_resize_algo_ov, hparams.image_pad_ov, hparams.image_pad_color_ov); - clip_image_u8 refined; + mtmd_image_u8 refined; img_tool::resize(img, refined, refined_size, hparams.image_resize_algo_rf, hparams.image_pad_rf, hparams.image_pad_color_rf); // stack the overview and the tiles on the Y axis, so the whole grid goes through one graph - clip_image_u8 stacked; + mtmd_image_u8 stacked; stacked.set_size({tile_size, tile_size * (1 + grid_x * grid_y)}, false); - auto copy_tile = [&](const clip_image_u8 & src, int src_x, int src_y, int dst_idx) { + auto copy_tile = [&](const mtmd_image_u8 & src, int src_x, int src_y, int dst_idx) { for (int py = 0; py < tile_size; py++) { for (int px = 0; px < tile_size; px++) { stacked.set_pixel(px, dst_idx * tile_size + py, src.get_pixel(src_x + px, src_y + py)); @@ -2049,7 +2049,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_im // // Replicates transformers' get_aspect_ratio_preserving_size -static clip_image_size muse_glimmer_grid_size(int img_w, int img_h, int patch_hw, int max_tokens) { +static mtmd_image_size muse_glimmer_grid_size(int img_w, int img_h, int patch_hw, int max_tokens) { double i_nph = (double) img_h / patch_hw; double i_npw = (double) img_w / patch_hw; const double ratio = i_nph > 0.0 ? i_npw / i_nph : 1.0; @@ -2084,24 +2084,140 @@ static clip_image_size muse_glimmer_grid_size(int img_w, int img_h, int patch_hw best_nph = std::max(1, (int) std::lround(i_nph)); best_npw = std::max(1, (int) std::lround(i_npw)); } - return clip_image_size{ best_npw * patch_hw, best_nph * patch_hw }; + return mtmd_image_size{ best_npw * patch_hw, best_nph * patch_hw }; } -mtmd_image_preproc_out mtmd_image_preprocessor_muse_glimmer::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_muse_glimmer::preprocess(const mtmd_image_u8 & img) const { const int patch_hw = hparams.patch_size * hparams.n_merge; const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; GGML_ASSERT(patch_area > 0 && hparams.image_max_pixels > 0); const int max_tokens = hparams.image_max_pixels / patch_area; - const clip_image_size original_size = img.get_size(); - const clip_image_size target_size = muse_glimmer_grid_size( + const mtmd_image_size original_size = img.get_size(); + const mtmd_image_size target_size = muse_glimmer_grid_size( original_size.width, original_size.height, patch_hw, max_tokens); // PIL resizes directly to (target_w, target_h) -- a stretch, no padding. - clip_image_u8 resized_image; + mtmd_image_u8 resized_image; img_tool::resize(img, resized_image, target_size, hparams.image_resize_algo, PAD_NONE); mtmd_image_preproc_out output; output.append(hparams, resized_image, true); return output; } + +//#define MTMD_DEBUG_FUNCTIONS + +#ifdef MTMD_DEBUG_FUNCTIONS +static void mtmd_image_write_image_to_ppm(const mtmd_image_u8& img, const std::string& filename) { + std::ofstream file(filename, std::ios::binary); + if (!file.is_open()) { + LOG_ERR("Failed to open file for writing: %s\n", filename.c_str()); + return; + } + + // PPM header: P6 format, width, height, and max color value + const auto ppm_size = img.get_size(); + file << "P6\n" << ppm_size.width << " " << ppm_size.height << "\n255\n"; + + // Write pixel data + const auto & ppm_buf = img.get_ro_buf(); + for (size_t i = 0; i < ppm_buf.size(); i += 3) { + // PPM expects binary data in RGB format, which matches our image buffer + file.write(reinterpret_cast(&ppm_buf[i]), 3); + } + + file.close(); +} + +static void mtmd_image_save_to_bmp(const mtmd_image_u8& img, const std::string& filename) { + std::ofstream file(filename, std::ios::binary); + if (!file.is_open()) { + LOG_ERR("Failed to open file for writing: %s\n", filename.c_str()); + return; + } + + const auto bmp_size = img.get_size(); + int fileSize = 54 + 3 * bmp_size.width * bmp_size.height; // File header + info header + pixel data + int bytesPerPixel = 3; + int widthInBytes = bmp_size.width * bytesPerPixel; + int paddingAmount = (4 - (widthInBytes % 4)) % 4; + int stride = widthInBytes + paddingAmount; + + // Bitmap file header + unsigned char fileHeader[14] = { + 'B','M', // Signature + 0,0,0,0, // Image file size in bytes + 0,0,0,0, // Reserved + 54,0,0,0 // Start of pixel array + }; + + // Total file size + fileSize = 54 + (stride * bmp_size.height); + fileHeader[2] = (unsigned char)(fileSize); + fileHeader[3] = (unsigned char)(fileSize >> 8); + fileHeader[4] = (unsigned char)(fileSize >> 16); + fileHeader[5] = (unsigned char)(fileSize >> 24); + + // Bitmap information header (BITMAPINFOHEADER) + unsigned char infoHeader[40] = { + 40,0,0,0, // Size of this header (40 bytes) + 0,0,0,0, // Image width + 0,0,0,0, // Image height + 1,0, // Number of color planes + 24,0, // Bits per pixel + 0,0,0,0, // No compression + 0,0,0,0, // Image size (can be 0 for no compression) + 0,0,0,0, // X pixels per meter (not specified) + 0,0,0,0, // Y pixels per meter (not specified) + 0,0,0,0, // Total colors (color table not used) + 0,0,0,0 // Important colors (all are important) + }; + + // Width and height in the information header + infoHeader[4] = (unsigned char)(bmp_size.width); + infoHeader[5] = (unsigned char)(bmp_size.width >> 8); + infoHeader[6] = (unsigned char)(bmp_size.width >> 16); + infoHeader[7] = (unsigned char)(bmp_size.width >> 24); + infoHeader[8] = (unsigned char)(bmp_size.height); + infoHeader[9] = (unsigned char)(bmp_size.height >> 8); + infoHeader[10] = (unsigned char)(bmp_size.height >> 16); + infoHeader[11] = (unsigned char)(bmp_size.height >> 24); + + // Write file headers + file.write(reinterpret_cast(fileHeader), sizeof(fileHeader)); + file.write(reinterpret_cast(infoHeader), sizeof(infoHeader)); + + // Pixel data + std::vector padding(3, 0); // Max padding size to be added to each row + for (int y = bmp_size.height - 1; y >= 0; --y) { // BMP files are stored bottom-to-top + for (int x = 0; x < bmp_size.width; ++x) { + // Each pixel + const auto px = img.get_pixel(x, y); + unsigned char pixel[3] = { + px[2], // BMP stores pixels in BGR format + px[1], + px[0] + }; + file.write(reinterpret_cast(pixel), 3); + } + // Write padding for the row + file.write(reinterpret_cast(padding.data()), paddingAmount); + } + + file.close(); +} + +// debug function to convert f32 to u8 +static void mtmd_image_convert_f32_to_u8(const mtmd_image_f32& src, mtmd_image_u8& dst) { + dst.set_size(src.get_size(), false); + const auto & src_buf = src.get_ro_buf(); + std::vector dst_buf(src.n_elements()); + for (size_t i = 0; i < src.n_elements(); ++i) { + dst_buf[i] = static_cast(std::min(std::max(int(src_buf[i] * 255.0f), 0), 255)); + } + dst.cpy_buf(dst_buf); +} +#endif + + diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index 6eb49ca34..2cb9454f0 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -1,7 +1,7 @@ #pragma once #include "ggml.h" -#include "clip-model.h" +#include "mtmd-model.h" #include #include @@ -9,18 +9,18 @@ #define MTMD_INTERNAL_HEADER struct mtmd_image_preproc_out { - std::vector entries; + std::vector entries; // grid size is required for llava-uhd style models - clip_image_f32 overview; // overview image (downscaled image) + mtmd_image_f32 overview; // overview image (downscaled image) int grid_x = 0; int grid_y = 0; - void append(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized = true); - void append(const clip_hparams & hparams, const std::vector & imgs, bool normalized = true); - void append(const clip_hparams & hparams, clip_image_f32 & img, bool normalized = true); + void append(const mtmd_hparams & hparams, const mtmd_image_u8 & img, bool normalized = true); + void append(const mtmd_hparams & hparams, const std::vector & imgs, bool normalized = true); + void append(const mtmd_hparams & hparams, mtmd_image_f32 & img, bool normalized = true); - void append_overview(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized = true); + void append_overview(const mtmd_hparams & hparams, const mtmd_image_u8 & img, bool normalized = true); bool has_overview() const { return overview.nx() > 0 || overview.ny() > 0; } @@ -28,17 +28,17 @@ struct mtmd_image_preproc_out { // base class, models must inherit from this class struct mtmd_image_preprocessor { - const clip_hparams & hparams; + const mtmd_hparams & hparams; - mtmd_image_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {} - mtmd_image_preprocessor(const clip_hparams & hparams): hparams(hparams) {} + mtmd_image_preprocessor(const mtmd_model * model): hparams(model->hparams) {} + mtmd_image_preprocessor(const mtmd_hparams & hparams): hparams(hparams) {} virtual ~mtmd_image_preprocessor() = default; - virtual mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const = 0; + virtual mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const = 0; // frames of a video, which the video helper may have already sized - virtual mtmd_image_preproc_out preprocess_video_frame(const clip_image_u8 & img) const { return preprocess(img); } + virtual mtmd_image_preproc_out preprocess_video_frame(const mtmd_image_u8 & img) const { return preprocess(img); } // frame size of a video with n_frames sampled frames, as the reference video processor; false if none - virtual bool get_video_frame_size(int /*n_frames*/, int /*width*/, int /*height*/, clip_image_size & /*out*/) const { return false; } + virtual bool get_video_frame_size(int /*n_frames*/, int /*width*/, int /*height*/, mtmd_image_size & /*out*/) const { return false; } }; /** @@ -63,30 +63,30 @@ struct mtmd_image_preprocessor { * NOTE: for the ordering of overview, set "ov_img_first" on the mtmd_context */ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor { - mtmd_image_preprocessor_llava_uhd(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preprocessor_llava_uhd(const clip_hparams & hparams) : mtmd_image_preprocessor(hparams) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_llava_uhd(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preprocessor_llava_uhd(const mtmd_hparams & hparams) : mtmd_image_preprocessor(hparams) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; struct slice_coordinates { int x; int y; - clip_image_size size; + mtmd_image_size size; }; struct slice_instructions { - clip_image_size overview_size; // size of downscaled image - clip_image_size refined_size; // size of image right before slicing (must be multiple of slice size) - clip_image_size grid_size; // grid_size.width * grid_size.height = number of slices + mtmd_image_size overview_size; // size of downscaled image + mtmd_image_size refined_size; // size of image right before slicing (must be multiple of slice size) + mtmd_image_size grid_size; // grid_size.width * grid_size.height = number of slices std::vector slices; }; - virtual slice_instructions get_slice_instructions(const clip_image_size & original_size) const; + virtual slice_instructions get_slice_instructions(const mtmd_image_size & original_size) const; struct slice_output { - clip_image_u8 overview; - std::vector slices; + mtmd_image_u8 overview; + std::vector slices; }; - slice_output slice_image(const clip_image_u8 & img, const slice_instructions & inst) const; + slice_output slice_image(const mtmd_image_u8 & img, const slice_instructions & inst) const; protected: // align slices to a multiple of the merger factor (integer merger tokens per slice) @@ -100,7 +100,7 @@ protected: return static_cast(std::round(v)); } - clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false) const; + mtmd_image_size get_best_resize(const mtmd_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false) const; /** * Selects the best resolution from a list of possible resolutions based on the original size. @@ -117,68 +117,68 @@ protected: * @param possible_resolutions A list of possible resolutions * @return The best fit resolution */ - clip_image_size select_best_resolution(const clip_image_size & original_size, const std::vector & possible_resolutions) const; + mtmd_image_size select_best_resolution(const mtmd_image_size & original_size, const std::vector & possible_resolutions) const; private: - clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max) const; + mtmd_image_size resize_maintain_aspect_ratio(const mtmd_image_size & orig, const mtmd_image_size & target_max) const; int ensure_divide(int length, int patch_size) const; - clip_image_size get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false) const; - clip_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const; + mtmd_image_size get_refine_size(const mtmd_image_size & original_size, const mtmd_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false) const; + mtmd_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const; }; // downscale or upscale the input image to fixed size struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor { - mtmd_image_preprocessor_fixed_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_fixed_size(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; // resize image to multiple of patch_size*n_merge, while preserving aspect ratio // if image_resize_pad is true, the resized image will be padded, otherwise it will be either stretched or center-cropped depending on image_resize_pad // this is used by models with native support for dynamic image size, for example: Qwen-VL, Pixtral, Kimi-VL, etc struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor { - mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preprocessor_dyn_size(const clip_hparams & hparams) : mtmd_image_preprocessor(hparams) {} - static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size); - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; - virtual clip_image_size target_size(const clip_image_size & original_size) const { return get_target_size(hparams, original_size); } + mtmd_image_preprocessor_dyn_size(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preprocessor_dyn_size(const mtmd_hparams & hparams) : mtmd_image_preprocessor(hparams) {} + static mtmd_image_size get_target_size(const mtmd_hparams & hparams, const mtmd_image_size & original_size); + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; + virtual mtmd_image_size target_size(const mtmd_image_size & original_size) const { return get_target_size(hparams, original_size); } }; // Qwen3-VL, same as dyn_size for images, video frames are sized as Qwen3VLVideoProcessor struct mtmd_image_preprocessor_qwen3vl : mtmd_image_preprocessor_dyn_size { using mtmd_image_preprocessor_dyn_size::mtmd_image_preprocessor_dyn_size; // the video helper sizes the frames (get_video_frame_size), they are only snapped to the grid - mtmd_image_preproc_out preprocess_video_frame(const clip_image_u8 & img) const override; - bool get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const override; + mtmd_image_preproc_out preprocess_video_frame(const mtmd_image_u8 & img) const override; + bool get_video_frame_size(int n_frames, int width, int height, mtmd_image_size & out) const override; - mtmd_image_preproc_out resize_to(const clip_image_u8 & img, const clip_image_size & size) const; + mtmd_image_preproc_out resize_to(const mtmd_image_u8 & img, const mtmd_image_size & size) const; // with temporal_factor > 1 it is the Qwen3-VL video variant over n_frames - static clip_image_size smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels, + static mtmd_image_size smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels, int n_frames = 1, int temporal_factor = 1); }; // LFM3-VL, same as qwen3vl but images are sized exactly as the transformers Qwen2-VL smart_resize struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_qwen3vl { using mtmd_image_preprocessor_qwen3vl::mtmd_image_preprocessor_qwen3vl; - clip_image_size target_size(const clip_image_size & original_size) const override; + mtmd_image_size target_size(const mtmd_image_size & original_size) const override; }; // GLM 5.3 flash, similar to dyn_size, but each edge is aligned up to patch_size*n_merge, and max budget is met by a search over the height with the width scaled proportionally struct mtmd_image_preprocessor_glm5v : mtmd_image_preprocessor { - mtmd_image_preprocessor_glm5v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_glm5v(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; // similar to mtmd_image_preprocessor_dyn_size, but cap the longest edge at hparams.image_longest_edge, while preserving aspect ratio struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor { - mtmd_image_preprocessor_longest_edge(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size); - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_longest_edge(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + static mtmd_image_size get_target_size(const mtmd_hparams & hparams, const mtmd_image_size & original_size); + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; // ref: inference/image_processor.py in the HF repo (DeepSeek-V4-Flash-Vision) struct mtmd_image_preprocessor_deepseek4v : mtmd_image_preprocessor { - mtmd_image_preprocessor_deepseek4v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_deepseek4v(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; private: struct grid_info { @@ -193,8 +193,8 @@ private: // ref: inference/image_processor.py in the HF repo (DeepSeek-V4.1-Flash): plan_image_grid / load_image struct mtmd_image_preprocessor_deepseek41v : mtmd_image_preprocessor { - mtmd_image_preprocessor_deepseek41v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_deepseek41v(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; // target pixel size (multiples of patch_size) for an image of the given size, exposed for tests static void plan_grid(int width, int height, int p, int r, int max_n_token, int min_pixels, int & best_width, int & best_height); @@ -203,7 +203,7 @@ struct mtmd_image_preprocessor_deepseek41v : mtmd_image_preprocessor { // custom llava-uhd slicing logic for MiniCPM-V struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd { using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd; - slice_instructions get_slice_instructions(const clip_image_size & original_size) const override; + slice_instructions get_slice_instructions(const mtmd_image_size & original_size) const override; protected: // always patch_size * 4, even in 4x mode (the 2x2 vit_merger slot stays) @@ -242,50 +242,50 @@ protected: // ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd { using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd; - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; - slice_instructions get_slice_instructions(const clip_image_size & original_size) const override; + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; + slice_instructions get_slice_instructions(const mtmd_image_size & original_size) const override; - static bool should_tile(const clip_hparams & hparams, const clip_image_size & original_size); - static clip_image_size smart_resize(const clip_hparams & hparams, const clip_image_size & original_size); - static clip_image_size get_grid_layout(const clip_hparams & hparams, int height, int width); + static bool should_tile(const mtmd_hparams & hparams, const mtmd_image_size & original_size); + static mtmd_image_size smart_resize(const mtmd_hparams & hparams, const mtmd_image_size & original_size); + static mtmd_image_size get_grid_layout(const mtmd_hparams & hparams, int height, int width); private: - static clip_image_size find_closest_aspect_ratio( - const clip_hparams & hparams, + static mtmd_image_size find_closest_aspect_ratio( + const mtmd_hparams & hparams, double aspect_ratio, - const std::vector & target_ratios, + const std::vector & target_ratios, int width, int height); - static std::vector get_target_ratios(const clip_hparams & hparams); + static std::vector get_target_ratios(const mtmd_hparams & hparams); }; struct mtmd_image_preprocessor_idefics3 : mtmd_image_preprocessor_llava_uhd { - mtmd_image_preprocessor_idefics3(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_idefics3(const mtmd_model * model) : mtmd_image_preprocessor_llava_uhd(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; struct mtmd_image_preprocessor_internvl : mtmd_image_preprocessor_llava_uhd { - mtmd_image_preprocessor_internvl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_internvl(const mtmd_model * model) : mtmd_image_preprocessor_llava_uhd(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; // stretch the image to a grid of tiles, add a thumbnail if there is more than 1 tile // ref: https://github.com/huggingface/transformers/blob/main/src/transformers/models/cohere2_vision/image_processing_cohere2_vision.py struct mtmd_image_preprocessor_cohere2v : mtmd_image_preprocessor_llava_uhd { - mtmd_image_preprocessor_cohere2v(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; - slice_instructions get_slice_instructions(const clip_image_size & original_size) const override; + mtmd_image_preprocessor_cohere2v(const mtmd_model * model) : mtmd_image_preprocessor_llava_uhd(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; + slice_instructions get_slice_instructions(const mtmd_image_size & original_size) const override; }; // DeepSeek-OCR (v1/v2) global view + optional local tile grid struct mtmd_image_preprocessor_deepseekocr : mtmd_image_preprocessor { - mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx) - : mtmd_image_preprocessor(ctx), - fuse_row(clip_get_projector_type(ctx) == PROJECTOR_TYPE_DEEPSEEKOCR), + mtmd_image_preprocessor_deepseekocr(const mtmd_model * model) + : mtmd_image_preprocessor(model), + fuse_row(model->proj_type == PROJECTOR_TYPE_DEEPSEEKOCR), base_size(hparams.image_size), tile_size(hparams.preproc_tile_size), min_tiles(hparams.preproc_min_tiles), max_tiles(hparams.preproc_max_tiles) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; private: bool fuse_row; // v1 fuses a tile-row into one image; v2 keeps tiles separate @@ -294,19 +294,19 @@ private: int min_tiles; int max_tiles; - std::vector get_target_ratios() const; - clip_image_size find_closest_aspect_ratio( + std::vector get_target_ratios() const; + mtmd_image_size find_closest_aspect_ratio( float aspect_ratio, - const std::vector & target_ratios, + const std::vector & target_ratios, int width, int height) const; }; // custom image preprocessing for Step3VL // ref: https://huggingface.co/stepfun-ai/Step3-VL-10B/blob/main/processing_step3.py struct mtmd_image_preprocessor_step3vl : mtmd_image_preprocessor_llava_uhd { - mtmd_image_preprocessor_step3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; - static slice_instructions build_slice_instructions(const clip_hparams & params, const clip_image_size & prepared_size); + mtmd_image_preprocessor_step3vl(const mtmd_model * model) : mtmd_image_preprocessor_llava_uhd(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; + static slice_instructions build_slice_instructions(const mtmd_hparams & params, const mtmd_image_size & prepared_size); private: static constexpr int default_image_longest_edge = 3024; @@ -316,33 +316,33 @@ private: static constexpr float crop_rounding_threshold = 0.2f; void img_u8_resize_bilinear_to_f32( - const clip_image_u8 & src, - clip_image_f32 & dst, + const mtmd_image_u8 & src, + mtmd_image_f32 & dst, int target_width, int target_height, const float mean[3], const float std[3]) const; - static int get_image_longest_edge(const clip_hparams & params); - static int determine_window_size(const clip_hparams & params, int longer, int shorter); + static int get_image_longest_edge(const mtmd_hparams & params); + static int determine_window_size(const mtmd_hparams & params, int longer, int shorter); static int calc_crop_extent(int length, int window_size); static std::vector calc_grid(int length, int window_size); - static clip_image_u8 prepare_image(const clip_image_u8 & img, const clip_hparams & params); - static clip_image_u8 crop_with_black_padding(const clip_image_u8 & image, int x, int y, int w, int h); + static mtmd_image_u8 prepare_image(const mtmd_image_u8 & img, const mtmd_hparams & params); + static mtmd_image_u8 crop_with_black_padding(const mtmd_image_u8 & image, int x, int y, int w, int h); }; struct mtmd_image_preprocessor_youtuvl : mtmd_image_preprocessor { - mtmd_image_preprocessor_youtuvl(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_youtuvl(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; -// llava-next "anyres": stacks the overview and all tiles into one image, assembled by clip in a single graph +// llava-next "anyres": stacks the overview and all tiles into one image, assembled in a single graph struct mtmd_image_preprocessor_granite : mtmd_image_preprocessor_llava_uhd { - mtmd_image_preprocessor_granite(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_granite(const mtmd_model * model) : mtmd_image_preprocessor_llava_uhd(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; // pick the patch grid closest to the input aspect ratio under the per-image token cap, stretch-resize. struct mtmd_image_preprocessor_muse_glimmer : mtmd_image_preprocessor { - mtmd_image_preprocessor_muse_glimmer(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_muse_glimmer(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/mtmd-impl.h similarity index 92% rename from tools/mtmd/clip-impl.h rename to tools/mtmd/mtmd-impl.h index 8a4098da8..e245f4914 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/mtmd-impl.h @@ -3,7 +3,7 @@ #include "ggml.h" #include "ggml-cpp.h" #include "gguf.h" -#include "clip.h" +#include "mtmd.h" #include #include @@ -25,7 +25,7 @@ #include #endif -// Internal header for clip.cpp +// !!! Internal header, to be used by mtmd only !!! #define MTMD_INTERNAL_HEADER @@ -490,42 +490,7 @@ #define TN_A_FUSION_KERNEL "a.fusion.spk_kernel" // align x to upper multiple of n -#define CLIP_ALIGN(x, n) ((((x) + (n) - 1) / (n)) * (n)) - -// sliding window KV cache of a causal audio encoder, see clip_enc_state_init() -struct clip_enc_state { - int32_t n_ctx = 0; // ring slots: attention window + frames per call - int32_t n_past = 0; // absolute position of the next frame - - std::vector slot_pos; // absolute position held by each slot, -1 if empty - std::vector stem_tail; // last conv stem output frame, empty on a cold start - - ggml_context_ptr ctx_data; - ggml_backend_buffer_ptr buf; - std::vector k; // per layer [n_embd_kv, n_ctx] - std::vector v; - - // lfm3a: same order as list_lfm3a_state_slots(), F32 [ne0, ne1], updated in place - std::vector slots; - - // nemotron transcribe: the window's normalized log-mel, cache and FIFO rows in; its speaker rows and predictions out - std::vector mel_norm; - std::vector spk_prev; - std::vector spk_chunk, spk_preds; -}; - -// per-sequence state of audio generation, see clip_gen_state_init() -struct clip_gen_state { - std::mt19937 rng; - - ggml_context_ptr ctx_data; - ggml_backend_buffer_ptr buf; - std::vector slots; // same order as list_gen_state_slots(), F32 [ne0, ne1] -}; - -// forward declaration -// TODO: improve this later -struct clip_ctx; +#define MTMD_ALIGN(x, n) ((((x) + (n) - 1) / (n)) * (n)) enum projector_type { PROJECTOR_TYPE_MLP, @@ -555,7 +520,7 @@ enum projector_type { PROJECTOR_TYPE_QWEN2A, PROJECTOR_TYPE_QWEN3A, PROJECTOR_TYPE_GLMA, - PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on clip_ctx + PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on mtmd_subcontext PROJECTOR_TYPE_VOXTRAL, PROJECTOR_TYPE_VOXTRAL_RT, PROJECTOR_TYPE_MERALION, @@ -699,7 +664,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_CHATTERBOX_SPKENC, "chatterbox_spkenc"}, }; -static projector_type clip_projector_type_from_string(const std::string & str) { +static projector_type projector_type_from_string(const std::string & str) { for (const auto & pair : PROJECTOR_TYPE_NAMES) { if (pair.second == str) { return pair.first; @@ -708,13 +673,30 @@ static projector_type clip_projector_type_from_string(const std::string & str) { return PROJECTOR_TYPE_UNKNOWN; } +struct mtmd_image_size { + int width; + int height; + bool operator==(const mtmd_image_size & other) const { + return width == other.width && height == other.height; + } + bool operator!=(const mtmd_image_size & other) const { + return !(*this == other); + } + int area() const { + // avoid overflow when computing area + GGML_ASSERT(width >= 0 && width <= 46000); + GGML_ASSERT(height >= 0 && height <= 46000); + return width * height; + } +}; + // RGB uint8 image -struct clip_image_u8 { - clip_image_size get_size() const { +struct mtmd_image_u8 { + mtmd_image_size get_size() const { return { nx, ny }; } - void set_size(clip_image_size size, bool is_placeholder) { + void set_size(mtmd_image_size size, bool is_placeholder) { nx = size.width; ny = size.height; if (is_placeholder) { @@ -730,7 +712,7 @@ struct clip_image_u8 { const std::vector & get_ro_buf() const { if (is_placeholder()) { - throw std::runtime_error("this clip_image_u8 is a placeholder"); + throw std::runtime_error("this mtmd_image_u8 is a placeholder"); } return buf; } @@ -777,20 +759,20 @@ struct clip_image_u8 { struct mtmd_serialization; // forward declaration // separators appended after the image tokens of one entry, as rows of v.tok_embd_sep -enum clip_suffix_type : int32_t { - CLIP_SUFFIX_NONE = 0, +enum mtmd_suffix_type : int32_t { + MTMD_SUFFIX_NONE = 0, // MiniCPM-V 4.7 tiles - CLIP_SUFFIX_MINICPMV_OV, // - CLIP_SUFFIX_MINICPMV_OV_SLICE, // - CLIP_SUFFIX_MINICPMV_SLICE, // - CLIP_SUFFIX_MINICPMV_ROW_END, // \n - CLIP_SUFFIX_MINICPMV_LAST, // - CLIP_SUFFIX_COUNT, + MTMD_SUFFIX_MINICPMV_OV, // + MTMD_SUFFIX_MINICPMV_OV_SLICE, // + MTMD_SUFFIX_MINICPMV_SLICE, // + MTMD_SUFFIX_MINICPMV_ROW_END, // \n + MTMD_SUFFIX_MINICPMV_LAST, // + MTMD_SUFFIX_COUNT, }; // rows of v.tok_embd_sep for each suffix type // MiniCPM-V 4.7 rows (set by the converter): 0 = , 1 = , 2 = , 3 = \n -static inline const std::vector & clip_suffix_rows(clip_suffix_type type) { +static inline const std::vector & mtmd_suffix_rows(mtmd_suffix_type type) { static const std::vector none; static const std::vector minicpmv_ov = { 0 }; static const std::vector minicpmv_ov_slice = { 0, 1 }; @@ -798,12 +780,12 @@ static inline const std::vector & clip_suffix_rows(clip_suffix_type type) { static const std::vector minicpmv_row_end = { 2, 3, 1 }; static const std::vector minicpmv_last = { 2 }; switch (type) { - case CLIP_SUFFIX_NONE: return none; - case CLIP_SUFFIX_MINICPMV_OV: return minicpmv_ov; - case CLIP_SUFFIX_MINICPMV_OV_SLICE: return minicpmv_ov_slice; - case CLIP_SUFFIX_MINICPMV_SLICE: return minicpmv_slice; - case CLIP_SUFFIX_MINICPMV_ROW_END: return minicpmv_row_end; - case CLIP_SUFFIX_MINICPMV_LAST: return minicpmv_last; + case MTMD_SUFFIX_NONE: return none; + case MTMD_SUFFIX_MINICPMV_OV: return minicpmv_ov; + case MTMD_SUFFIX_MINICPMV_OV_SLICE: return minicpmv_ov_slice; + case MTMD_SUFFIX_MINICPMV_SLICE: return minicpmv_slice; + case MTMD_SUFFIX_MINICPMV_ROW_END: return minicpmv_row_end; + case MTMD_SUFFIX_MINICPMV_LAST: return minicpmv_last; default: GGML_ABORT("invalid suffix type"); } } @@ -814,7 +796,7 @@ static inline const std::vector & clip_suffix_rows(clip_suffix_type type) { // Memory layout: RGBRGB...RGBRGB... (nt times) // For audio, only one channel is used, buf.size() == nx*ny // nx will be n_frames and ny will be n_mel -struct clip_image_f32 { +struct mtmd_image_f32 { // marks the global view in e.g., DeepSeek-OCR Models bool add_viewsep = false; // appends a learned newline (or EOI) token after the image @@ -824,7 +806,7 @@ struct clip_image_f32 { // depends on the chunk position, set at tokenize time (see mtmd_tokenizer::add_media) int32_t lead_pad = 0; // separators appended after the image tokens - clip_suffix_type suffix_type = CLIP_SUFFIX_NONE; + mtmd_suffix_type suffix_type = MTMD_SUFFIX_NONE; // tile grid of the image group this entry belongs to // llava-next "anyres" (Granite4 Vision): the whole grid is encoded and assembled in a single graph @@ -841,14 +823,14 @@ struct clip_image_f32 { }; anyres_info anyres; - clip_image_size get_size() const { + mtmd_image_size get_size() const { return { nx_, ny_ }; } int nx() const { return nx_; } int ny() const { return ny_; } - void set_size(clip_image_size size, bool is_placeholder, bool is_audio) { + void set_size(mtmd_image_size size, bool is_placeholder, bool is_audio) { nx_ = size.width; ny_ = size.height; if (is_placeholder) { @@ -866,7 +848,7 @@ struct clip_image_f32 { buf = new_buf; } - void from_u8(const clip_image_u8 & img) { + void from_u8(const mtmd_image_u8 & img) { auto size = img.get_size(); nx_ = size.width; ny_ = size.height; @@ -898,7 +880,7 @@ struct clip_image_f32 { const std::vector & get_ro_buf() const { if (is_placeholder()) { - throw std::runtime_error("this clip_image_f32 is a placeholder"); + throw std::runtime_error("this mtmd_image_f32 is a placeholder"); } return buf; } @@ -924,7 +906,7 @@ struct clip_image_f32 { // token area kept after removing the padding added by the anyres resize // ref: https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/llava_next/modeling_llava_next.py#L109 -static inline void clip_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_h, +static inline void mtmd_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_h, int & off_x, int & off_y, int & out_w, int & out_h) { off_x = 0; off_y = 0; @@ -966,21 +948,21 @@ static inline dsv4_block_layout dsv4_get_block_layout(int n_llm_w, int n_llm_h, // logging // -static void clip_log_callback_default(enum ggml_log_level level, const char * text, void * user_data) { +static void mtmd_log_callback_default(enum ggml_log_level level, const char * text, void * user_data) { (void) level; (void) user_data; fputs(text, stderr); fflush(stderr); } -struct clip_logger_state { +struct mtmd_logger_state { ggml_log_callback log_callback; void * log_callback_user_data; }; -extern struct clip_logger_state g_logger_state; +extern struct mtmd_logger_state g_logger_state; -static void clip_log_internal_v(enum ggml_log_level level, const char * format, va_list args) { +static void mtmd_log_internal_v(enum ggml_log_level level, const char * format, va_list args) { if (format == NULL) { return; } @@ -1000,31 +982,31 @@ static void clip_log_internal_v(enum ggml_log_level level, const char * format, va_end(args_copy); } -static void clip_log_internal(enum ggml_log_level level, const char * format, ...) { +static void mtmd_log_internal(enum ggml_log_level level, const char * format, ...) { va_list args; va_start(args, format); - clip_log_internal_v(level, format, args); + mtmd_log_internal_v(level, format, args); va_end(args); } -#define LOG_TRC(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) -#define LOG_DBG(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) -#define LOG_INF(...) clip_log_internal(GGML_LOG_LEVEL_INFO, __VA_ARGS__) -#define LOG_WRN(...) clip_log_internal(GGML_LOG_LEVEL_WARN, __VA_ARGS__) -#define LOG_ERR(...) clip_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__) -#define LOG_CNT(...) clip_log_internal(GGML_LOG_LEVEL_CONT, __VA_ARGS__) +#define LOG_TRC(...) mtmd_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) +#define LOG_DBG(...) mtmd_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) +#define LOG_INF(...) mtmd_log_internal(GGML_LOG_LEVEL_INFO, __VA_ARGS__) +#define LOG_WRN(...) mtmd_log_internal(GGML_LOG_LEVEL_WARN, __VA_ARGS__) +#define LOG_ERR(...) mtmd_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__) +#define LOG_CNT(...) mtmd_log_internal(GGML_LOG_LEVEL_CONT, __VA_ARGS__) // // cpp wrappers // -struct clip_image_f32_batch { - std::vector entries; +struct mtmd_image_f32_batch { + std::vector entries; bool is_audio = false; bool is_video = false; - clip_image_f32_batch clone() const { - clip_image_f32_batch new_batch{ + mtmd_image_f32_batch clone() const { + mtmd_image_f32_batch new_batch{ /* entries */ {}, /* is_audio */ is_audio, /* is_video */ is_video, @@ -1244,9 +1226,3 @@ static void print_tensor_data(ggml_tensor * t, uint8_t * data, int64_t n) { } } -// -// API used internally with mtmd -// - -projector_type clip_get_projector_type(const struct clip_ctx * ctx); -void clip_set_debug_output_embeddings(struct clip_ctx * ctx, bool debug); diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/mtmd-model.cpp similarity index 57% rename from tools/mtmd/clip.cpp rename to tools/mtmd/mtmd-model.cpp index 0a3b3c98d..ae7a942ac 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/mtmd-model.cpp @@ -1,15 +1,11 @@ -#include "clip.h" -#include "clip-impl.h" -#include "clip-model.h" -#include "clip-graph.h" +#include "mtmd-impl.h" +#include "mtmd-model.h" #include "models/models.h" -#include "mtmd-audio.h" #include "ggml.h" #include "ggml-cpp.h" #include "ggml-alloc.h" #include "ggml-backend.h" -#include "ggml-cpu.h" #include "gguf.h" #include @@ -19,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -30,1318 +25,11 @@ #include #include -struct clip_logger_state g_logger_state = {clip_log_callback_default, NULL}; - -//#define CLIP_DEBUG_FUNCTIONS - -#ifdef CLIP_DEBUG_FUNCTIONS -static void clip_image_write_image_to_ppm(const clip_image_u8& img, const std::string& filename) { - std::ofstream file(filename, std::ios::binary); - if (!file.is_open()) { - LOG_ERR("Failed to open file for writing: %s\n", filename.c_str()); - return; - } - - // PPM header: P6 format, width, height, and max color value - const auto ppm_size = img.get_size(); - file << "P6\n" << ppm_size.width << " " << ppm_size.height << "\n255\n"; - - // Write pixel data - const auto & ppm_buf = img.get_ro_buf(); - for (size_t i = 0; i < ppm_buf.size(); i += 3) { - // PPM expects binary data in RGB format, which matches our image buffer - file.write(reinterpret_cast(&ppm_buf[i]), 3); - } - - file.close(); -} - -static void clip_image_save_to_bmp(const clip_image_u8& img, const std::string& filename) { - std::ofstream file(filename, std::ios::binary); - if (!file.is_open()) { - LOG_ERR("Failed to open file for writing: %s\n", filename.c_str()); - return; - } - - const auto bmp_size = img.get_size(); - int fileSize = 54 + 3 * bmp_size.width * bmp_size.height; // File header + info header + pixel data - int bytesPerPixel = 3; - int widthInBytes = bmp_size.width * bytesPerPixel; - int paddingAmount = (4 - (widthInBytes % 4)) % 4; - int stride = widthInBytes + paddingAmount; - - // Bitmap file header - unsigned char fileHeader[14] = { - 'B','M', // Signature - 0,0,0,0, // Image file size in bytes - 0,0,0,0, // Reserved - 54,0,0,0 // Start of pixel array - }; - - // Total file size - fileSize = 54 + (stride * bmp_size.height); - fileHeader[2] = (unsigned char)(fileSize); - fileHeader[3] = (unsigned char)(fileSize >> 8); - fileHeader[4] = (unsigned char)(fileSize >> 16); - fileHeader[5] = (unsigned char)(fileSize >> 24); - - // Bitmap information header (BITMAPINFOHEADER) - unsigned char infoHeader[40] = { - 40,0,0,0, // Size of this header (40 bytes) - 0,0,0,0, // Image width - 0,0,0,0, // Image height - 1,0, // Number of color planes - 24,0, // Bits per pixel - 0,0,0,0, // No compression - 0,0,0,0, // Image size (can be 0 for no compression) - 0,0,0,0, // X pixels per meter (not specified) - 0,0,0,0, // Y pixels per meter (not specified) - 0,0,0,0, // Total colors (color table not used) - 0,0,0,0 // Important colors (all are important) - }; - - // Width and height in the information header - infoHeader[4] = (unsigned char)(bmp_size.width); - infoHeader[5] = (unsigned char)(bmp_size.width >> 8); - infoHeader[6] = (unsigned char)(bmp_size.width >> 16); - infoHeader[7] = (unsigned char)(bmp_size.width >> 24); - infoHeader[8] = (unsigned char)(bmp_size.height); - infoHeader[9] = (unsigned char)(bmp_size.height >> 8); - infoHeader[10] = (unsigned char)(bmp_size.height >> 16); - infoHeader[11] = (unsigned char)(bmp_size.height >> 24); - - // Write file headers - file.write(reinterpret_cast(fileHeader), sizeof(fileHeader)); - file.write(reinterpret_cast(infoHeader), sizeof(infoHeader)); - - // Pixel data - std::vector padding(3, 0); // Max padding size to be added to each row - for (int y = bmp_size.height - 1; y >= 0; --y) { // BMP files are stored bottom-to-top - for (int x = 0; x < bmp_size.width; ++x) { - // Each pixel - const auto px = img.get_pixel(x, y); - unsigned char pixel[3] = { - px[2], // BMP stores pixels in BGR format - px[1], - px[0] - }; - file.write(reinterpret_cast(pixel), 3); - } - // Write padding for the row - file.write(reinterpret_cast(padding.data()), paddingAmount); - } - - file.close(); -} - -// debug function to convert f32 to u8 -static void clip_image_convert_f32_to_u8(const clip_image_f32& src, clip_image_u8& dst) { - dst.set_size(src.get_size(), false); - const auto & src_buf = src.get_ro_buf(); - std::vector dst_buf(src.n_elements()); - for (size_t i = 0; i < src.n_elements(); ++i) { - dst_buf[i] = static_cast(std::min(std::max(int(src_buf[i] * 255.0f), 0), 255)); - } - dst.cpy_buf(dst_buf); -} -#endif - - -// ggml_backend_cpu_set_n_threads(backend_cpu, n_threads); -static void clip_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) { - ggml_backend_dev_t dev = ggml_backend_get_device(backend_cpu); - ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; - if (reg) { - auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); - if (ggml_backend_set_n_threads_fn) { - ggml_backend_set_n_threads_fn(backend_cpu, n_threads); - } - } -} - -static std::shared_ptr clip_threadpool_init(int n_threads) { - ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); - ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; - if (!reg || n_threads < 2) { - return nullptr; - } - auto * new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new"); - auto * free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free"); - if (!new_fn || !free_fn) { - return nullptr; - } - ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads); - tpp.poll = 0; // the pool is idle most of the time, don't take CPU away from the text model - ggml_threadpool_t threadpool = new_fn(&tpp); - if (!threadpool) { - throw std::runtime_error("failed to create threadpool"); - } - return std::shared_ptr(threadpool, free_fn); -} - -struct clip_ctx { - clip_model model; - - gguf_context_ptr ctx_gguf; - ggml_context_ptr ctx_data; - - std::vector buf_compute_meta; - - std::vector backend_ptrs; - std::vector backend_buft; - - std::shared_ptr threadpool; // shared by the contexts of a model - - ggml_backend_t backend = nullptr; - ggml_backend_t backend_cpu = nullptr; - ggml_backend_buffer_ptr buf; - - - int max_nodes = 8192; - ggml_backend_sched_ptr sched; - clip_flash_attn_type flash_attn_type = CLIP_FLASH_ATTN_TYPE_AUTO; - bool is_allocated = false; - - bool debug_output_embeddings = false; - - // for measuring memory usage - bool no_alloc = false; - std::map mem_usage; - std::map mem_compute; - - bool support_batch = false; - - clip_ctx(clip_context_params & ctx_params, std::shared_ptr threadpool) : threadpool(std::move(threadpool)) { - flash_attn_type = ctx_params.flash_attn_type; - no_alloc = ctx_params.no_alloc; - backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); - if (!backend_cpu) { - throw std::runtime_error("failed to initialize CPU backend"); - } - if (this->threadpool) { - ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); - auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); - if (set_threadpool_fn) { - set_threadpool_fn(backend_cpu, this->threadpool.get()); - } - } - if (ctx_params.use_gpu) { - if (ctx_params.device != nullptr) { - backend = ggml_backend_dev_init(ctx_params.device, nullptr); - if (!backend) { - throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n", - __func__, ggml_backend_dev_name(ctx_params.device))); - } - } else { - backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr); - backend = backend ? backend : ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU, nullptr); - } - } - - if (backend) { - LOG_INF("%s: CLIP using %s backend\n", __func__, ggml_backend_name(backend)); - backend_ptrs.push_back(backend); - backend_buft.push_back(ggml_backend_get_default_buffer_type(backend)); - } else { - backend = backend_cpu; - LOG_INF("%s: CLIP using CPU backend\n", __func__); - } - - if (ctx_params.image_min_tokens > 0) { - model.hparams.custom_image_min_tokens = ctx_params.image_min_tokens; - } - if (ctx_params.image_max_tokens > 0) { - model.hparams.custom_image_max_tokens = ctx_params.image_max_tokens; - } - - backend_ptrs.push_back(backend_cpu); - backend_buft.push_back(ggml_backend_get_default_buffer_type(backend_cpu)); - - sched.reset( - ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), 8192, false, true) - ); - - if (ctx_params.cb_eval != nullptr) { - ggml_backend_sched_set_eval_callback(sched.get(), ctx_params.cb_eval, ctx_params.cb_eval_user_data); - } - - debug_output_embeddings = std::getenv("MTMD_DEBUG_EMBEDDINGS") != nullptr; - } - - ~clip_ctx() { - ggml_backend_free(backend); - if (backend != backend_cpu) { - ggml_backend_free(backend_cpu); - } - } - - // this function is added so that we don't change too much of the existing code - projector_type proj_type() const { - return model.proj_type; - } -}; - // -// clip_graph +// mtmd_model_loader // -clip_graph::clip_graph(clip_ctx * ctx, const clip_image_f32 & img) : - model(ctx->model), - hparams(model.hparams), - proj_type(ctx->proj_type()), - img(img), - patch_size(hparams.patch_size), - n_patches_x(img.nx() / patch_size), - n_patches_y(img.ny() / patch_size), - n_patches(n_patches_x * n_patches_y), - n_embd(hparams.n_embd), - n_head(hparams.n_head), - n_head_kv(hparams.n_head_kv), - d_head(hparams.n_embd_head > 0 ? hparams.n_embd_head : (n_head > 0 ? n_embd / n_head : 0)), - n_layer(hparams.n_layer), - n_mmproj_embd(clip_n_mmproj_embd(ctx)), - eps(hparams.eps), - kq_scale(d_head > 0 ? 1.0f / sqrtf((float)d_head) : 0.0f), - flash_attn_type(ctx->flash_attn_type) { - struct ggml_init_params params = { - /*.mem_size =*/ ctx->buf_compute_meta.size(), - /*.mem_buffer =*/ ctx->buf_compute_meta.data(), - /*.no_alloc =*/ true, - }; - ctx0_ptr.reset(ggml_init(params)); - ctx0 = ctx0_ptr.get(); - gf = ggml_new_graph_custom(ctx0, ctx->max_nodes, false); -} - -clip_graph::clip_graph(const clip_graph & parent) : - model(parent.model), - hparams(parent.hparams), - proj_type(parent.proj_type), - img(parent.img), - patch_size(parent.patch_size), - n_patches_x(parent.n_patches_x), - n_patches_y(parent.n_patches_y), - n_patches(parent.n_patches), - n_embd(parent.n_embd), - n_head(parent.n_head), - n_head_kv(parent.n_head_kv), - d_head(parent.d_head), - n_layer(parent.n_layer), - n_mmproj_embd(parent.n_mmproj_embd), - eps(parent.eps), - kq_scale(parent.kq_scale), - flash_attn_type(parent.flash_attn_type) { - // reuse from parent - ctx0 = parent.ctx0; - gf = parent.gf; -} - -ggml_tensor * clip_graph::build_mm(ggml_tensor * w, ggml_tensor * x) const { - return ggml_mul_mat(ctx0, w, x); -} - -void clip_graph::cb(ggml_tensor * cur, const char * name, int il) const { - if (il >= 0) { - ggml_format_name(cur, "%s-%d", name, il); - } else { - ggml_set_name(cur, name); - } -} - -// siglip2 naflex -ggml_tensor * clip_graph::resize_position_embeddings(uint32_t interpolation_mode) { - ggml_tensor * pos_embd = model.position_embeddings; - const int height = img.ny() / patch_size; - const int width = img.nx() / patch_size; - const uint32_t mode = interpolation_mode; - const int n_per_side = (int)std::sqrt(pos_embd->ne[1]); - - GGML_ASSERT(pos_embd); - - if (height == n_per_side && width == n_per_side) { - return pos_embd; - } - - pos_embd = ggml_reshape_3d(ctx0, pos_embd, n_embd, n_per_side, n_per_side); // -> (n_embd, n_per_side, n_per_side) - pos_embd = ggml_permute(ctx0, pos_embd, 2, 0, 1, 3); // -> (n_per_side, n_per_side, n_embd) - pos_embd = ggml_interpolate(ctx0, pos_embd, width, height, n_embd, 1, mode); // -> (width, height, n_embd) - pos_embd = ggml_permute(ctx0, pos_embd, 1, 2, 0, 3); // -> (n_embd, width, height) - pos_embd = ggml_cont_2d(ctx0, pos_embd, n_embd, width * height); // -> (n_embd, width * height) - - return pos_embd; -} - -// build vision transformer (ViT) cgraph -// this function should cover most of the models -// if your model has specific features, you should probably duplicate this function -ggml_tensor * clip_graph::build_vit( - ggml_tensor * inp, - int64_t n_pos, - norm_type norm_t, - ffn_op_type ffn_t, - ggml_tensor * learned_pos_embd, - std::function add_pos, - const build_vit_opts & opts - ) { - // batch dim: inp is [n_embd, n_pos, B] - const int64_t B = inp->ne[2]; - - if (learned_pos_embd) { - inp = ggml_add(ctx0, inp, learned_pos_embd); - cb(inp, "pos_embed", -1); - } - - // flatten batch; unflatten again in attention - inp = ggml_reshape_2d(ctx0, inp, n_embd, n_pos * B); - - ggml_tensor * inpL = inp; - - // pre-layernorm - if (model.pre_ln_w) { - inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, norm_t, eps, -1); - cb(inpL, "pre_ln", -1); - } - - // loop over layers - for (int il = 0; il < n_layer; il++) { - auto & layer = model.layers[il]; - ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states - - ggml_tensor * attn_mask = opts.attn_mask; - if (opts.attn_mask_layers.size() > (size_t) il) { - attn_mask = opts.attn_mask_layers[il]; - } - - // layernorm1 - cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il); - cb(cur, "layer_inp_normed", il); - - // self-attention - { - ggml_tensor * Qcur = nullptr; - ggml_tensor * Kcur = nullptr; - ggml_tensor * Vcur = nullptr; - if (layer.qkv_w != nullptr) { - // fused qkv - cur = build_mm(layer.qkv_w, cur); - if (layer.qkv_b != nullptr) { - cur = ggml_add(ctx0, cur, layer.qkv_b); - } - - // Q/K/V as [d_head, n_head, n_pos, B], the batch stride is cur->nb[1]*n_pos. - Qcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B, - /* nb1 */ ggml_row_size(cur->type, d_head), - /* nb2 */ cur->nb[1], - /* nb3 */ cur->nb[1] * n_pos, - /* offset */ 0); - - Kcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B, - /* nb1 */ ggml_row_size(cur->type, d_head), - /* nb2 */ cur->nb[1], - /* nb3 */ cur->nb[1] * n_pos, - /* offset */ ggml_row_size(cur->type, n_head * d_head)); - - Vcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B, - /* nb1 */ ggml_row_size(cur->type, d_head), - /* nb2 */ cur->nb[1], - /* nb3 */ cur->nb[1] * n_pos, - /* offset */ ggml_row_size(cur->type, 2 * n_head * d_head)); - - if (layer.q_norm) { - GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]); - Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il); - cb(Qcur, "Qcur_norm", il); - } - - if (layer.k_norm) { - GGML_ASSERT(layer.k_norm->ne[0] == Kcur->ne[0]); - Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il); - cb(Kcur, "Kcur_norm", il); - } - - } else { - // separate q, k, v - Qcur = build_mm(layer.q_w, cur); - if (layer.q_b) { - Qcur = ggml_add(ctx0, Qcur, layer.q_b); - } - - Kcur = build_mm(layer.k_w, cur); - if (layer.k_b) { - Kcur = ggml_add(ctx0, Kcur, layer.k_b); - } - - Vcur = build_mm(layer.v_w, cur); - if (layer.v_b) { - Vcur = ggml_add(ctx0, Vcur, layer.v_b); - } - - // if true, norm must be applied after reshaping to (d_head, n_head, n_pos) - bool norm_per_head = layer.q_norm && layer.q_norm->ne[0] == d_head; - - if (!norm_per_head) { - if (layer.q_norm) { - Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il); - cb(Qcur, "Qcur_norm", il); - } - if (layer.k_norm) { - Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il); - cb(Kcur, "Kcur_norm", il); - } - } - - Qcur = ggml_reshape_4d(ctx0, Qcur, d_head, n_head, n_pos, B); - Kcur = ggml_reshape_4d(ctx0, Kcur, d_head, n_head_kv, n_pos, B); - Vcur = ggml_reshape_4d(ctx0, Vcur, d_head, n_head_kv, n_pos, B); - - if (norm_per_head) { - if (layer.q_norm) { - Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il); - cb(Qcur, "Qcur_norm_per_head", il); - } - if (layer.k_norm) { - Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il); - cb(Kcur, "Kcur_norm_per_head", il); - } - } - } - - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - if (add_pos) { - Qcur = add_pos(Qcur, layer); - Kcur = add_pos(Kcur, layer); - cb(Qcur, "Qcur_pos", il); - cb(Kcur, "Kcur_pos", il); - } - - if (proj_type == PROJECTOR_TYPE_GEMMA4V) { - Vcur = ggml_rms_norm(ctx0, Vcur, eps); - cb(Vcur, "Vcur_normed", il); - } - - // build_attn returns a flat 2D [n_embd, n_pos*B] - cur = build_attn(layer.o_w, layer.o_b, - Qcur, Kcur, Vcur, attn_mask, kq_scale, il); - cb(cur, "attn_out", il); - } - - if (layer.ls_1_w) { - cur = ggml_mul(ctx0, cur, layer.ls_1_w); - cb(cur, "attn_out_scaled", il); - } - - if (layer.attn_post_norm_w) { - cur = build_norm(cur, layer.attn_post_norm_w, nullptr, norm_t, eps, il); - cb(cur, "attn_post_normed", il); - } - - // re-add the layer input, e.g., residual - cur = ggml_add(ctx0, cur, inpL); - - inpL = cur; // inpL = residual, cur = hidden_states - - if (opts.callback_layer_out) { - opts.callback_layer_out(cur, il); - } - - cb(cur, "ffn_inp", il); - - // layernorm2 (pre-ffn norm) - cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il); - cb(cur, "ffn_inp_normed", il); - - // ffn - cur = layer.ff_gate_exps_w - ? build_moe_ffn(cur, layer, ffn_t, il) - : build_ffn(cur, - layer.ff_up_w, layer.ff_up_b, - layer.ff_gate_w, layer.ff_gate_b, - layer.ff_down_w, layer.ff_down_b, - ffn_t, il); - - cb(cur, "ffn_out", il); - - if (layer.ff_post_norm_w) { - cur = build_norm(cur, layer.ff_post_norm_w, nullptr, norm_t, eps, il); - cb(cur, "ffn_post_normed", il); - } - - if (layer.ls_2_w) { - cur = ggml_mul(ctx0, cur, layer.ls_2_w); - cb(cur, "ffn_out_scaled", il); - } - - // residual 2 - cur = ggml_add(ctx0, inpL, cur); - cb(cur, "layer_out", il); - - if (layer.ls_out_w) { - cur = ggml_mul(ctx0, cur, layer.ls_out_w); - cb(cur, "layer_out_scaled", il); - } - - inpL = cur; - } - - if (model.audio_has_avgpool()) { - ggml_tensor * cur = inpL; - cur = ggml_transpose(ctx0, cur); - cur = ggml_cont(ctx0, cur); - cur = ggml_pool_1d(ctx0, cur, GGML_OP_POOL_AVG, 2, 2, 0); - cur = ggml_transpose(ctx0, cur); - cur = ggml_cont(ctx0, cur); - inpL = cur; - } - - // post-layernorm - if (model.post_ln_w && !opts.skip_post_ln) { - inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, norm_t, eps, -1); - } - - // restore the batch dim - GGML_ASSERT(inpL->ne[1] % B == 0); - inpL = ggml_reshape_3d(ctx0, inpL, n_embd, inpL->ne[1] / B, B); - return inpL; -} - -// build the input after conv2d (inp_raw --> patches) -// returns tensor with shape [n_embd, n_patches] -ggml_tensor * clip_graph::build_inp() { - ggml_tensor * inp_raw = build_inp_raw(); - ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1); - inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, n_batch); - inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); - if (model.patch_bias) { - inp = ggml_add(ctx0, inp, model.patch_bias); - cb(inp, "patch_bias", -1); - } - return inp; -} - -ggml_tensor * clip_graph::build_inp_raw(int channels) { - ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, img.nx(), img.ny(), channels, n_batch); - ggml_set_name(inp_raw, "inp_raw"); - ggml_set_input(inp_raw); - return inp_raw; -} - -// qwen-vl style temporal merge: Conv3D(kt=2) split into 2 Conv2D -ggml_tensor * clip_graph::build_inp_temporal_merge() { - ggml_tensor * inp_raw = build_inp_raw(); - - GGML_ASSERT(img.nx() % (patch_size * 2) == 0); - GGML_ASSERT(img.ny() % (patch_size * 2) == 0); - - const size_t nb1 = ggml_row_size(inp_raw->type, img.nx()); - const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny()); - - if (n_batch == 1) { - // still image input - return ggml_add(ctx0, - ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1), - ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1)); - } else if (n_batch == 2) { - // 2 frames input (video input) - ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw, - img.nx(), img.ny(), 3, nb1, nb2, 0); - ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw, - img.nx(), img.ny(), 3, nb1, nb2, - nb2 * 3); // move to the second frame - return ggml_add(ctx0, - ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1), - ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1)); - } else { - GGML_ASSERT(false && "n_batch > 2 is not supported"); - } -} - -static std::string get_attn_mask_name(int idx) { - return idx == 0 ? "attn_mask" : "attn_mask_" + std::to_string(idx); -} - -ggml_tensor * clip_graph::build_inp_attn_mask(int64_t n_kv, int64_t n_q, int idx) { - const ggml_type type = flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED ? GGML_TYPE_F16 : GGML_TYPE_F32; - ggml_tensor * mask = ggml_new_tensor_2d(ctx0, type, n_kv, n_q); - ggml_set_name(mask, get_attn_mask_name(idx).c_str()); - ggml_set_input(mask); - return mask; -} - -ggml_tensor * clip_graph::build_norm( - ggml_tensor * cur, - ggml_tensor * mw, - ggml_tensor * mb, - norm_type type, - float norm_eps, - int il) const { - - cur = type == NORM_TYPE_RMS - ? ggml_rms_norm(ctx0, cur, norm_eps) - : ggml_norm(ctx0, cur, norm_eps); - - if (mw) { - cur = ggml_mul(ctx0, cur, mw); - cb(cur, "norm_w", il); - } - - if (mb) { - cur = ggml_add(ctx0, cur, mb); - cb(cur, "norm_b", il); - } - - return cur; -} - -ggml_tensor * clip_graph::build_ffn( - ggml_tensor * cur, - ggml_tensor * up, - ggml_tensor * up_b, - ggml_tensor * gate, - ggml_tensor * gate_b, - ggml_tensor * down, - ggml_tensor * down_b, - ffn_op_type type_op, - int il) const { - - ggml_tensor * tmp = up ? build_mm(up, cur) : cur; - cb(tmp, "ffn_up", il); - - if (up_b) { - tmp = ggml_add(ctx0, tmp, up_b); - cb(tmp, "ffn_up_b", il); - } - - if (gate) { - cur = build_mm(gate, cur); - cb(cur, "ffn_gate", il); - - if (gate_b) { - cur = ggml_add(ctx0, cur, gate_b); - cb(cur, "ffn_gate_b", il); - } - } else { - cur = tmp; - } - - // we only support parallel ffn for now - switch (type_op) { - case FFN_SILU: - if (gate) { - cur = hparams.swiglu_clamp > 0.0f - ? ggml_swiglu_clamp(ctx0, cur, tmp, hparams.swiglu_clamp) - : ggml_swiglu_split(ctx0, cur, tmp); - cb(cur, "ffn_swiglu", il); - } else { - cur = ggml_silu(ctx0, cur); - cb(cur, "ffn_silu", il); - } break; - case FFN_GELU: - if (gate) { - cur = ggml_geglu_split(ctx0, cur, tmp); - cb(cur, "ffn_geglu", il); - } else { - cur = ggml_gelu(ctx0, cur); - cb(cur, "ffn_gelu", il); - } break; - case FFN_GELU_ERF: - if (gate) { - cur = ggml_geglu_erf_split(ctx0, cur, tmp); - cb(cur, "ffn_geglu_erf", il); - } else { - cur = ggml_gelu_erf(ctx0, cur); - cb(cur, "ffn_gelu_erf", il); - } break; - case FFN_GELU_QUICK: - if (gate) { - cur = ggml_geglu_quick_split(ctx0, cur, tmp); - cb(cur, "ffn_geglu_quick", il); - } else { - cur = ggml_gelu_quick(ctx0, cur); - cb(cur, "ffn_gelu_quick", il); - } break; - case FFN_RELU_SQR: - { - cur = ggml_relu(ctx0, cur); - cur = ggml_sqr(ctx0, cur); - cb(cur, "ffn_relu_sqr", il); - } break; - } - - if (down) { - cur = build_mm(down, cur); - } - - if (down_b) { - cb(cur, "ffn_down", il); - } - - if (down_b) { - cur = ggml_add(ctx0, cur, down_b); - } - - return cur; -} - -// MoE FFN with sigmoid router and normalized top-k weights (dots3note vision) -// the router runs in fp32; exp_probs_b only affects expert selection, not the weights -ggml_tensor * clip_graph::build_moe_ffn(ggml_tensor * cur, const clip_layer & layer, ffn_op_type type_op, int il) const { - const int64_t n_tokens = cur->ne[1]; - const int64_t n_expert = layer.ff_gate_exps_w->ne[2]; - const int64_t n_expert_used = std::min((int64_t) hparams.n_expert_used, n_expert); - GGML_ASSERT(n_expert_used > 0); - GGML_ASSERT(type_op == FFN_SILU); - - ggml_tensor * probs = ggml_sigmoid(ctx0, build_mm(layer.ff_gate_inp_w, cur)); // [n_expert, n_tokens] - cb(probs, "ffn_moe_probs", il); - - ggml_tensor * sel = layer.ff_exp_probs_b - ? ggml_add(ctx0, probs, layer.ff_exp_probs_b) - : probs; - ggml_tensor * selected = ggml_top_k(ctx0, sel, n_expert_used); // [n_expert_used, n_tokens] - - ggml_tensor * weights = ggml_get_rows(ctx0, - ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens), selected); - weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens); - weights = ggml_div(ctx0, weights, ggml_sum_rows(ctx0, weights)); - weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens); - cb(weights, "ffn_moe_weights", il); - - cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], 1, n_tokens); - ggml_tensor * gate = ggml_mul_mat_id(ctx0, layer.ff_gate_exps_w, cur, selected); // [n_ff, n_expert_used, n_tokens] - ggml_tensor * up = ggml_mul_mat_id(ctx0, layer.ff_up_exps_w, cur, selected); - cur = ggml_mul(ctx0, ggml_silu(ctx0, gate), up); - cur = ggml_mul_mat_id(ctx0, layer.ff_down_exps_w, cur, selected); // [n_embd, n_expert_used, n_tokens] - cur = ggml_mul(ctx0, cur, weights); - - // sum over the selected experts - ggml_tensor * out = nullptr; - for (int64_t i = 0; i < n_expert_used; i++) { - ggml_tensor * v = ggml_view_2d(ctx0, cur, cur->ne[0], n_tokens, cur->nb[2], i * cur->nb[1]); - out = out ? ggml_add(ctx0, out, v) : v; - } - if (n_expert_used == 1) { - out = ggml_cont(ctx0, out); - } - cb(out, "ffn_moe_out", il); - return out; -} - -ggml_tensor * clip_graph::build_attn( - ggml_tensor * wo, - ggml_tensor * wo_b, - ggml_tensor * q_cur, - ggml_tensor * k_cur, - ggml_tensor * v_cur, - ggml_tensor * kq_mask, - float kq_scale, - int il, - ggml_tensor * sinks) const { - // these nodes are added to the graph together so that they are not reordered - // by doing so, the number of splits in the graph is reduced - // the order is fixed without the compute flag, so an unselected branch stays out of the compute set - ggml_build_forward_order(gf, q_cur); - ggml_build_forward_order(gf, k_cur); - ggml_build_forward_order(gf, v_cur); - - ggml_tensor * q = ggml_permute(ctx0, q_cur, 0, 2, 1, 3); - //cb(q, "q", il); - - ggml_tensor * k = ggml_permute(ctx0, k_cur, 0, 2, 1, 3); - //cb(k, "k", il); - - ggml_tensor * cur; - - if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { - ggml_tensor * v = ggml_permute(ctx0, v_cur, 0, 2, 1, 3); - - k = ggml_cast(ctx0, k, GGML_TYPE_F16); - v = ggml_cast(ctx0, v, GGML_TYPE_F16); - // mask must be f16 here, use build_inp_attn_mask() - GGML_ASSERT(!kq_mask || kq_mask->type == GGML_TYPE_F16); - - cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f); - ggml_prec_set_acc(cur, GGML_PREC_F32); - if (sinks != nullptr) { - ggml_flash_attn_ext_add_sinks(cur, sinks); - } - - cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]); - - } else { - ggml_tensor * v = ggml_permute(ctx0, v_cur, 1, 2, 0, 3); - v = ggml_cont(ctx0, v); - - ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); - // TODO: this most likely requires GGML_PREC_BF16, but will enable it later to avoid scope-creep - // ggml_prec_set_acc(kq, GGML_PREC_BF16); - - kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, 0.0f); - if (sinks != nullptr) { - ggml_soft_max_add_sinks(kq, sinks); - } - - ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq); - cur = ggml_permute(ctx0, kqv, 0, 2, 1, 3); - cur = ggml_cont_2d(ctx0, cur, cur->ne[0] * cur->ne[1], cur->ne[2] * cur->ne[3]); - } - - cb(cur, "kqv_out", il); - - if (wo) { - cur = build_mm(wo, cur); - } - - if (wo_b) { - cur = ggml_add(ctx0, cur, wo_b); - } - - return cur; -} - -// implementation of the 2D RoPE without adding a new op in ggml -ggml_tensor * clip_graph::build_rope_2d( - ggml_context * ctx0, - ggml_tensor * cur, - ggml_tensor * pos_a, // first half - ggml_tensor * pos_b, // second half - const float freq_base, - const bool interleave_freq -) { - const int64_t n_dim = cur->ne[0]; - - // for example, if we have cur tensor of shape (n_dim=8, n_head, n_pos) - // we will have a list of 4 inv_freq: 1e-0, 1e-1, 1e-2, 1e-3 - // first half of cur will use 1e-0, 1e-2 (even) - // second half of cur will use 1e-1, 1e-3 (odd) - // the trick here is to rotate just half of n_dim, so inv_freq will automatically be even - // ^ don't ask me why, it's math! -2(2i) / n_dim == -2i / (n_dim/2) - // then for the second half, we use freq_scale to shift the inv_freq - // ^ why? replace (2i) with (2i+1) in the above equation - const float freq_scale_odd = interleave_freq - ? std::pow(freq_base, (float)-2/n_dim) - : 1.0; - - // first half, dims [0, n_dim/2) - cur = ggml_rope_ext( - ctx0, - cur, - pos_a, // positions - nullptr, // freq factors - n_dim/2, // n_dims - 0, 0, freq_base, - 1.0f, 0.0f, 1.0f, 0.0f, 0.0f - ); - - // second half, dims [n_dim/2, n_dim) - cur = ggml_rope_ext( - ctx0, - cur, - pos_b, // positions - nullptr, // freq factors - n_dim/2, // n_dims - 0, 0, freq_base, - freq_scale_odd, - 0.0f, 1.0f, 0.0f, 0.0f - ); - cur = ggml_rope_set_offset(cur, n_dim/2); - - return cur; -} - -// Generic function to stack frames for audio processing -// Abstracts out the StackAudioFrames logic used by ultravox -ggml_tensor * clip_graph::build_stack(ggml_tensor * cur, int32_t stack_factor, int32_t n_embed) { - if (stack_factor <= 1) { - return cur; - } - - int64_t total_elements = ggml_nelements(cur); - int64_t stride = n_embed * stack_factor; - - // Calculate padded length - int64_t padded_len = GGML_PAD(total_elements, stride); - int64_t pad = padded_len - total_elements; - - if (pad > 0) { - // Pad the tensor to make it divisible by stride - cur = ggml_view_1d(ctx0, cur, total_elements, 0); - cur = ggml_pad(ctx0, cur, pad, 0, 0, 0); - } - - // Reshape to [stride, padded_len / stride] - cur = ggml_view_2d(ctx0, cur, stride, padded_len / stride, - ggml_row_size(cur->type, stride), 0); - return cur; -} - -// aka pixel_shuffle / pixel_unshuffle / patch_merger (Kimi-VL) -// support dynamic resolution -ggml_tensor * clip_graph::build_suffix(ggml_tensor * cur) { - for (int idx : clip_suffix_rows(img.suffix_type)) { - GGML_ASSERT(model.tok_embd_sep && idx < model.tok_embd_sep->ne[1]); - ggml_tensor * row = ggml_view_2d(ctx0, model.tok_embd_sep, model.tok_embd_sep->ne[0], 1, - model.tok_embd_sep->nb[1], idx * model.tok_embd_sep->nb[1]); - cur = ggml_concat(ctx0, cur, ggml_cast(ctx0, row, cur->type), 1); - } - return cur; -} - -ggml_tensor * clip_graph::build_patch_merge_permute(ggml_tensor * cur, int scale_factor) { - GGML_ASSERT(scale_factor > 1); - - const int n_embd = cur->ne[0]; - int width = img.nx() / patch_size; - int height = img.ny() / patch_size; - - // pad width and height to factor - const int64_t pad_width = CLIP_ALIGN(width, scale_factor) - width; - const int64_t pad_height = CLIP_ALIGN(height, scale_factor) - height; - cur = ggml_reshape_3d(ctx0, cur, n_embd, width, height); - if (pad_width || pad_height) { - cur = ggml_pad(ctx0, cur, 0, pad_width, pad_height, 0); - width += pad_width; - height += pad_height; - } - - // unshuffle h - cur = ggml_reshape_3d(ctx0, cur, n_embd * scale_factor, width / scale_factor, height); - cur = ggml_permute(ctx0, cur, 0, 2, 1, 3); - - // unshuffle w - cur = ggml_cont_3d(ctx0, cur, n_embd * scale_factor * scale_factor, height / scale_factor, width / scale_factor); - cur = ggml_permute(ctx0, cur, 0, 2, 1, 3); - - cur = ggml_cont_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]); - cb(cur, "pixel_shuffle", -1); - - return cur; -} - -static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const clip_image_f32_batch & imgs, - const clip_encode_params * params = nullptr) { - const clip_image_f32 & img = imgs.entries[0]; - std::unique_ptr builder; - - switch (ctx->proj_type()) { - case PROJECTOR_TYPE_GEMMA3: - case PROJECTOR_TYPE_IDEFICS3: - case PROJECTOR_TYPE_COHERE2V: - case PROJECTOR_TYPE_LFM2: - case PROJECTOR_TYPE_D1OMNI_V: - case PROJECTOR_TYPE_JANUS_PRO: - case PROJECTOR_TYPE_PHI4: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_GEMMA3NV: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_GEMMA4V: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_GEMMA4UV: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_PIXTRAL: - case PROJECTOR_TYPE_LIGHTONOCR: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_DOTS_OCR: - case PROJECTOR_TYPE_DOTS3NOTE_V: // same ViT + merger; pyramid MoE is handled by build_vit - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_DOTS3NOTE_A: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_QWEN2VL: - case PROJECTOR_TYPE_QWEN25VL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_QWEN3VL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MAGEVL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_SENSENOVA_U1: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_LFM3VL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_LING3VL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_EXAONE4_5: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MIMOVL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MINIMAX_M3: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MUSE_GLIMMER: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_STEP3VL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MINICPMV: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MINICPMV4_6: - case PROJECTOR_TYPE_MINICPMV4_7: - case PROJECTOR_TYPE_MINICPM_OCR_1_0: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_INTERNVL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_NEMOTRON_V2_VL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_NEMOTRON_OMNI: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_LLAMA4: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_VOXTRAL_RT: - { - builder = std::make_unique(ctx, img, params ? params->enc_state : nullptr); - } break; - case PROJECTOR_TYPE_ULTRAVOX: - case PROJECTOR_TYPE_VOXTRAL: - case PROJECTOR_TYPE_QWEN2A: - case PROJECTOR_TYPE_GLMA: - case PROJECTOR_TYPE_MERALION: - case PROJECTOR_TYPE_MUSIC_FLAMINGO: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_KIMIVL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_PADDLEOCR: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_KIMIK25: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_DEEPSEEK4V: - case PROJECTOR_TYPE_DEEPSEEK41V: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_COGVLM: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_HUNYUANVL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MLP: - case PROJECTOR_TYPE_MLP_NORM: - case PROJECTOR_TYPE_LDP: - case PROJECTOR_TYPE_LDPV2: - case PROJECTOR_TYPE_GLM_EDGE: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_DEEPSEEKOCR: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_DEEPSEEKOCR2: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_LFM2A: - case PROJECTOR_TYPE_D1OMNI_A: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_LFM3A: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_GEMMA4A: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_GEMMA4UA: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_GRANITE_SPEECH: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_GLM4V: - case PROJECTOR_TYPE_GLM5V: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_QWEN3A: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MIMO_AUDIO: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_QWEN3TTS_SPKENC: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_POCKETTTS_SPKENC: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_POCKETTTS_GEN: - { - const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE; - const int n_step = ctx->model.hparams.flow_n_step; - const int64_t n_latent = ctx->model.gen_input_lin_w->ne[0]; - GGML_ASSERT(n_step > 0); - GGML_ASSERT(n_latent > 0); - // "inp_feats" takes the caller's buffer as-is, the graph must consume all of it - if (params && params->feats) { - GGML_ASSERT(params->feats->size() % (size_t) n_latent == 0); - GGML_ASSERT(params->feats->size() >= (size_t) n_latent); - } - const int n_frames = params && params->feats ? (int) (params->feats->size() / n_latent) : 1; - builder = std::make_unique(ctx, img, gen_process, n_step, n_frames); - } break; - case PROJECTOR_TYPE_CHATTERBOX_SPKENC: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_CHATTERBOX: - { - const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE; - int n_tokens = params && params->codes ? (int) params->codes->size() : 0; - int n_prompt_mel = 0; - if (n_tokens > 0) { - GGML_ASSERT(params->ref); - n_tokens += (int) params->ref->tokens.size(); - n_prompt_mel = (int) (params->ref->feat.size() / 80); - } - const int vnm = params ? params->vocode_n_mel : 0; - const int vns = params ? params->vocode_n_stft : 0; - builder = std::make_unique(ctx, img, gen_process, n_tokens, n_prompt_mel, vnm, vns); - } break; - case PROJECTOR_TYPE_LFM3A_GEN: - { - const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE; - const int64_t n_cb = ctx->model.gen_code_embd_w->ne[2]; - bool all_known = false; - int n_frames = 1; - if (params && params->codes) { - GGML_ASSERT(params->codes->size() % (size_t) n_cb == 0); - if (gen_process == CLIP_GEN_PROCESS_GEN_WAV) { - n_frames = (int) (params->codes->size() / n_cb); - GGML_ASSERT(n_frames > 0); - } else { - // -1 marks a masked codebook, none masked means only the feedback embedding is needed - all_known = std::all_of(params->codes->begin(), params->codes->end(), [](int32_t c) { return c >= 0; }); - } - } - builder = std::make_unique(ctx, img, gen_process, all_known, n_frames); - } break; - case PROJECTOR_TYPE_DRAMA3_GEN: - { - const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE; - const int64_t n_cb = ctx->model.gen_code_embd_w->ne[2]; - const int n_frames = params && params->codes ? (int) (params->codes->size() / n_cb) : 1; - builder = std::make_unique(ctx, img, gen_process, params ? params->gen_state : nullptr, n_frames, - params ? params->top_k : 30, params ? params->top_p : 0.9f, - params ? params->temp : 1.0f); - } break; - case PROJECTOR_TYPE_QWEN3TTS_GEN: - { - const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE; - const int top_k = params ? params->top_k : 50; - const float top_p = params ? params->top_p : 1.0f; - builder = std::make_unique(ctx, img, gen_process, top_k, top_p); - } break; - case PROJECTOR_TYPE_SOPRANO: - { - const int n_frames = params && params->feats ? (int) (params->feats->size() / ctx->model.hparams.projection_dim) : 2; - builder = std::make_unique(ctx, img, n_frames); - } break; - case PROJECTOR_TYPE_YOUTUVL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_YASA2: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_PARAKEET: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: - { - builder = std::make_unique(ctx, img, params ? params->enc_state : nullptr); - } break; - case PROJECTOR_TYPE_GRANITE4_VISION: - { - builder = std::make_unique(ctx, img); - } break; - default: - GGML_ABORT("missing cgraph builder"); - } - - builder->img_batch = &imgs; - - // TODO [QWEN_VIDEO]: improve this in the future - builder->n_batch = imgs.entries.size(); - - return builder; -} - -// -// clip_model_loader -// - -struct clip_model_loader { +struct mtmd_model_loader { ggml_context_ptr ctx_meta; gguf_context_ptr ctx_gguf; @@ -1356,8 +44,7 @@ struct clip_model_loader { mtmd_progress_callback progress_callback = nullptr; void * progress_callback_user_data = nullptr; - // TODO @ngxson : we should not pass clip_ctx here, it should be clip_model - clip_model_loader(const char * fname, + mtmd_model_loader(const char * fname, bool skip_tensors = false, mtmd_progress_callback progress_cb = nullptr, void * progress_user_data = nullptr) @@ -1373,7 +60,7 @@ struct clip_model_loader { ctx_gguf = gguf_context_ptr(gguf_init_from_file(fname, params)); if (!ctx_gguf.get()) { - throw std::runtime_error(string_format("%s: failed to load CLIP model from %s. Does this file exist?\n", __func__, fname)); + throw std::runtime_error(string_format("%s: failed to load mmproj model from %s. Does this file exist?\n", __func__, fname)); } ctx_meta.reset(meta); @@ -1427,16 +114,16 @@ struct clip_model_loader { } } - void load_hparams(clip_model & model, clip_modality modality) { + void load_hparams(mtmd_model & model, mtmd_modality modality) { auto & hparams = model.hparams; std::string log_ffn_op; // for logging // sanity check - if (modality == CLIP_MODALITY_VISION) { + if (modality == MTMD_MODALITY_VISION) { GGML_ASSERT(has_vision); - } else if (modality == CLIP_MODALITY_AUDIO) { + } else if (modality == MTMD_MODALITY_AUDIO) { GGML_ASSERT(has_audio); - } else if (modality == CLIP_MODALITY_GEN_AUDIO) { + } else if (modality == MTMD_MODALITY_GEN_AUDIO) { GGML_ASSERT(has_gen_audio); } model.modality = modality; @@ -1450,18 +137,18 @@ struct clip_model_loader { // for models with mixed modalities if (proj_type.empty()) { - if (modality == CLIP_MODALITY_VISION) { + if (modality == MTMD_MODALITY_VISION) { get_string(KEY_VISION_PROJ_TYPE, proj_type, false); - } else if (modality == CLIP_MODALITY_AUDIO) { + } else if (modality == MTMD_MODALITY_AUDIO) { get_string(KEY_AUDIO_PROJ_TYPE, proj_type, false); - } else if (modality == CLIP_MODALITY_GEN_AUDIO) { + } else if (modality == MTMD_MODALITY_GEN_AUDIO) { get_string(KEY_GEN_AUDIO_PROJ_TYPE, proj_type, false); } else { GGML_ABORT("unknown modality"); } } - model.proj_type = clip_projector_type_from_string(proj_type); + model.proj_type = projector_type_from_string(proj_type); if (model.proj_type == PROJECTOR_TYPE_UNKNOWN) { throw std::runtime_error(string_format("%s: unknown projector type: %s\n", __func__, proj_type.c_str())); @@ -1469,15 +156,15 @@ struct clip_model_loader { // correct arch for multimodal models (legacy method) if (model.proj_type == PROJECTOR_TYPE_QWEN25O) { - model.proj_type = modality == CLIP_MODALITY_VISION + model.proj_type = modality == MTMD_MODALITY_VISION ? PROJECTOR_TYPE_QWEN25VL : PROJECTOR_TYPE_QWEN2A; } } - const bool is_vision = model.modality == CLIP_MODALITY_VISION; - const bool is_audio = model.modality == CLIP_MODALITY_AUDIO; - const bool is_gen_audio = model.modality == CLIP_MODALITY_GEN_AUDIO; + const bool is_vision = model.modality == MTMD_MODALITY_VISION; + const bool is_audio = model.modality == MTMD_MODALITY_AUDIO; + const bool is_gen_audio = model.modality == MTMD_MODALITY_GEN_AUDIO; // other hparams { @@ -1692,7 +379,7 @@ struct clip_model_loader { { get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor); GGML_ASSERT(hparams.subsampling_factor == 8 && - "subsampling_factor must match the conv strides in clip_graph_parakeet::build()"); + "subsampling_factor must match the conv strides in mtmd_graph_parakeet::build()"); get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size); GGML_ASSERT(hparams.audio_conv_kernel_size > 0 && hparams.audio_conv_kernel_size % 2 == 1 && "audio_conv_kernel_size must be a positive odd integer"); @@ -1720,7 +407,7 @@ struct clip_model_loader { hparams.audio_window_len = 400; hparams.audio_hop_len = 160; // one full window - using nt = clip_graph_nemotron_transcribe; + using nt = mtmd_graph_nemotron_transcribe; hparams.warmup_audio_size = (nt::window_left + nt::window_core + nt::window_right) * hparams.subsampling_factor; } break; case PROJECTOR_TYPE_IDEFICS3: @@ -2054,7 +741,7 @@ struct clip_model_loader { hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.image_resize_pad = PAD_NONE; get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); - // n_merge is used as a divisor in clip_image_batch_encode + // n_merge is used as a divisor in mtmd_subcontext::encode() // (gh / n_merge); reject 0 to avoid int div-by-zero (DoS). GGML_ASSERT(hparams.n_merge > 0); hparams.rope_theta = 10000.0f; // vision_config.rope_theta @@ -2221,7 +908,7 @@ struct clip_model_loader { hparams.audio_n_fft = 400; hparams.audio_window_len = 400; hparams.audio_hop_len = 160; - // 2 context frames + 8 s of new frames, see clip_enc_state_init() + // 2 context frames + 8 s of new frames, see mtmd_subcontext::enc_state_init() hparams.warmup_audio_size = 2 + 2*400; } break; case PROJECTOR_TYPE_MIMO_AUDIO: @@ -2281,6 +968,9 @@ struct clip_model_loader { } break; case PROJECTOR_TYPE_LFM3A_GEN: { + // TODO: CUDA has no flash attention kernel for the depthformer head dim 32, enable it once it does + model.flash_attn_auto = false; + // fixed by LiquidCodecConfig, head counts come from the tensors hparams.rope_theta = 1000000.0f; @@ -2359,6 +1049,9 @@ struct clip_model_loader { } break; case PROJECTOR_TYPE_CHATTERBOX: { + // the classic cfm solver unrolls up to 10 steps x 2 cfg estimator evaluations in one graph + model.max_nodes = 65536; + // s3gen output rate hparams.audio_sample_rate = 24000; } break; @@ -2617,8 +1310,7 @@ struct clip_model_loader { } } - void load_tensors(clip_ctx & ctx_clip) { - auto & model = ctx_clip.model; + void load_tensors(mtmd_model & model, ggml_backend_buffer_type_t buft, bool no_alloc) { auto & hparams = model.hparams; std::map tensor_offset; std::vector tensors_to_load; @@ -2629,8 +1321,8 @@ struct clip_model_loader { } // TODO @ngxson : support both audio and video in the future - const char * prefix = model.modality == CLIP_MODALITY_AUDIO ? "a" - : model.modality == CLIP_MODALITY_GEN_AUDIO ? "a.gen.code" + const char * prefix = model.modality == MTMD_MODALITY_AUDIO ? "a" + : model.modality == MTMD_MODALITY_GEN_AUDIO ? "a.gen.code" : "v"; // get offsets @@ -2645,8 +1337,8 @@ struct clip_model_loader { /*.mem_buffer =*/ NULL, /*.no_alloc =*/ true, }; - ctx_clip.ctx_data.reset(ggml_init(params)); - if (!ctx_clip.ctx_data) { + model.ctx_data.reset(ggml_init(params)); + if (!model.ctx_data) { throw std::runtime_error(string_format("%s: failed to init ggml context\n", __func__)); } @@ -2663,18 +1355,18 @@ struct clip_model_loader { } if (cur) { tensors_to_load.push_back(cur); - ggml_tensor * data_tensor = ggml_dup_tensor(ctx_clip.ctx_data.get(), cur); + ggml_tensor * data_tensor = ggml_dup_tensor(model.ctx_data.get(), cur); ggml_set_name(data_tensor, cur->name); loaded_tensor_names.insert(name); cur = data_tensor; // add to weight memory counter - ctx_clip.mem_usage[ggml_backend_get_device(ctx_clip.backend)] += ggml_nbytes(cur); + model.mem_weights[ggml_backend_buft_get_device(buft)] += ggml_nbytes(cur); } return cur; }; // pocket-tts: the encoder and the decoder share the same layout, only the prefix differs - auto load_seanet = [&](clip_seanet & seanet, bool is_decoder) { + auto load_seanet = [&](mtmd_seanet & seanet, bool is_decoder) { const char * conv_in = is_decoder ? TN_A_GEN_WAV_SEANET_CONV_IN : TN_A_SEANET_CONV_IN; const char * conv_out = is_decoder ? TN_A_GEN_WAV_SEANET_CONV_OUT : TN_A_SEANET_CONV_OUT; const char * res1 = is_decoder ? TN_A_GEN_WAV_SEANET_RES_CONV1 : TN_A_SEANET_RES_CONV1; @@ -3706,7 +2398,7 @@ struct clip_model_loader { if (probe == nullptr) { break; } - clip_flow_net::block blk; + mtmd_flow_net::block blk; blk.norm_w = probe; blk.norm_b = get_tensor(string_format(TN_A_GEN_FLOW_BLK_NORM, il, "bias")); blk.up_w = get_tensor(string_format(TN_A_GEN_FLOW_BLK_UP, il, "weight")); @@ -3866,7 +2558,7 @@ struct clip_model_loader { auto has = [&](const std::string & name) { return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0; }; - auto load_enc_layer = [&](const std::string & p, clip_chatterbox::enc_layer & l) { + auto load_enc_layer = [&](const std::string & p, mtmd_chatterbox::enc_layer & l) { l.norm_mha_w = get_tensor(p + ".norm_mha.weight"); l.norm_mha_b = get_tensor(p + ".norm_mha.bias"); l.attn_q_w = get_tensor(p + ".self_attn.linear_q.weight"); @@ -3887,13 +2579,13 @@ struct clip_model_loader { l.ffn_2_w = get_tensor(p + ".feed_forward.w_2.weight"); l.ffn_2_b = get_tensor(p + ".feed_forward.w_2.bias"); }; - auto load_causal = [&](const std::string & p, clip_chatterbox::causal_block & b) { + auto load_causal = [&](const std::string & p, mtmd_chatterbox::causal_block & b) { b.conv_w = get_tensor(p + ".block.0.weight"); b.conv_b = get_tensor(p + ".block.0.bias"); b.norm_w = get_tensor(p + ".block.2.weight"); b.norm_b = get_tensor(p + ".block.2.bias"); }; - auto load_resnet = [&](const std::string & p, clip_chatterbox::resnet & r) { + auto load_resnet = [&](const std::string & p, mtmd_chatterbox::resnet & r) { load_causal(p + ".block1", r.block1); load_causal(p + ".block2", r.block2); r.mlp_w = get_tensor(p + ".mlp.1.weight"); @@ -3901,7 +2593,7 @@ struct clip_model_loader { r.res_conv_w = get_tensor(p + ".res_conv.weight"); r.res_conv_b = get_tensor(p + ".res_conv.bias"); }; - auto load_tfm = [&](const std::string & p, clip_chatterbox::tfm_block & b) { + auto load_tfm = [&](const std::string & p, mtmd_chatterbox::tfm_block & b) { b.norm1_w = get_tensor(p + ".norm1.weight"); b.norm1_b = get_tensor(p + ".norm1.bias"); b.attn_q_w = get_tensor(p + ".attn1.to_q.weight"); @@ -3916,10 +2608,10 @@ struct clip_model_loader { b.ff_out_w = get_tensor(p + ".ff.net.2.weight"); b.ff_out_b = get_tensor(p + ".ff.net.2.bias"); }; - auto load_stage = [&](const std::string & p, clip_chatterbox::est_stage & s, bool boundary) { + auto load_stage = [&](const std::string & p, mtmd_chatterbox::est_stage & s, bool boundary) { load_resnet(p + ".0", s.res); for (int j = 0; has(p + ".1." + std::to_string(j) + ".norm1.weight"); j++) { - clip_chatterbox::tfm_block b; + mtmd_chatterbox::tfm_block b; load_tfm(p + ".1." + std::to_string(j), b); s.tfm.push_back(b); } @@ -3928,10 +2620,10 @@ struct clip_model_loader { s.conv_b = get_tensor(p + ".2.bias"); } }; - auto load_hres = [&](const std::string & p, clip_chatterbox::hift_res & r) { + auto load_hres = [&](const std::string & p, mtmd_chatterbox::hift_res & r) { for (int j = 0; has(p + ".convs1." + std::to_string(j) + ".weight"); j++) { const std::string js = std::to_string(j); - clip_chatterbox::hift_res_unit u; + mtmd_chatterbox::hift_res_unit u; u.act1_alpha = get_tensor(p + ".activations1." + js + ".alpha"); u.act2_alpha = get_tensor(p + ".activations2." + js + ".alpha"); u.conv1_w = get_tensor(p + ".convs1." + js + ".weight"); @@ -3955,7 +2647,7 @@ struct clip_model_loader { c.pre_conv2_w = get_tensor("a.gen.fenc.pre_lookahead_layer.conv2.weight"); c.pre_conv2_b = get_tensor("a.gen.fenc.pre_lookahead_layer.conv2.bias"); for (int i = 0; has("a.gen.fenc.encoders." + std::to_string(i) + ".norm_mha.weight"); i++) { - clip_chatterbox::enc_layer l; + mtmd_chatterbox::enc_layer l; load_enc_layer("a.gen.fenc.encoders." + std::to_string(i), l); c.enc.push_back(l); } @@ -3966,7 +2658,7 @@ struct clip_model_loader { c.up_embed_norm_w = get_tensor("a.gen.fenc.up_embed.out.1.weight"); c.up_embed_norm_b = get_tensor("a.gen.fenc.up_embed.out.1.bias"); for (int i = 0; has("a.gen.fenc.up_encoders." + std::to_string(i) + ".norm_mha.weight"); i++) { - clip_chatterbox::enc_layer l; + mtmd_chatterbox::enc_layer l; load_enc_layer("a.gen.fenc.up_encoders." + std::to_string(i), l); c.up_enc.push_back(l); } @@ -3983,7 +2675,7 @@ struct clip_model_loader { c.time_embed_mixer_w = get_tensor("a.gen.est.time_embed_mixer.weight", false); load_stage("a.gen.est.down_blocks.0", c.est_down, true); for (int i = 0; has("a.gen.est.mid_blocks." + std::to_string(i) + ".0.block1.block.0.weight"); i++) { - clip_chatterbox::est_stage s; + mtmd_chatterbox::est_stage s; load_stage("a.gen.est.mid_blocks." + std::to_string(i), s, false); c.est_mid.push_back(std::move(s)); } @@ -3999,7 +2691,7 @@ struct clip_model_loader { c.hift_post_b = get_tensor("a.gen.hift.conv_post.bias"); for (int i = 0; has("a.gen.hift.ups." + std::to_string(i) + ".weight"); i++) { const std::string is = std::to_string(i); - clip_chatterbox::hift_up up; + mtmd_chatterbox::hift_up up; up.up_w = get_tensor("a.gen.hift.ups." + is + ".weight"); up.up_b = get_tensor("a.gen.hift.ups." + is + ".bias"); up.source_down_w = get_tensor("a.gen.hift.source_downs." + is + ".weight"); @@ -4012,7 +2704,7 @@ struct clip_model_loader { } for (int i = 0; has("a.gen.hift.f0_predictor.condnet." + std::to_string(i) + ".weight"); i += 2) { const std::string is = std::to_string(i); - clip_chatterbox::f0_conv fc; + mtmd_chatterbox::f0_conv fc; fc.w = get_tensor("a.gen.hift.f0_predictor.condnet." + is + ".weight"); fc.b = get_tensor("a.gen.hift.f0_predictor.condnet." + is + ".bias"); c.f0_condnet.push_back(fc); @@ -4027,7 +2719,7 @@ struct clip_model_loader { c.s3tok_conv2_b = get_tensor("a.s3tok.encoder.conv2.bias"); for (int i = 0; has("a.s3tok.encoder.blocks." + std::to_string(i) + ".attn_ln.weight"); i++) { const std::string p = "a.s3tok.encoder.blocks." + std::to_string(i); - clip_chatterbox::s3tok_block b; + mtmd_chatterbox::s3tok_block b; b.attn_ln_w = get_tensor(p + ".attn_ln.weight"); b.attn_ln_b = get_tensor(p + ".attn_ln.bias"); b.attn_q_w = get_tensor(p + ".attn.query.weight"); @@ -4121,13 +2813,13 @@ struct clip_model_loader { auto has = [&](const std::string & name) { return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0; }; - auto load_bn = [&](const std::string & p, clip_chatterbox::bn & n) { + auto load_bn = [&](const std::string & p, mtmd_chatterbox::bn & n) { n.mean = get_tensor(p + ".running_mean"); n.var = get_tensor(p + ".running_var"); n.w = get_tensor(p + ".weight", false); n.b = get_tensor(p + ".bias", false); }; - auto load_res2d = [&](const std::string & p, clip_chatterbox::spk_res2d & r) { + auto load_res2d = [&](const std::string & p, mtmd_chatterbox::spk_res2d & r) { r.conv1_w = get_tensor(p + ".conv1.weight"); load_bn(p + ".bn1", r.bn1); r.conv2_w = get_tensor(p + ".conv2.weight"); @@ -4137,10 +2829,10 @@ struct clip_model_loader { load_bn(p + ".shortcut.1", r.shortcut_bn); } }; - auto load_cam_block = [&](const std::string & bp, const std::string & tp, clip_chatterbox::spk_cam_block & blk) { + auto load_cam_block = [&](const std::string & bp, const std::string & tp, mtmd_chatterbox::spk_cam_block & blk) { for (int li = 1; has(bp + ".tdnnd" + std::to_string(li) + ".linear1.weight"); li++) { const std::string p = bp + ".tdnnd" + std::to_string(li); - clip_chatterbox::spk_cam_layer l; + mtmd_chatterbox::spk_cam_layer l; load_bn(p + ".nonlinear1.batchnorm", l.nl1_bn); l.linear1_w = get_tensor(p + ".linear1.weight"); load_bn(p + ".nonlinear2.batchnorm", l.nl2_bn); @@ -4812,14 +3504,13 @@ struct clip_model_loader { } // alloc memory and offload data - ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(ctx_clip.backend); - ctx_clip.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_clip.ctx_data.get(), buft)); - ggml_backend_buffer_set_usage(ctx_clip.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + model.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(model.ctx_data.get(), buft)); + ggml_backend_buffer_set_usage(model.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); // read the weight from file - if (!ctx_clip.no_alloc) { + if (!no_alloc) { size_t data_loaded = 0; for (auto & t : tensors_to_load) { - ggml_tensor * cur = ggml_get_tensor(ctx_clip.ctx_data.get(), t->name); + ggml_tensor * cur = ggml_get_tensor(model.ctx_data.get(), t->name); GGML_ASSERT(cur && "tensor not found in ctx_data"); auto it_off = tensor_offset.find(t->name); GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor"); @@ -4855,170 +3546,6 @@ struct clip_model_loader { } - struct support_info_op { - ggml_tensor * op; - - // true if the op runs on the accelerated ctx_clip.backend - bool is_accel = true; - }; - - struct support_info_graph { - // whether the clip_ctx.backend supports flash attention - bool fattn = true; - ggml_tensor * fattn_op = nullptr; // for debugging - - std::vector ops; - }; - - static clip_image_f32_batch get_dummy_batch(clip_ctx & ctx_clip) { - // create a fake batch - const auto & hparams = ctx_clip.model.hparams; - clip_image_f32_batch batch; - clip_image_f32 img; - if (ctx_clip.model.modality == CLIP_MODALITY_VISION) { - const int sz = hparams.warmup_image_size; - img.set_size({sz, sz}, false, false); - LOG_INF("%s: warmup with image size = %d x %d\n", __func__, sz, sz); - } else { - // GEMMA4UA uses n_mel_bins as a raw-waveform frame size (640), not a mel-bin count, - // so the [1, 256] bound only applies to FFT-based models. - const bool fft_based = ctx_clip.model.proj_type != PROJECTOR_TYPE_GEMMA4UA; - if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) { - throw std::runtime_error(string_format("%s: invalid n_mel_bins (%d), must be in [1, 256]\n", __func__, hparams.n_mel_bins)); - } - img.set_size({hparams.warmup_audio_size, hparams.n_mel_bins}, false, false); - LOG_INF("%s: warmup with audio size = %d\n", __func__, hparams.warmup_audio_size); - } - batch.entries.push_back(img); - return batch; - } - - static void init_ctx(clip_ctx & ctx_clip) { - ctx_clip.buf_compute_meta.resize(ctx_clip.max_nodes * ggml_tensor_overhead() + ggml_graph_overhead()); - - // check batching support - auto batch = get_dummy_batch(ctx_clip); - auto builder = clip_get_graph_builder(&ctx_clip, batch); - ctx_clip.support_batch = builder->support_batch(); - } - - static void warmup(clip_ctx & ctx_clip) { - auto batch = get_dummy_batch(ctx_clip); - warmup(ctx_clip, batch); - } - - static void warmup(clip_ctx & ctx_clip, const clip_image_f32_batch & batch) { - support_info_graph info; - - if (ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_AUTO) { - // try to enable flash attention to see if it's supported - ctx_clip.flash_attn_type = CLIP_FLASH_ATTN_TYPE_ENABLED; - info = reserve_compute_meta(ctx_clip, batch); - if (!info.fattn && info.fattn_op) { - auto op = info.fattn_op; - LOG_WRN("%s: *****************************************************************\n", __func__); - LOG_WRN("%s: WARNING: flash attention not supported by %s, memory usage will increase\n", __func__, ggml_backend_name(ctx_clip.backend)); - LOG_WRN("%s: op params: \n", __func__); - static auto print_shape = [](const char * fn, const char * name, ggml_tensor * t) { - LOG_WRN("%s: %s: type = %s, ne = [%d %d %d %d], nb = [%d %d %d %d]\n", fn, - name, ggml_type_name(t->type), - t->ne[0], t->ne[1], t->ne[2], t->ne[3], - t->nb[0], t->nb[1], t->nb[2], t->nb[3]); - }; - print_shape(__func__, " dst", op); - print_shape(__func__, "src0", op->src[0]); - print_shape(__func__, "src1", op->src[1]); - print_shape(__func__, "src2", op->src[2]); - LOG_WRN("%s: please report this on github as an issue\n", __func__); - LOG_WRN("%s: *****************************************************************\n", __func__); - ctx_clip.flash_attn_type = CLIP_FLASH_ATTN_TYPE_DISABLED; - reserve_compute_meta(ctx_clip, batch); - } - } else { - info = reserve_compute_meta(ctx_clip, batch); - if (!info.fattn && ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { - LOG_WRN("%s: flash attention is not supported by the current backend; falling back to CPU (performance will be degraded)\n", __func__); - } - } - - ctx_clip.is_allocated = true; // mark buffers as allocated - - LOG_INF("%s: flash attention is %s\n", __func__, - (ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled"); - - // print ops that are not supported by the GPU backend (if there is one) - if (ctx_clip.backend && ctx_clip.backend != ctx_clip.backend_cpu) { - std::vector unsupported_ops; - for (const auto & op : info.ops) { - if (!op.is_accel) { - unsupported_ops.push_back(op); - } - } - if (!unsupported_ops.empty()) { - LOG_WRN("%s: *****************************************************************\n", __func__); - LOG_WRN("%s: WARNING: the CLIP graph uses unsupported operators by the backend\n", __func__); - LOG_WRN("%s: the performance will be suboptimal \n", __func__); - LOG_WRN("%s: list of unsupported ops (backend=%s):\n", __func__, ggml_backend_name(ctx_clip.backend)); - for (const auto & op : unsupported_ops) { - LOG_WRN("%s: %16s: type = %s, ne = [%d %d %d %d]\n", __func__, - ggml_op_name(op.op->op), - ggml_type_name(op.op->type), - op.op->ne[0], op.op->ne[1], op.op->ne[2], op.op->ne[3]); - } - LOG_WRN("%s: flash attention is %s\n", __func__, - (ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled"); - LOG_WRN("%s: please report this on github as an issue\n", __func__); - LOG_WRN("%s: ref: https://github.com/ggml-org/llama.cpp/pull/16837#issuecomment-3461676118\n", __func__); - LOG_WRN("%s: *****************************************************************\n", __func__); - } - } - } - - // only initialize backend buffers, but do not allocate them yet - static support_info_graph reserve_compute_meta(clip_ctx & ctx_clip, const clip_image_f32_batch & batch) { - ggml_cgraph * gf = clip_get_graph_builder(&ctx_clip, batch)->build(); - ggml_backend_sched_reserve(ctx_clip.sched.get(), gf); - - ctx_clip.mem_compute.clear(); - for (size_t i = 0; i < ctx_clip.backend_ptrs.size(); ++i) { - ggml_backend_t backend = ctx_clip.backend_ptrs[i]; - ggml_backend_buffer_type_t buft = ctx_clip.backend_buft[i]; - size_t size = ggml_backend_sched_get_buffer_size(ctx_clip.sched.get(), backend); - if (size > 1) { - LOG_INF("%s: %10s compute buffer size = %8.2f MiB\n", __func__, - ggml_backend_buft_name(buft), - size / 1024.0 / 1024.0); - } - ctx_clip.mem_compute[ggml_backend_get_device(backend)] += size; - } - - const int n_splits = ggml_backend_sched_get_n_splits(ctx_clip.sched.get()); - const int n_nodes = ggml_graph_n_nodes(gf); - - LOG_INF("%s: graph splits = %d, nodes = %d\n", __func__, n_splits, n_nodes); - - support_info_graph res { - /*.fattn = */ true, - /*.fattn_op = */ nullptr, - /*.ops = */ {}, - }; - - // check op support - for (int i = 0; i < ggml_graph_n_nodes(gf); i++) { - ggml_tensor * node = ggml_graph_node(gf, i); - res.ops.push_back({node, true}); - if (!ggml_backend_supports_op(ctx_clip.backend, node)) { - res.ops.back().is_accel = false; - if (node->op == GGML_OP_FLASH_ATTN_EXT) { - res.fattn = false; - res.fattn_op = node; - } - } - } - - return res; - } - void get_bool(const std::string & key, bool & output, bool required = true) const { const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); if (i < 0) { @@ -5132,14 +3659,14 @@ struct clip_model_loader { } } - static void set_llava_uhd_res_candidates(clip_model & model, const int max_patches_per_side) { + static void set_llava_uhd_res_candidates(mtmd_model & model, const int max_patches_per_side) { auto & hparams = model.hparams; for (int x = 1; x <= max_patches_per_side; x++) { for (int y = 1; y <= max_patches_per_side; y++) { if (x == 1 && y == 1) { continue; // skip the first point } - hparams.image_res_candidates.push_back(clip_image_size{ + hparams.image_res_candidates.push_back(mtmd_image_size{ x*hparams.image_size, y*hparams.image_size, }); @@ -5147,7 +3674,7 @@ struct clip_model_loader { } } - static void set_internvl_dhr_res_candidates(clip_model & model) { + static void set_internvl_dhr_res_candidates(mtmd_model & model) { auto & hparams = model.hparams; int min_num = hparams.preproc_min_tiles; int max_num = hparams.preproc_max_tiles; @@ -5160,7 +3687,7 @@ struct clip_model_loader { b_lo = std::max(b_lo, min_num); b_hi = std::min(b_hi, max_num); for (int b = b_lo; b <= b_hi; ++b) { - hparams.image_res_candidates.push_back(clip_image_size { + hparams.image_res_candidates.push_back(mtmd_image_size { a*hparams.image_size, b*hparams.image_size, }); @@ -5169,95 +3696,59 @@ struct clip_model_loader { } }; -struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params) { - clip_ctx * ctx_vision = nullptr; - clip_ctx * ctx_audio = nullptr; - clip_ctx * ctx_gen_audio = nullptr; +// +// loading +// - std::shared_ptr threadpool; +mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params, ggml_backend_buffer_type_t buft) { + mtmd_model_loader loader(fname, + /* skip_tensors */ false, + params.progress_callback, + params.progress_callback_user_data); - try { - // no_alloc only measures memory, nothing is computed - threadpool = clip_threadpool_init(ctx_params.no_alloc ? 1 : ctx_params.n_threads); + auto load = [&](mtmd_modality modality) { + auto model = std::make_unique(); + model->hparams.custom_image_min_tokens = params.image_min_tokens; + model->hparams.custom_image_max_tokens = params.image_max_tokens; + loader.load_hparams(*model, modality); + loader.load_tensors(*model, buft, params.no_alloc); + return model; + }; - clip_model_loader loader(fname, - /* skip_tensors */ false, - ctx_params.progress_callback, - ctx_params.progress_callback_user_data); - bool skip_audio = false; - - if (loader.has_vision) { - ctx_vision = new clip_ctx(ctx_params, threadpool); - loader.load_hparams(ctx_vision->model, CLIP_MODALITY_VISION); - loader.load_tensors(*ctx_vision); - loader.init_ctx(*ctx_vision); - if (ctx_params.warmup) { - loader.warmup(*ctx_vision); - } - - // TODO: we don't support audio for Gemma 3N, but GGUF contains audio tensors - // we can remove this check when we implement audio support for Gemma 3N - skip_audio = ctx_vision->model.proj_type == PROJECTOR_TYPE_GEMMA3NV; - } - - if (loader.has_audio && !skip_audio) { - ctx_audio = new clip_ctx(ctx_params, threadpool); - loader.load_hparams(ctx_audio->model, CLIP_MODALITY_AUDIO); - loader.load_tensors(*ctx_audio); - loader.init_ctx(*ctx_audio); - if (ctx_params.warmup) { - loader.warmup(*ctx_audio); - } - } - - if (loader.has_gen_audio) { - ctx_gen_audio = new clip_ctx(ctx_params, threadpool); - loader.load_hparams(ctx_gen_audio->model, CLIP_MODALITY_GEN_AUDIO); - // TODO: CUDA has no flash attention kernel for the depthformer head dim 32, enable it once it does - if (ctx_gen_audio->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN && ctx_gen_audio->flash_attn_type == CLIP_FLASH_ATTN_TYPE_AUTO) { - ctx_gen_audio->flash_attn_type = CLIP_FLASH_ATTN_TYPE_DISABLED; - } - loader.load_tensors(*ctx_gen_audio); - if (ctx_gen_audio->model.proj_type == PROJECTOR_TYPE_CHATTERBOX) { - // the classic cfm solver unrolls up to 10 steps x 2 cfg estimator evaluations in one graph - ctx_gen_audio->max_nodes = 65536; - ctx_gen_audio->sched.reset( - ggml_backend_sched_new(ctx_gen_audio->backend_ptrs.data(), ctx_gen_audio->backend_buft.data(), - ctx_gen_audio->backend_ptrs.size(), ctx_gen_audio->max_nodes, false, true)); - if (ctx_params.cb_eval != nullptr) { - ggml_backend_sched_set_eval_callback(ctx_gen_audio->sched.get(), ctx_params.cb_eval, ctx_params.cb_eval_user_data); - } - } - // TODO: fix warmup - ctx_gen_audio->buf_compute_meta.resize(ctx_gen_audio->max_nodes * ggml_tensor_overhead() + - ggml_graph_overhead_custom(ctx_gen_audio->max_nodes, false)); - } - - } catch (const std::exception & e) { - LOG_ERR("%s: failed to load model '%s': %s\n", __func__, fname, e.what()); - - delete ctx_vision; - delete ctx_audio; - delete ctx_gen_audio; - - return {nullptr, nullptr, nullptr}; + mtmd_models res; + if (loader.has_vision) { + res.v = load(MTMD_MODALITY_VISION); } - return {ctx_vision, ctx_audio, ctx_gen_audio}; + // TODO: we don't support audio for Gemma 3N, but GGUF contains audio tensors + // we can remove this check when we implement audio support for Gemma 3N + const bool skip_audio = res.v && res.v->proj_type == PROJECTOR_TYPE_GEMMA3NV; + if (loader.has_audio && !skip_audio) { + res.a = load(MTMD_MODALITY_AUDIO); + } + + if (loader.has_gen_audio) { + res.gen_a = load(MTMD_MODALITY_GEN_AUDIO); + } + + return res; } -struct clip_cap clip_get_cap(const char * fname) { - clip_cap res; - clip_model_loader loader(fname, /* skip_tensors= */ true); +mtmd_model_caps mtmd_model_get_caps(const char * fname) { + mtmd_model_caps res; + mtmd_model_loader loader(fname, /* skip_tensors= */ true); res.has_vision = loader.has_vision; res.has_audio = loader.has_audio; res.has_gen_audio = loader.has_gen_audio; return res; } -int clip_get_image_max_tokens(const clip_ctx * ctx) { - const auto & hparams = ctx->model.hparams; - if (ctx->proj_type() == PROJECTOR_TYPE_DEEPSEEK4V || ctx->proj_type() == PROJECTOR_TYPE_DEEPSEEK41V) { +// +// mtmd_model +// + +int mtmd_model::image_max_tokens() const { + if (proj_type == PROJECTOR_TYPE_DEEPSEEK4V || proj_type == PROJECTOR_TYPE_DEEPSEEK41V) { return hparams.dsv4_max_n_token; } if (hparams.image_max_pixels <= 0) { @@ -5267,21 +3758,10 @@ int clip_get_image_max_tokens(const clip_ctx * ctx) { return hparams.image_max_pixels / patch_area; } -void clip_free(clip_ctx * ctx) { - if (ctx == nullptr) { - return; - } - delete ctx; -} - -const char * clip_patch_merge_type(const struct clip_ctx * ctx) { - return ctx->model.hparams.mm_patch_merge_type == PATCH_MERGE_SPATIAL_UNPAD ? "spatial_unpad" : "flat"; -} - -int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) { - const auto & params = ctx->model.hparams; - const int n_total = clip_n_output_tokens(ctx, img); - const auto & proj = ctx->proj_type(); +int mtmd_model::n_output_tokens_x(const mtmd_image_f32 * img) const { + const auto & params = hparams; + const int n_total = n_output_tokens(img); + const auto & proj = proj_type; switch (proj) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: @@ -5313,9 +3793,9 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) { return n_total; } -int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) { - const auto & params = ctx->model.hparams; - const auto & proj = ctx->proj_type(); +int mtmd_model::n_output_tokens_y(const mtmd_image_f32 * img) const { + const auto & params = hparams; + const auto & proj = proj_type; switch (proj) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: @@ -5344,14 +3824,14 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) { return 1; } -int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { - const auto & params = ctx->model.hparams; +int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const { + const auto & params = hparams; // for models with fixed size image, the input image is already pre-processed and resized to square int patch_size = params.patch_size; int n_patches = (img->nx() / patch_size) * (img->ny() / patch_size); - projector_type proj = ctx->proj_type(); + projector_type proj = proj_type; switch (proj) { case PROJECTOR_TYPE_MLP: @@ -5370,7 +3850,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_GLM_EDGE: { n_patches /= 4; - if (ctx->model.mm_boi) { + if (mm_boi) { n_patches += 2; // for BOI and EOI token embeddings } } break; @@ -5444,14 +3924,14 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_LLAMA4: { // both X and Y are downscaled by the scale factor - int scale_factor = ctx->model.hparams.n_merge; + int scale_factor = hparams.n_merge; n_patches /= (scale_factor * scale_factor); } break; case PROJECTOR_TYPE_GEMMA3NV: { // MobileNetV5 MSFA adapter always outputs fixed 16x16 resolution // regardless of input size (see architecture description) - n_patches = ctx->model.hparams.image_size / ctx->model.hparams.patch_size; + n_patches = hparams.image_size / hparams.patch_size; } break; case PROJECTOR_TYPE_LFM2: case PROJECTOR_TYPE_D1OMNI_V: @@ -5459,23 +3939,23 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_KIMIK25: { // dynamic size - int out_patch_size = params.patch_size * ctx->model.hparams.n_merge; - int x_patch = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size; - int y_patch = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size; + int out_patch_size = params.patch_size * hparams.n_merge; + int x_patch = MTMD_ALIGN(img->nx(), out_patch_size) / out_patch_size; + int y_patch = MTMD_ALIGN(img->ny(), out_patch_size) / out_patch_size; n_patches = x_patch * y_patch; } break; case PROJECTOR_TYPE_DEEPSEEK4V: { const int out_patch_size = params.patch_size * params.n_merge; - const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size; - const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size; + const int n_llm_w = MTMD_ALIGN(img->nx(), out_patch_size) / out_patch_size; + const int n_llm_h = MTMD_ALIGN(img->ny(), out_patch_size) / out_patch_size; n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out; } break; case PROJECTOR_TYPE_DEEPSEEK41V: { const int out_patch_size = params.patch_size * params.n_merge; - const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size; - const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size; + const int n_llm_w = MTMD_ALIGN(img->nx(), out_patch_size) / out_patch_size; + const int n_llm_h = MTMD_ALIGN(img->ny(), out_patch_size) / out_patch_size; n_patches = dsv41_n_block_tokens(n_llm_w, n_llm_h); } break; case PROJECTOR_TYPE_PADDLEOCR: @@ -5483,9 +3963,9 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_DOTS3NOTE_V: { // dynamic size - int n_merge = ctx->model.hparams.n_merge; + int n_merge = hparams.n_merge; int stride = n_merge * n_merge; - n_patches = CLIP_ALIGN(n_patches, stride) / stride; + n_patches = MTMD_ALIGN(n_patches, stride) / stride; } break; case PROJECTOR_TYPE_DOTS3NOTE_A: { @@ -5496,10 +3976,10 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_LIGHTONOCR: { // dynamic size - int n_merge = ctx->model.hparams.n_merge; + int n_merge = hparams.n_merge; int n_patches_x = img->nx() / patch_size / n_merge; int n_patches_y = img->ny() / patch_size / n_merge; - if (ctx->model.token_embd_img_break) { + if (token_embd_img_break) { n_patches = n_patches_y * n_patches_x + n_patches_y - 1; // + one [IMG_BREAK] per row, except the last row } else { n_patches = n_patches_y * n_patches_x; @@ -5508,11 +3988,11 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_VOXTRAL_RT: { // 2 context frames, conv stride 2, then stacked frames - n_patches = (img->nx() - 2) / 2 / ctx->model.hparams.proj_stack_factor; + n_patches = (img->nx() - 2) / 2 / hparams.proj_stack_factor; } break; case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: { - const int64_t fold = ctx->model.hparams.audio_samples_per_token(); + const int64_t fold = hparams.audio_samples_per_token(); n_patches = (int) ((img->nx() + fold - 1) / fold); } break; case PROJECTOR_TYPE_VOXTRAL: @@ -5523,17 +4003,17 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { { n_patches = img->nx(); - const int proj_stack_factor = ctx->model.hparams.proj_stack_factor; - if (ctx->model.audio_has_stack_frames()) { + const int proj_stack_factor = hparams.proj_stack_factor; + if (audio_has_stack_frames()) { GGML_ASSERT(proj_stack_factor > 0); - const int n_len = CLIP_ALIGN(n_patches, proj_stack_factor); + const int n_len = MTMD_ALIGN(n_patches, proj_stack_factor); n_patches = n_len / proj_stack_factor; } // whisper downscales input token by half after conv1d n_patches /= 2; - if (ctx->model.audio_has_avgpool()) { + if (audio_has_avgpool()) { // divide by 2 because of nn.AvgPool1d(2, stride=2) n_patches /= 2; } @@ -5551,7 +4031,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { // whisper downscales input token by half after conv1d n_patches /= 2; // reshape by merge_factor - n_patches /= ctx->model.hparams.proj_stack_factor; + n_patches /= hparams.proj_stack_factor; // for BOI and EOI token embeddings n_patches += 2; } break; @@ -5580,7 +4060,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { } break; case PROJECTOR_TYPE_HUNYUANVL: { - int merge = ctx->model.hparams.n_merge; + int merge = hparams.n_merge; int ow = (img->nx() / patch_size) / merge; int oh = (img->ny() / patch_size) / merge; n_patches = (ow + 1) * oh + 2; @@ -5630,8 +4110,8 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { } break; case PROJECTOR_TYPE_GRANITE_SPEECH: { - const int ws = ctx->model.hparams.audio_proj_window_size; - const int ds = ctx->model.hparams.audio_proj_downsample_rate; + const int ws = hparams.audio_proj_window_size; + const int ds = hparams.audio_proj_downsample_rate; n_patches = ((img->nx() + ws - 1) / ws) * (ws / ds); } break; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: @@ -5654,7 +4134,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_POCKETTTS_SPKENC: { // one conditioning row per 12.5Hz frame - const int hop = ctx->model.hparams.mimi_downsample * 120; + const int hop = hparams.mimi_downsample * 120; n_patches = img->nx() / hop; } break; case PROJECTOR_TYPE_POCKETTTS_GEN: @@ -5667,8 +4147,8 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { // Per-tile output token count: each projector block outputs // query_side^2 tokens per window x n^2 windows. // For 384x384 input: n = 24/8 = 3, query_side = 4 -> 144. - const int window_side = ctx->model.hparams.downsample_window_side; - const int query_side = ctx->model.hparams.downsample_query_side; + const int window_side = hparams.downsample_window_side; + const int query_side = hparams.downsample_query_side; const int side = img->nx() / params.patch_size; const int n = side / window_side; const int out_side = query_side * n; @@ -5676,7 +4156,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { if (img->anyres.is_tiled()) { // overview tile, then the unpadded tile grid with one newline per row int off_x, off_y, w, h; - clip_anyres_unpad(img->anyres.grid_x * out_side, img->anyres.grid_y * out_side, + mtmd_anyres_unpad(img->anyres.grid_x * out_side, img->anyres.grid_y * out_side, img->anyres.orig_nx, img->anyres.orig_ny, off_x, off_y, w, h); n_patches += h * (w + 1); } @@ -5685,311 +4165,13 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { GGML_ABORT("unsupported projector type"); } - n_patches += (int) clip_suffix_rows(img->suffix_type).size(); + n_patches += (int) mtmd_suffix_rows(img->suffix_type).size(); return n_patches; } -ggml_backend_sched_t clip_get_sched(const struct clip_ctx * ctx) { - return ctx->sched.get(); -} - -// TODO: remove this and use ggml_mul_mat with GGML_PREC_F32 once all backends honor it for F32 x F32 -// ggml_mul_mat for 2D F32 tensors, always in F32 on CPU: some GPU backends round the inputs to F16 or TF32, which loses the quiet bins of a spectrum -static void mtmd_mul_mat_prec_op(ggml_tensor * dst, int ith, int nth, void * /*userdata*/) { - const ggml_tensor * a = dst->src[0]; - const ggml_tensor * b = dst->src[1]; - const int64_t n_k = a->ne[0]; - - for (int64_t n = ith; n < b->ne[1]; n += nth) { - const float * b_row = (const float *) ((const char *) b->data + n * b->nb[1]); - float * dst_row = (float *) ((char *) dst->data + n * dst->nb[1]); - for (int64_t m = 0; m < a->ne[1]; m++) { - const float * a_row = (const float *) ((const char *) a->data + m * a->nb[1]); - // independent partial sums, so that the compiler can vectorize the loop - float acc[16] = {}; - int64_t k = 0; - for (; k + 16 <= n_k; k += 16) { - for (int l = 0; l < 16; l++) { - acc[l] += a_row[k + l] * b_row[k + l]; - } - } - float sum = 0.0f; - for (; k < n_k; k++) { - sum += a_row[k] * b_row[k]; - } - for (int l = 0; l < 16; l++) { - sum += acc[l]; - } - dst_row[m] = sum; - } - } -} - -static ggml_tensor * mtmd_mul_mat_prec(ggml_context * ctx, ggml_tensor * a, ggml_tensor * b) { - GGML_ASSERT(a->type == GGML_TYPE_F32 && b->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_n_dims(a) <= 2 && ggml_n_dims(b) <= 2 && a->ne[0] == b->ne[0]); - GGML_ASSERT(a->nb[0] == sizeof(float) && b->nb[0] == sizeof(float)); - ggml_tensor * args[] = { a, b }; - return ggml_custom_4d(ctx, GGML_TYPE_F32, a->ne[1], b->ne[1], 1, 1, args, 2, mtmd_mul_mat_prec_op, GGML_N_TASKS_MAX, nullptr); -} - -bool clip_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out) { - ggml_backend_sched_t sched = config.sched; - ggml_backend_t backend_cpu = ggml_backend_sched_get_backend(sched, ggml_backend_sched_get_n_backends(sched) - 1); - clip_backend_cpu_set_n_threads(backend_cpu, std::max(1, config.n_threads)); - - const auto & cache = *config.cache; - const auto & filters = cache.filters.data; - const int64_t n_fft_bins = config.n_fft_bins; - const int64_t frame_size = (n_fft_bins - 1) * 2; - const int64_t n_mel = config.n_mel; - GGML_ASSERT(filters.size() == (size_t) (n_mel * n_fft_bins)); - GGML_ASSERT(cache.cos_vals.size() == (size_t) frame_size && cache.sin_vals.size() == (size_t) frame_size); - - // DFT matrix with the window folded in, the first n_fft_bins rows give the real part - std::vector dft((size_t) 2 * n_fft_bins * frame_size); - for (int64_t k = 0; k < n_fft_bins; k++) { - for (int64_t j = 0; j < frame_size; j++) { - const int64_t q = (j * k) % frame_size; - dft[k * frame_size + j] = window[j] * cache.cos_vals[q]; - dft[(k + n_fft_bins) * frame_size + j] = window[j] * cache.sin_vals[q]; - } - } - - // the spectrum is several times larger than the result, so long audio is processed in blocks - const int64_t n_frames_max = 8192; - - for (int64_t off = 0; off < n_frames; off += n_frames_max) { - const int64_t n_cur = std::min(n_frames_max, n_frames - off); - - ggml_init_params params = { - /*.mem_size =*/ ggml_tensor_overhead() * 32 + ggml_graph_overhead(), - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - ggml_context_ptr ctx_ptr(ggml_init(params)); - ggml_context * ctx0 = ctx_ptr.get(); - - ggml_tensor * inp_samples = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, (n_cur - 1) * config.hop_length + frame_size); - ggml_tensor * inp_dft = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, frame_size, 2 * n_fft_bins); - ggml_tensor * inp_filters = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_fft_bins, n_mel); - ggml_set_input(inp_samples); - ggml_set_input(inp_dft); - ggml_set_input(inp_filters); - - ggml_tensor * cur = ggml_cont(ctx0, ggml_view_2d(ctx0, inp_samples, frame_size, n_cur, config.hop_length * ggml_element_size(inp_samples), 0)); - cur = mtmd_mul_mat_prec(ctx0, inp_dft, cur); // [2 * n_fft_bins, n_cur] - - cur = ggml_sqr(ctx0, cur); - cur = ggml_add(ctx0, - ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], 0), - ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], n_fft_bins * ggml_element_size(cur))); - if (config.use_magnitude) { - cur = ggml_sqrt(ctx0, cur); - } - - cur = mtmd_mul_mat_prec(ctx0, cur, inp_filters); // [n_cur, n_mel] - cur = config.mel_floor_add - ? ggml_scale_bias(ctx0, cur, 1.0f, config.mel_floor) - : ggml_clamp(ctx0, cur, config.mel_floor, FLT_MAX); - cur = ggml_log(ctx0, cur); - if (!config.use_natural_log) { - cur = ggml_scale(ctx0, cur, 1.0f / logf(10.0f)); - } - ggml_set_output(cur); - - ggml_cgraph * gf = ggml_new_graph(ctx0); - ggml_build_forward_expand(gf, cur); - - ggml_backend_sched_reset(sched); - if (!ggml_backend_sched_alloc_graph(sched, gf)) { - LOG_ERR("%s: failed to allocate compute graph\n", __func__); - return false; - } - - ggml_backend_tensor_set(inp_samples, samples + off * config.hop_length, 0, ggml_nbytes(inp_samples)); - ggml_backend_tensor_set(inp_dft, dft.data(), 0, ggml_nbytes(inp_dft)); - ggml_backend_tensor_set(inp_filters, filters.data(), 0, ggml_nbytes(inp_filters)); - - auto status = ggml_backend_sched_graph_compute(sched, gf); - if (status != GGML_STATUS_SUCCESS) { - LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status); - return false; - } - - for (int64_t j = 0; j < n_mel; j++) { - ggml_backend_tensor_get(cur, out + j * n_frames + off, j * cur->nb[1], n_cur * ggml_element_size(cur)); - } - } - - return true; -} - -bool clip_image_encode(struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector & out_vec) { - clip_image_f32_batch imgs; - clip_image_f32 img_copy = *img; - imgs.entries.push_back(std::move(img_copy)); - - return clip_image_batch_encode(ctx, n_threads, &imgs, out_vec); -} - -bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32_batch * imgs_c_ptr, std::vector & out_batch_embd) { - clip_encode_params params; - params.imgs = imgs_c_ptr; - params.n_threads = n_threads; - params.out_embd = &out_batch_embd; - - if (ctx->proj_type() == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) { - return clip_graph_nemotron_transcribe::encode_windows(ctx, ctx->model, ¶ms); - } - return clip_encode(ctx, ¶ms); -} - -// persisted state slots of the gen-audio decoders and the streaming audio encoders -static std::vector list_gen_state_slots(const clip_hparams & hparams, const clip_model & model) { - switch (model.proj_type) { - case PROJECTOR_TYPE_QWEN3TTS_GEN: return list_c2w_state_slots(hparams, model); - case PROJECTOR_TYPE_DRAMA3_GEN: return list_drama3_state_slots(hparams, model); - case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model); - case PROJECTOR_TYPE_LFM3A_GEN: return list_lfm3a_gen_state_slots(hparams, model); - default: return {}; - } -} - -// state_out can have another shape than its slot, only the byte size must match -static void copy_gen_state_out(ggml_tensor * src, ggml_tensor * dst) { - GGML_ASSERT(ggml_nbytes(src) == ggml_nbytes(dst)); - if (src->type == dst->type && ggml_are_same_shape(src, dst) && ggml_is_contiguous(src)) { - ggml_backend_tensor_copy(src, dst); - return; - } - std::vector tmp(ggml_nbytes(src)); - ggml_backend_tensor_get(src, tmp.data(), 0, tmp.size()); - ggml_backend_tensor_set(dst, tmp.data(), 0, tmp.size()); -} - -clip_gen_state * clip_gen_state_init(clip_ctx * ctx, uint32_t seed) { - auto st = std::make_unique(); - st->rng.seed(seed == UINT32_MAX ? std::random_device{}() : seed); - - const auto slots = list_gen_state_slots(ctx->model.hparams, ctx->model); - if (slots.empty()) { - return st.release(); - } - - ggml_init_params params = { - /*.mem_size =*/ ggml_tensor_overhead() * slots.size(), - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - st->ctx_data.reset(ggml_init(params)); - for (const auto & slot : slots) { - ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1); - ggml_format_name(t, "gen_state_%s", slot.name.c_str()); - st->slots.push_back(t); - } - st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), ctx->backend)); - if (!st->buf) { - LOG_ERR("%s: failed to allocate the gen state\n", __func__); - return nullptr; - } - // a zero state is the cold start - ggml_backend_buffer_clear(st->buf.get(), 0); - - return st.release(); -} - -void clip_gen_state_free(clip_gen_state * state) { - delete state; -} - -void clip_gen_state_reset(clip_gen_state * state) { - if (state->buf) { - ggml_backend_buffer_clear(state->buf.get(), 0); - } -} - -const std::map & clip_gen_get_voices(const clip_ctx * ctx) { - return ctx->model.gen_voices; -} - -clip_enc_state * clip_enc_state_init(clip_ctx * ctx, int32_t n_frames_max) { - const auto & hparams = ctx->model.hparams; - if (ctx->proj_type() == PROJECTOR_TYPE_LFM3A) { - // downsampler context, short-conv taps and attention window, see list_lfm3a_state_slots() - const auto slots = list_lfm3a_state_slots(hparams, ctx->model); - auto st = std::make_unique(); - ggml_init_params params = { - /*.mem_size =*/ ggml_tensor_overhead() * slots.size(), - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - st->ctx_data.reset(ggml_init(params)); - for (const auto & slot : slots) { - ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1); - ggml_format_name(t, "enc_state_%s", slot.name.c_str()); - st->slots.push_back(t); - } - st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), ctx->backend)); - if (!st->buf) { - LOG_ERR("%s: failed to allocate the encoder state\n", __func__); - return nullptr; - } - // a zero state is the cold start - ggml_backend_buffer_clear(st->buf.get(), 0); - return st.release(); - } - if (ctx->proj_type() != PROJECTOR_TYPE_VOXTRAL_RT || hparams.attn_window_size <= 0) { - LOG_ERR("%s: the encoder does not support streaming input\n", __func__); - return nullptr; - } - if (n_frames_max <= 0) { - LOG_ERR("%s: invalid n_frames_max %d\n", __func__, n_frames_max); - return nullptr; - } - - const int n_layer = hparams.n_layer; - const int d_head = hparams.n_embd_head > 0 ? hparams.n_embd_head : hparams.n_embd / hparams.n_head; - const int n_embd_kv = hparams.n_head_kv * d_head; - - auto st = std::make_unique(); - st->n_ctx = hparams.attn_window_size + n_frames_max; - st->slot_pos.assign(st->n_ctx, -1); - - ggml_init_params params = { - /*.mem_size =*/ ggml_tensor_overhead() * 2 * n_layer, - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - st->ctx_data.reset(ggml_init(params)); - for (int il = 0; il < n_layer; il++) { - ggml_tensor * k = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F16, n_embd_kv, st->n_ctx); - ggml_tensor * v = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F16, n_embd_kv, st->n_ctx); - ggml_format_name(k, "enc_state_k_%d", il); - ggml_format_name(v, "enc_state_v_%d", il); - st->k.push_back(k); - st->v.push_back(v); - } - st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), ctx->backend)); - if (!st->buf) { - LOG_ERR("%s: failed to allocate the encoder state\n", __func__); - return nullptr; - } - // empty slots are masked out, but they must not hold NaN - ggml_backend_buffer_clear(st->buf.get(), 0); - - LOG_INF("%s: encoder state: %d slots, %.2f MiB\n", __func__, st->n_ctx, ggml_backend_buffer_get_size(st->buf.get()) / 1024.0 / 1024.0); - - return st.release(); -} - -void clip_enc_state_free(clip_enc_state * state) { - delete state; -} - -size_t clip_cbx_read_tensor(struct clip_ctx * ctx, const char * name, float * out, size_t n_max) { - const auto & tensors = ctx->model.cbx_tensors; +size_t mtmd_model::cbx_read_tensor(const char * name, float * out, size_t n_max) const { + const auto & tensors = cbx_tensors; auto it = tensors.find(name); if (it == tensors.end() || !it->second) { return 0; @@ -6014,2158 +4196,12 @@ size_t clip_cbx_read_tensor(struct clip_ctx * ctx, const char * name, float * ou return n; } -bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { - const clip_image_f32_batch & imgs = *params->imgs; - int n_batch_cur = imgs.entries.size(); - - // [QWEN_VIDEO] for video models, the batch dimension is used as temporal dimension for merged frames - if (!ctx->support_batch && n_batch_cur > clip_model_n_temporal_merge(ctx)) { - LOG_ERR("%s: batch size %d exceeds maximum supported batch/temporal-merge size %d\n", __func__, n_batch_cur, clip_model_n_temporal_merge(ctx)); - return false; - } - - // if buffers are not allocated, we need to do a warmup run to allocate them - if (!ctx->is_allocated) { - clip_model_loader::warmup(*ctx, *params->imgs); - } - - const bool needs_gen_state = params->gen_process == CLIP_GEN_PROCESS_GEN_CODE || - params->gen_process == CLIP_GEN_PROCESS_GEN_WAV || - params->gen_process == CLIP_GEN_PROCESS_TTS; - if (needs_gen_state && params->gen_state == nullptr) { - LOG_ERR("%s: audio gen requires a gen state\n", __func__); - return false; - } - if (ctx->proj_type() == PROJECTOR_TYPE_DRAMA3_GEN && params->gen_process == CLIP_GEN_PROCESS_GEN_CODE) { - float pos = 0.0f; - ggml_backend_tensor_get(params->gen_state->slots[0], &pos, 0, sizeof(float)); - if ((int) pos >= clip_graph_drama3_gen::n_pos_max) { - LOG_ERR("%s: the codebook tower holds at most %d rows\n", __func__, clip_graph_drama3_gen::n_pos_max); - return false; - } - } - - // build the inference graph - ggml_backend_sched_reset(ctx->sched.get()); - ggml_cgraph * gf = clip_get_graph_builder(ctx, imgs, params)->build(); - - // the last node is the embedding tensor, code2wav has no out_embd - // read it before scheduling: backends may reorder the nodes of graphs with several outputs (Metal) - ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr; - - if (!ggml_backend_sched_alloc_graph(ctx->sched.get(), gf)) { - LOG_ERR("%s: failed to allocate compute graph\n", __func__); - return false; - } - - // set inputs - const auto & model = ctx->model; - const auto & hparams = model.hparams; - - const int image_size_width = imgs.entries[0].nx(); - const int image_size_height = imgs.entries[0].ny(); - - const int patch_size = hparams.patch_size; - const int num_patches = ((image_size_width / patch_size) * (image_size_height / patch_size)); - const int n_pos = num_patches + (model.class_embedding ? 1 : 0); - const int pos_w = image_size_width / patch_size; - const int pos_h = image_size_height / patch_size; - - - auto get_inp_tensor = [&gf](const char * name) { - ggml_tensor * inp = ggml_graph_get_tensor(gf, name); - if (inp == nullptr) { - GGML_ABORT("Failed to get tensor %s", name); - } - if (!(inp->flags & GGML_TENSOR_FLAG_INPUT)) { - GGML_ABORT("Tensor %s is not an input tensor", name); - } - return inp; - }; - - auto set_input_f32 = [&get_inp_tensor](const char * name, const std::vector & values) { - ggml_tensor * cur = get_inp_tensor(name); - GGML_ASSERT(cur->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size()); - ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); - }; - - // mask from build_inp_attn_mask(), f16 if flash attn is enabled - auto set_input_attn_mask = [&get_inp_tensor](const std::vector & values, int idx = 0) { - ggml_tensor * cur = get_inp_tensor(get_attn_mask_name(idx).c_str()); - GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size()); - if (cur->type == GGML_TYPE_F16) { - std::vector values_f16(values.size()); - ggml_fp32_to_fp16_row(values.data(), values_f16.data(), values.size()); - ggml_backend_tensor_set(cur, values_f16.data(), 0, ggml_nbytes(cur)); - } else { - GGML_ASSERT(cur->type == GGML_TYPE_F32); - ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); - } - }; - - auto set_input_i32 = [&get_inp_tensor](const char * name, std::vector & values) { - ggml_tensor * cur = get_inp_tensor(name); - GGML_ASSERT(cur->type == GGML_TYPE_I32); - GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size()); - ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); - }; - - // state of the previous call, zero on a cold start - auto set_gen_state_in = [&]() { - const auto & slots = params->gen_state->slots; - size_t i = 0; - for (const auto & slot : list_gen_state_slots(hparams, model)) { - GGML_ASSERT(i < slots.size()); - if (slot.in_place) { - i++; - continue; - } - ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str())); - } - }; - - // lfm3a encoder and codec: causal sliding window over [cached window | this call] - auto set_lfm3a_window_inputs = [&](int base, int n_pos, int window) { - std::vector positions(n_pos); - for (int i = 0; i < n_pos; i++) { - positions[i] = base + i; - } - set_input_i32("positions", positions); - - // key j holds position base - prefix + j - const int prefix = window - 1; - const int n_kv = prefix + n_pos; - std::vector mask((size_t) n_kv * n_pos, -INFINITY); - for (int q = 0; q < n_pos; q++) { - for (int j = std::max(0, q + prefix - window + 1); j <= q + prefix; j++) { - if (base - prefix + j >= 0) { - mask[(size_t) q * n_kv + j] = 0.0f; - } - } - } - set_input_attn_mask(mask); - }; - - // rope positions and attention mask of the mimi transformers (pocket-tts). - // the mask is causal with a sliding window, see _build_attention_mask() in the reference - auto set_pockettts_tfm_inputs = [&]() { - const int64_t n_pos = ggml_nelements(get_inp_tensor("inp_pos")); - GGML_ASSERT(n_pos > 0); - std::vector positions((size_t) n_pos); - for (int64_t i = 0; i < n_pos; i++) { - positions[(size_t) i] = (int32_t) i; - } - set_input_i32("inp_pos", positions); - - // the preprocessor truncates the waveform to keep this mask bounded - const int64_t max_pos = (int64_t) clip_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate / 120; - GGML_ASSERT(n_pos <= max_pos && "pocket-tts speaker reference too long for a dense mask"); - - const int64_t context = hparams.mimi_tfm_context; - std::vector mask((size_t) n_pos * n_pos, -INFINITY); - for (int64_t q = 0; q < n_pos; q++) { - for (int64_t k = 0; k < n_pos; k++) { - const int64_t delta = q - k; - if (delta >= 0 && delta < context) { - mask[(size_t) q * n_pos + k] = 0.0f; - } - } - } - set_input_attn_mask(mask); - }; - - // set input pixel values - if (!imgs.is_audio) { - size_t nelem = 0; - for (const auto & img : imgs.entries) { - nelem += img.nx() * img.ny() * 3; - } - std::vector inp_raw(nelem); - - // layout of data (note: the channel dim is unrolled to better visualize the layout): - // - // ┌──W──┐ - // │ H │ channel = R - // ├─────┤ │ - // │ H │ channel = G - // ├─────┤ │ - // │ H │ channel = B - // └─────┘ │ - // ──────┘ x B - - // IMPORTANT: [QWEN_VIDEO] the batch dim is currently used for temporal dim in Qwen-VL models - // All entries must have the same spatial size (enforced by can_batch_with() during merging) - { - const int nx = imgs.entries[0].nx(); - const int ny = imgs.entries[0].ny(); - const int n = nx * ny; - - for (int b = 0; b < n_batch_cur; b++) { - LOG_DBG("%s: copying image %d/%d to input buffer (nx=%d, ny=%d)\n", __func__, b+1, n_batch_cur, nx, ny); - const auto & buf = imgs.entries[b].get_ro_buf(); - float * batch_entry = inp_raw.data() + b * (3*n); - for (int y = 0; y < ny; y++) { - for (int x = 0; x < nx; x++) { - size_t base_src = 3*(y * nx + x); - size_t base_dst = y * nx + x; - batch_entry[ base_dst] = buf[base_src ]; - batch_entry[1*n + base_dst] = buf[base_src + 1]; - batch_entry[2*n + base_dst] = buf[base_src + 2]; - } - } - } - } - set_input_f32("inp_raw", inp_raw); - - } else if (params->gen_process != CLIP_GEN_PROCESS_GEN_WAV && - params->gen_process != CLIP_GEN_PROCESS_TTS && - params->gen_process != CLIP_GEN_PROCESS_TTS_VOCODE && - ggml_graph_get_tensor(gf, "inp_raw") != nullptr) { - // audio input. GEN_WAV and the chatterbox TTS stages are not here: they take codes or feats, set in the switch below. - // a GEN_CODE graph may have no hidden-state input (lfm3a_gen feedback embedding) - GGML_ASSERT(imgs.entries.size() == 1); - - const auto & mel_inp = imgs.entries[0]; - const auto & buf = mel_inp.get_ro_buf(); - const int n_step = mel_inp.nx(); - const int n_mel = mel_inp.ny(); - GGML_ASSERT((size_t)n_step * n_mel == buf.size()); - - set_input_f32("inp_raw", buf); - } - - // set input per projector - switch (ctx->model.proj_type) { - case PROJECTOR_TYPE_VOXTRAL_RT: - { - clip_enc_state * st = params->enc_state; - if (!st) { - LOG_ERR("%s: this encoder needs a clip_enc_state\n", __func__); - return false; - } - const int n_window = hparams.attn_window_size; - const int n_new = (imgs.entries[0].nx() - 2) / 2; - GGML_ASSERT(n_new <= st->n_ctx - n_window); - - std::vector pos(n_new); - std::vector idx(n_new); - std::vector slot_pos = st->slot_pos; // including the frames written by this call - for (int i = 0; i < n_new; i++) { - pos[i] = st->n_past + i; - idx[i] = pos[i] % st->n_ctx; - slot_pos[idx[i]] = pos[i]; - } - set_input_i32("inp_pos", pos); - ggml_backend_tensor_set(get_inp_tensor("inp_kv_idx"), idx.data(), 0, idx.size() * sizeof(int64_t)); - - // each frame sees itself and the n_window - 1 frames before it - std::vector mask((size_t) st->n_ctx * n_new, -INFINITY); - for (int i = 0; i < n_new; i++) { - for (int s = 0; s < st->n_ctx; s++) { - const int32_t p = slot_pos[s]; - if (p >= 0 && p <= pos[i] && p > pos[i] - n_window) { - mask[(size_t) i * st->n_ctx + s] = 0.0f; - } - } - } - set_input_attn_mask(mask); - - std::vector tail = st->stem_tail; - tail.resize(hparams.n_embd, 0.0f); - set_input_f32("inp_stem_tail", tail); - } break; - case PROJECTOR_TYPE_MUSE_GLIMMER: - { - const int grid_w = pos_w; // image_size_width / patch_size - const int grid_h = pos_h; // image_size_height / patch_size - const int n_tok = grid_w * grid_h; - const int pgrid = (int) std::sqrt((double) ctx->model.position_embeddings->ne[1]); // 32 - const int f = hparams.n_merge; // downsample 2 - - // pixel patchify runs inside the graph via build_inp() (ggml_conv_2d); - // pos-emb bilinear interp via resize_position_embeddings(). - - // --- sparse window grouping (pgrid x pgrid windows) --- - const int win = pgrid; - const int nwin_h = (grid_h + win - 1) / win; - const int nwin_w = (grid_w + win - 1) / win; - std::vector sp_perm; sp_perm.reserve(n_tok); - std::vector sp_slens; - for (int wy = 0; wy < nwin_h; wy++) { - for (int wx = 0; wx < nwin_w; wx++) { - int cnt = 0; - for (int hh = 0; hh < win; hh++) { - for (int ww = 0; ww < win; ww++) { - const int gy = wy * win + hh; - const int gx = wx * win + ww; - if (gy < grid_h && gx < grid_w) { sp_perm.push_back(gy * grid_w + gx); cnt++; } - } - } - if (cnt > 0) sp_slens.push_back(cnt); - } - } - std::vector rpos_w(n_tok), rpos_h(n_tok), inv_perm(n_tok); - for (int i = 0; i < n_tok; i++) { - const int orig = sp_perm[i]; - rpos_w[i] = (orig % grid_w) + 1; // 1-indexed - rpos_h[i] = (orig / grid_w) + 1; - inv_perm[orig] = i; - } - set_input_i32("muse_glimmer_sp_perm", sp_perm); - set_input_i32("muse_glimmer_inv_perm", inv_perm); - set_input_i32("muse_glimmer_pos_w", rpos_w); - set_input_i32("muse_glimmer_pos_h", rpos_h); - - // block-diagonal window mask (permuted order) - std::vector sp_mask((size_t) n_tok * n_tok, -INFINITY); - { - int off = 0; - for (int s : sp_slens) { - for (int a = 0; a < s; a++) - for (int b = 0; b < s; b++) - sp_mask[(size_t) (off + a) * n_tok + (off + b)] = 0.0f; - off += s; - } - } - set_input_attn_mask(sp_mask); - - // pixel-shuffle gather (original order): f*f spatial neighbours grouped - std::vector dsp; dsp.reserve(n_tok); - for (int oy = 0; oy < grid_h / f; oy++) - for (int ox = 0; ox < grid_w / f; ox++) - for (int ry = 0; ry < f; ry++) - for (int rx = 0; rx < f; rx++) - dsp.push_back((oy * f + ry) * grid_w + (ox * f + rx)); - set_input_i32("muse_glimmer_ds_perm", dsp); - } break; - case PROJECTOR_TYPE_MINICPMV: - { - // inspired from siglip: - // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit - // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit/blob/d66538faeba44480d0bfaa42145eef26f9423199/modeling_siglip.py#L316 - std::vector positions(pos_h * pos_w); - // note: sized by the actual patch counts; a tall/wide image produces more - // than 1024 patches per side and a fixed [1024] array would be overrun - std::vector bucket_coords_h(pos_h); - std::vector bucket_coords_w(pos_w); - for (int i = 0; i < pos_h; i++){ - bucket_coords_h[i] = std::floor(70.0*i/pos_h); - } - for (int i = 0; i < pos_w; i++){ - bucket_coords_w[i] = std::floor(70.0*i/pos_w); - } - for (int i = 0, id = 0; i < pos_h; i++){ - for (int j = 0; j < pos_w; j++){ - positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j]; - } - } - set_input_i32("positions", positions); - - // inputs for resampler projector - // set the 2D positions (using float for sinusoidal embedding) - int n_patches_per_col = image_size_width / patch_size; - std::vector pos_data(n_pos); - // dimension H - for (int i = 0; i < n_pos; i++) { - pos_data[i] = static_cast(i / n_patches_per_col); - } - set_input_f32("pos_h", pos_data); - // dimension W - for (int i = 0; i < n_pos; i++) { - pos_data[i] = static_cast(i % n_patches_per_col); - } - set_input_f32("pos_w", pos_data); - // base frequency omega - const float base_freq = 10000.0f; - const int n_embd_proj = clip_n_mmproj_embd(ctx); - std::vector omega(n_embd_proj / 4); - for (int i = 0; i < n_embd_proj / 4; ++i) { - omega[i] = 1.0f / std::pow(base_freq, static_cast(i) / (n_embd_proj / 4)); - } - set_input_f32("omega", omega); - } break; - case PROJECTOR_TYPE_MINICPMV4_6: - case PROJECTOR_TYPE_MINICPMV4_7: - case PROJECTOR_TYPE_MINICPM_OCR_1_0: - { - const bool is_4x = hparams.n_merge == 2; - - // SigLIP position buckets (same as resampler path) - std::vector positions(pos_h * pos_w); - std::vector bucket_coords_h(pos_h); - std::vector bucket_coords_w(pos_w); - for (int i = 0; i < pos_h; i++){ - bucket_coords_h[i] = std::floor(70.0*i/pos_h); - } - for (int i = 0; i < pos_w; i++){ - bucket_coords_w[i] = std::floor(70.0*i/pos_w); - } - for (int i = 0, id = 0; i < pos_h; i++){ - for (int j = 0; j < pos_w; j++){ - positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j]; - } - } - set_input_i32("positions", positions); - - const int half_h = pos_h / 2; - const int half_w = pos_w / 2; - - auto make_ds_idx = [](int off_r, int off_c, int ds_h, int ds_w, int stride_w) { - std::vector idx(ds_h * ds_w); - for (int i = 0; i < ds_h; i++) { - for (int j = 0; j < ds_w; j++) { - idx[i * ds_w + j] = (2*i + off_r) * stride_w + (2*j + off_c); - } - } - return idx; - }; - - if (!is_4x) { - // window reorder indices for 2x2 windows - std::vector window_idx(n_pos); - std::vector inv_window_idx(n_pos); - { - int k = 0; - for (int wi = 0; wi < half_h; wi++) { - for (int wj = 0; wj < half_w; wj++) { - window_idx[k++] = (2*wi ) * pos_w + (2*wj ); - window_idx[k++] = (2*wi ) * pos_w + (2*wj + 1); - window_idx[k++] = (2*wi + 1) * pos_w + (2*wj ); - window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1); - } - } - for (int i = 0; i < n_pos; i++) { - inv_window_idx[window_idx[i]] = i; - } - } - set_input_i32("vit_merger_window_idx", window_idx); - set_input_i32("vit_merger_inv_window_idx", inv_window_idx); - - // block-diagonal attention mask: tokens in the same 4-token - // window attend to each other (mask = 0), all other positions - // are masked out (-inf). matches the window-major reorder above. - std::vector window_mask_data(n_pos * n_pos, std::numeric_limits::lowest()); - for (int wi = 0; wi < n_pos / 4; wi++) { - for (int i = 0; i < 4; i++) { - for (int j = 0; j < 4; j++) { - window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f; - } - } - } - set_input_attn_mask(window_mask_data); - - // ViT merger 2x2 downsample indices - auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w); - auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w); - auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w); - auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w); - set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0); - set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1); - set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2); - set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3); - } - - const int merger_h = is_4x ? pos_h : half_h; - const int merger_w = is_4x ? pos_w : half_w; - auto m_ds_0 = make_ds_idx(0, 0, merger_h / 2, merger_w / 2, merger_w); - auto m_ds_1 = make_ds_idx(0, 1, merger_h / 2, merger_w / 2, merger_w); - auto m_ds_2 = make_ds_idx(1, 0, merger_h / 2, merger_w / 2, merger_w); - auto m_ds_3 = make_ds_idx(1, 1, merger_h / 2, merger_w / 2, merger_w); - set_input_i32("merger_ds_idx_0", m_ds_0); - set_input_i32("merger_ds_idx_1", m_ds_1); - set_input_i32("merger_ds_idx_2", m_ds_2); - set_input_i32("merger_ds_idx_3", m_ds_3); - } break; - case PROJECTOR_TYPE_LFM3VL: - { - // patches in raster-scan order (no interleave before encoder layers) - const int pw = image_size_width / patch_size; - const int ph = image_size_height / patch_size; - std::vector positions(n_pos * 4); - int ptr = 0; - for (int y = 0; y < ph; y++) { - for (int x = 0; x < pw; x++) { - positions[ ptr] = y; - positions[ num_patches + ptr] = x; - positions[2 * num_patches + ptr] = y; - positions[3 * num_patches + ptr] = x; - ptr++; - } - } - set_input_i32("positions", positions); - } break; - case PROJECTOR_TYPE_MAGEVL: - { - // t, h, w in 2x2 block order, t is always 0 for an image - const int merge_ratio = hparams.n_merge; - const int pw = image_size_width / patch_size / merge_ratio; - const int ph = image_size_height / patch_size / merge_ratio; - std::vector positions(n_pos * 4, 0); - int ptr = 0; - for (int ty = 0; ty < ph; ty++) { - for (int tx = 0; tx < pw; tx++) { - for (int dy = 0; dy < merge_ratio; dy++) { - for (int dx = 0; dx < merge_ratio; dx++) { - positions[ num_patches + ptr] = ty * merge_ratio + dy; - positions[2 * num_patches + ptr] = tx * merge_ratio + dx; - ptr++; - } - } - } - } - set_input_i32("positions", positions); - } break; - case PROJECTOR_TYPE_QWEN2VL: - case PROJECTOR_TYPE_QWEN3VL: - case PROJECTOR_TYPE_LING3VL: - case PROJECTOR_TYPE_GLM4V: - case PROJECTOR_TYPE_GLM5V: - { - const int merge_ratio = hparams.n_merge; - const int pw = image_size_width / patch_size / merge_ratio; - const int ph = image_size_height / patch_size / merge_ratio; - std::vector positions(n_pos * 4); - int ptr = 0; - for (int ty = 0; ty < ph; ty++) { - for (int tx = 0; tx < pw; tx++) { - for (int dy = 0; dy < merge_ratio; dy++) { - for (int dx = 0; dx < merge_ratio; dx++) { - const int y = ty * merge_ratio + dy; - const int x = tx * merge_ratio + dx; - positions[ ptr] = y; - positions[ num_patches + ptr] = x; - positions[2 * num_patches + ptr] = y; - positions[3 * num_patches + ptr] = x; - ptr++; - } - } - } - } - - set_input_i32("positions", positions); - } break; - case PROJECTOR_TYPE_STEP3VL: - { - std::vector pos_data(n_pos); - for (int i = 0; i < n_pos; i++) { - pos_data[i] = i / pos_w; - } - set_input_i32("pos_h", pos_data); - for (int i = 0; i < n_pos; i++) { - pos_data[i] = i % pos_w; - } - set_input_i32("pos_w", pos_data); - } break; - case PROJECTOR_TYPE_PADDLEOCR: - { - const int merge_ratio = hparams.n_merge; - const int pw = image_size_width / patch_size / merge_ratio; - const int ph = image_size_height / patch_size / merge_ratio; - std::vector positions(n_pos * 4); - int ptr = 0; - // NOTE: same as Qwen-VL, but x and y are swapped - for (int ty = 0; ty < ph; ty++) { - for (int dy = 0; dy < merge_ratio; dy++) { - for (int tx = 0; tx < pw; tx++) { - for (int dx = 0; dx < merge_ratio; dx++) { - const int y = ty * merge_ratio + dy; - const int x = tx * merge_ratio + dx; - positions[ ptr] = y; - positions[ num_patches + ptr] = x; - positions[2 * num_patches + ptr] = y; - positions[3 * num_patches + ptr] = x; - ptr++; - } - } - } - } - - set_input_i32("positions", positions); - } break; - case PROJECTOR_TYPE_MINIMAX_M3: - { - const int n_merge = hparams.n_merge; - const int gh = image_size_height / patch_size; - const int gw = image_size_width / patch_size; - std::vector pos_h, pos_w; - pos_h.reserve(gh * gw); - pos_w.reserve(gh * gw); - for (int bh = 0; bh < gh / n_merge; bh++) - for (int bw = 0; bw < gw / n_merge; bw++) - for (int mh = 0; mh < n_merge; mh++) - for (int mw = 0; mw < n_merge; mw++) { - pos_h.push_back(bh * n_merge + mh); - pos_w.push_back(bw * n_merge + mw); - } - set_input_i32("minimax_pos_h", pos_h); - set_input_i32("minimax_pos_w", pos_w); - } break; - case PROJECTOR_TYPE_DOTS_OCR: - case PROJECTOR_TYPE_DOTS3NOTE_V: - { - const int pw = image_size_width / patch_size; - const int ph = image_size_height / patch_size; - const int n_pos = ph * pw; - std::vector positions(n_pos * 4); - int ptr = 0; - - // flat layout: [h, w, h, w] for each patch - // patches are in raster order (matching conv2d output) - for (int y = 0; y < ph; y++) { - for (int x = 0; x < pw; x++) { - positions[ ptr] = y; - positions[ n_pos + ptr] = x; - positions[2*n_pos + ptr] = y; - positions[3*n_pos + ptr] = x; - ptr++; - } - } - - set_input_i32("positions", positions); - } break; - case PROJECTOR_TYPE_QWEN25VL: - case PROJECTOR_TYPE_EXAONE4_5: - case PROJECTOR_TYPE_YOUTUVL: - { - // pw * ph = number of tokens output by ViT after apply patch merger - // ipw * ipw = number of vision token been processed inside ViT - const bool use_window_attn = - (ctx->model.proj_type == PROJECTOR_TYPE_QWEN25VL || ctx->model.proj_type == PROJECTOR_TYPE_EXAONE4_5) - ? hparams.n_wa_pattern > 0 - : !hparams.wa_layer_indexes.empty(); - const int merge_ratio = 2; - const int pw = image_size_width / patch_size / merge_ratio; - const int ph = image_size_height / patch_size / merge_ratio; - const int ipw = image_size_width / patch_size; - const int iph = image_size_height / patch_size; - - std::vector idx (ph * pw); - std::vector inv_idx(ph * pw); - - if (use_window_attn) { - const int attn_window_size = hparams.attn_window_size > 0 ? hparams.attn_window_size : 112; - const int grid_window = attn_window_size / patch_size / merge_ratio; - int dst = 0; - // [num_vision_tokens, num_vision_tokens] attention mask tensor - std::vector mask(pow(ipw * iph, 2), std::numeric_limits::lowest()); - int mask_row = 0; - - for (int y = 0; y < ph; y += grid_window) { - for (int x = 0; x < pw; x += grid_window) { - const int win_h = std::min(grid_window, ph - y); - const int win_w = std::min(grid_window, pw - x); - const int dst_0 = dst; - // group all tokens belong to the same window togather (to a continue range) - for (int dy = 0; dy < win_h; dy++) { - for (int dx = 0; dx < win_w; dx++) { - const int src = (y + dy) * pw + (x + dx); - GGML_ASSERT(src < (int)idx.size()); - GGML_ASSERT(dst < (int)inv_idx.size()); - idx [src] = dst; - inv_idx[dst] = src; - dst++; - } - } - - for (int r=0; r < win_h * win_w * merge_ratio * merge_ratio; r++) { - int row_offset = mask_row * (ipw * iph); - std::fill( - mask.begin() + row_offset + (dst_0 * merge_ratio * merge_ratio), - mask.begin() + row_offset + (dst * merge_ratio * merge_ratio), - 0.0); - mask_row++; - } - } - } - - set_input_i32("window_idx", idx); - set_input_i32("inv_window_idx", inv_idx); - set_input_attn_mask(mask); - } else { - for (int i = 0; i < ph * pw; i++) { - idx[i] = i; - } - } - - const int mpow = merge_ratio * merge_ratio; - std::vector positions(n_pos * 4); - - int ptr = 0; - for (int ty = 0; ty < ph; ty++) { - for (int tx = 0; tx < pw; tx++) { - for (int dy = 0; dy < merge_ratio; dy++) { - for (int dx = 0; dx < merge_ratio; dx++) { - auto remap = idx[ptr / mpow]; - remap = (remap * mpow) + (ptr % mpow); - - const int y = ty * merge_ratio + dy; - const int x = tx * merge_ratio + dx; - positions[ remap] = y; - positions[ num_patches + remap] = x; - positions[2 * num_patches + remap] = y; - positions[3 * num_patches + remap] = x; - ptr++; - } - } - } - } - - set_input_i32("positions", positions); - } break; - case PROJECTOR_TYPE_MIMOVL: - { - const int merge = hparams.n_merge; // 2 - const int merge_unit = merge * merge; // 4 - const int patch = hparams.patch_size; // 16 - const int H = image_size_height / patch; - const int W = image_size_width / patch; - const int n_pos_full = H * W; - const int llm_h = H / merge; - const int llm_w = W / merge; - const int n_units = llm_h * llm_w; // n_pos / merge_unit - - // Row-major merge-tile-ordered (h, w) positions - std::vector pos_h_row(n_pos_full); - std::vector pos_w_row(n_pos_full); - { - int idx = 0; - for (int ty = 0; ty < llm_h; ty++) { - for (int tx = 0; tx < llm_w; tx++) { - for (int dy = 0; dy < merge; dy++) { - for (int dx = 0; dx < merge; dx++) { - pos_h_row[idx] = ty * merge + dy; - pos_w_row[idx] = tx * merge + dx; - idx++; - } - } - } - } - } - - // Col-major merge-unit permutation - std::vector idx_col(n_units); - for (int r = 0; r < llm_h; r++) { - for (int c = 0; c < llm_w; c++) { - int u_row = r * llm_w + c; - int u_col = c * llm_h + r; - idx_col[u_col] = (float) u_row; - } - } - - // Col-mode positions: permute pos_*_row by idx_col - std::vector pos_h_col(n_pos_full); - std::vector pos_w_col(n_pos_full); - for (int u = 0; u < n_units; u++) { - int src = (int) idx_col[u]; - for (int k = 0; k < merge_unit; k++) { - pos_h_col[u * merge_unit + k] = pos_h_row[src * merge_unit + k]; - pos_w_col[u * merge_unit + k] = pos_w_row[src * merge_unit + k]; - } - } - - // Pack into ggml_rope_multi VISION-mode layout. The non-CPU kernels - // only read slots 0 and 1, so pack h in slot 0, w in slot 1: - // positions[0..n_pos) = h - // positions[n_pos..2*n_pos) = w - // positions[2*n_pos..3*n_pos) = 0 - // positions[3*n_pos..4*n_pos) = 0 - std::vector positions_row(static_cast(n_pos_full) * 4, 0); - std::vector positions_col(static_cast(n_pos_full) * 4, 0); - for (int i = 0; i < n_pos_full; i++) { - positions_row[0 * n_pos_full + i] = pos_h_row[i]; - positions_row[1 * n_pos_full + i] = pos_w_row[i]; - positions_col[0 * n_pos_full + i] = pos_h_col[i]; - positions_col[1 * n_pos_full + i] = pos_w_col[i]; - } - - // Banded 1D sliding-window mask - const int window = hparams.attn_window_size; - GGML_ASSERT(window > 0); - std::vector mask(static_cast(n_pos_full) * n_pos_full, std::numeric_limits::lowest()); - for (int q = 0; q < n_pos_full; q++) { - int lo = std::max(0, q - window); - int hi = std::min(n_pos_full - 1, q + window); - for (int k = lo; k <= hi; k++) { - mask[static_cast(q) * n_pos_full + k] = 0.0f; - } - } - - set_input_i32("mimovl_positions_row", positions_row); - set_input_i32("mimovl_positions_col", positions_col); - set_input_f32("mimovl_idx_col", idx_col); - set_input_attn_mask(mask); - } break; - case PROJECTOR_TYPE_PIXTRAL: - case PROJECTOR_TYPE_KIMIVL: - case PROJECTOR_TYPE_KIMIK25: - case PROJECTOR_TYPE_LIGHTONOCR: - { - // set the 2D positions - int n_patches_per_col = image_size_width / patch_size; - std::vector pos_data(n_pos); - // dimension H - for (int i = 0; i < n_pos; i++) { - pos_data[i] = i / n_patches_per_col; - } - set_input_i32("pos_h", pos_data); - // dimension W - for (int i = 0; i < n_pos; i++) { - pos_data[i] = i % n_patches_per_col; - } - set_input_i32("pos_w", pos_data); - } break; - case PROJECTOR_TYPE_SENSENOVA_U1: - { - const int n_patches_per_row = image_size_width / patch_size; - std::vector pos_data(n_pos); - for (int i = 0; i < n_pos; i++) { - pos_data[i] = i / n_patches_per_row; - } - set_input_i32("pos_y", pos_data); - for (int i = 0; i < n_pos; i++) { - pos_data[i] = i % n_patches_per_row; - } - set_input_i32("pos_x", pos_data); - } break; - case PROJECTOR_TYPE_DEEPSEEK4V: - { - // set the 2D positions (mrope layout, only the first 2 channels are used) - int n_patches_per_row = image_size_width / patch_size; - std::vector positions(n_pos * 4, 0); - for (int i = 0; i < n_pos; i++) { - positions[i] = i / n_patches_per_row; // row - positions[n_pos + i] = i % n_patches_per_row; // col - } - set_input_i32("positions", positions); - - // token block layout index (see clip_graph_deepseek4v::build) - // rows [0, n_grid) are the aligner output, the sentinels follow - const int n_merge = hparams.n_merge; - const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge; - const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge; - const int n_grid = n_llm_w * n_llm_h; - const int idx_start = n_grid; - const int idx_end = n_grid + 1; - const int idx_newline = n_grid + 2; - const int idx_pad = n_grid + 3; - - const int lead_pad = imgs.entries[0].lead_pad; - const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad); - - std::vector idx; - idx.reserve(bl.n_out); - for (int i = 0; i < lead_pad; i++) { - idx.push_back(idx_pad); - } - idx.push_back(idx_start); - // pairs of adjacent rows are interleaved column-wise ("N-layout") - // ref: build_image_block in inference/image_processor.py - for (int t = 0; t < bl.rows * bl.row_len; t++) { - const int g = t / (2 * bl.row_len); - const int rem = t % (2 * bl.row_len); - const int c = rem / 2; // column - const int r = 2 * g + rem % 2; // row - if (r >= n_llm_h) { - idx.push_back(idx_pad); - } else if (c == n_llm_w) { - idx.push_back(idx_newline); - } else { - idx.push_back(r * n_llm_w + c); - } - } - for (int i = 0; i < bl.pad_last; i++) { - idx.push_back(idx_pad); - } - idx.push_back(idx_end); - set_input_i32("layout_idx", idx); - } break; - case PROJECTOR_TYPE_DEEPSEEK41V: - { - // mrope layout, only the first 2 channels are used - const int n_patches_per_row = image_size_width / patch_size; - std::vector positions(n_pos * 4, 0); - for (int i = 0; i < n_pos; i++) { - positions[i] = i / n_patches_per_row; - positions[n_pos + i] = i % n_patches_per_row; - } - set_input_i32("positions", positions); - - // reading-order block: rows [0, n_grid) are the aligner output, then start / end / newline - const int n_merge = hparams.n_merge; - const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge; - const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge; - const int n_grid = n_llm_w * n_llm_h; - std::vector idx; - idx.reserve(dsv41_n_block_tokens(n_llm_w, n_llm_h)); - idx.push_back(n_grid); // IMAGE_START - for (int r = 0; r < n_llm_h; r++) { - for (int c = 0; c < n_llm_w; c++) { - idx.push_back(r * n_llm_w + c); - } - idx.push_back(n_grid + 2); // IMAGE_NEW_LINE - } - idx.push_back(n_grid + 1); // IMAGE_END - set_input_i32("layout_idx", idx); - } break; - case PROJECTOR_TYPE_GLM_EDGE: - { - // llava and other models - std::vector positions(n_pos); - for (int i = 0; i < n_pos; i++) { - positions[i] = i; - } - set_input_i32("positions", positions); - } break; - case PROJECTOR_TYPE_MLP: - case PROJECTOR_TYPE_MLP_NORM: +int mtmd_model::n_mmproj_embd() const { + switch (proj_type) { case PROJECTOR_TYPE_LDP: + return mm_model_block_1_block_2_1_b->ne[0]; case PROJECTOR_TYPE_LDPV2: - { - // llava and other models - std::vector positions(n_pos); - for (int i = 0; i < n_pos; i++) { - positions[i] = i; - } - set_input_i32("positions", positions); - - // The patches vector is used to get rows to index into the embeds with; - // we should skip dim 0 only if we have CLS to avoid going out of bounds - // when retrieving the rows. - int patch_offset = model.class_embedding ? 1 : 0; - std::vector patches(num_patches); - for (int i = 0; i < num_patches; i++) { - patches[i] = i + patch_offset; - } - set_input_i32("patches", patches); - } break; - case PROJECTOR_TYPE_POCKETTTS_SPKENC: - { - set_pockettts_tfm_inputs(); - } break; - case PROJECTOR_TYPE_POCKETTTS_GEN: - { - if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) { - GGML_ASSERT(params->feats != nullptr); - set_input_f32("inp_feats", *params->feats); - // positions and mask are derived in-graph from the persisted counter - set_gen_state_in(); - } else { - // flow matching starts from gaussian noise, std = sqrt(temp) - ggml_tensor * t = get_inp_tensor("inp_noise"); - // Config.default_temperature, for a caller that does not set one - const float temp = params->temp > 0.0f ? params->temp : 0.7f; - std::normal_distribution dist(0.0f, std::sqrt(temp)); - std::vector noise(ggml_nelements(t)); - for (auto & v : noise) { - v = dist(params->gen_state->rng); - } - set_input_f32("inp_noise", noise); - } - } break; - case PROJECTOR_TYPE_SOPRANO: - { - set_input_f32("inp_feats", *params->feats); - } break; - case PROJECTOR_TYPE_GEMMA4V: - case PROJECTOR_TYPE_GEMMA4UV: - { - // set (col, row) patch positions for learned positional embedding - const int n_cols = image_size_width / patch_size; - std::vector pos_x(num_patches), pos_y(num_patches); - for (int i = 0; i < num_patches; i++) { - pos_x[i] = i % n_cols; - pos_y[i] = i / n_cols; - } - set_input_i32("pos_x", pos_x); - set_input_i32("pos_y", pos_y); - } break; - case PROJECTOR_TYPE_DEEPSEEKOCR: - case PROJECTOR_TYPE_DEEPSEEKOCR2: - { - GGML_ASSERT( - (pos_w == pos_h) // overview image - || (pos_h >= pos_w && pos_h % pos_w == 0) // tile images - ); - - const int window = hparams.attn_window_size; - const int pos = pos_w; - std::vector rel_pos_indices_local(window * window); - std::vector rel_pos_indices_global(pos * pos); - - for (int q = 0; q < window; q++) { - for (int k = 0; k < window; k++) { - rel_pos_indices_local[q * window + k] = q - k + window - 1; - } - } - - for (int q = 0; q < pos; q++) { - for (int k = 0; k < pos; k++) { - rel_pos_indices_global[q * pos + k] = q - k + pos - 1; - } - } - - set_input_i32("rel_pos_indices_local", rel_pos_indices_local); - set_input_i32("rel_pos_indices_global", rel_pos_indices_global); - - if (ctx->proj_type() == PROJECTOR_TYPE_DEEPSEEKOCR2) { - - // qwen2 encoder attention mask - - // num_image_tokens = num_patches / 16 - // 256 for 1024 global view - // 144 for 768 tile views - const int num_image_tokens = num_patches / 16; - const int seq_len = num_image_tokens * 2; - std::vector qwen2_mask(static_cast(seq_len) * seq_len, 0.0f); - - // attention mask layout - // +--------------+---------------+ - // | all 0 | all -inf | - // +--------------+---------------+ - // | all 0 | lower tri 0 | - // +--------------+---------------+ - for (int i = 0; i < seq_len; i++) { - for (int j = 0; j < seq_len; j++) { - const bool zero = i < num_image_tokens ? - j < num_image_tokens : - j < num_image_tokens || j <= i; - qwen2_mask[static_cast(i) * seq_len + j] = zero ? 0.0f : -1e9f; - } - } - set_input_attn_mask(qwen2_mask); - } - } break; - case PROJECTOR_TYPE_CHATTERBOX: - { - if (params->gen_process == CLIP_GEN_PROCESS_TTS_VOCODE) { - set_input_f32("inp_mel", *params->mel_in); - set_input_f32("inp_sstft", *params->sstft_in); - break; - } - if (params->gen_process == CLIP_GEN_PROCESS_TOKENIZE) { - const int T1 = (imgs.entries[0].nx() - 1) / 2 + 1; - const int T2 = (T1 - 1) / 2 + 1; - std::vector pos(T2); - for (int i = 0; i < T2; i++) { - pos[(size_t) i] = i; - } - set_input_i32("inp_pos", pos); - break; - } - if (params->gen_process != CLIP_GEN_PROCESS_TTS) { - break; - } - const int n_gen = (int) params->codes->size(); - GGML_ASSERT(params->ref); - const int n_prompt = (int) params->ref->tokens.size(); - std::vector tokens = params->ref->tokens; - tokens.resize((size_t) n_prompt + n_gen); - memcpy(tokens.data() + n_prompt, params->codes->data(), (size_t) n_gen * sizeof(int32_t)); - const int64_t n_codes = model.cbx.input_embedding_w->ne[1]; - for (int32_t t : tokens) { - if (t < 0 || t >= n_codes) { - LOG_ERR("%s: speech token out of range (%d, codebook size %lld)\n", __func__, t, (long long) n_codes); - return false; - } - } - const int T1 = n_prompt + n_gen; - const int T2 = 2 * T1; - set_input_i32("inp_tokens", tokens); - - // mel-rate reference conditioning of the voice - set_input_f32("inp_prompt_feat", params->ref->feat); - set_input_f32("inp_xvec", params->ref->spk); - - // espnet relative positional encoding, entry k holds the sinusoid of relative position (T-1) - k - auto fill_pos = [&](const char * name, int T) { - const int d = 512; - std::vector pos((size_t) (2 * T - 1) * d); - for (int k = 0; k < 2 * T - 1; k++) { - const double rel = (double) (T - 1 - k); - for (int i = 0; i < d / 2; i++) { - const double div = exp(-(double) (2 * i) * log(10000.0) / d); - pos[(size_t) k * d + 2 * i ] = (float) sin(rel * div); - pos[(size_t) k * d + 2 * i + 1] = (float) cos(rel * div); - } - } - set_input_f32(name, pos); - }; - fill_pos("inp_pos1", T1); - fill_pos("inp_pos2", T2); - - // noise and the time embeddings of the solver span points, matcha layout (sines then cosines, scale 1000) - std::vector noise((size_t) 80 * T2); - std::normal_distribution nd(0.0f, 1.0f); - for (auto & f : noise) f = nd(params->gen_state->rng); - set_input_f32("inp_noise", noise); - - const bool meanflow = model.cbx.time_embed_mixer_w != nullptr; - const int n_steps = meanflow ? 2 : 10; - std::vector temb((size_t) 320 * (n_steps + 1)); - for (int s = 0; s <= n_steps; s++) { - const double u = (double) s / n_steps; - const double t = meanflow ? u : 1.0 - cos(u * M_PI / 2.0); - for (int i = 0; i < 160; i++) { - const double div = exp(-(double) i * log(10000.0) / 159.0); - temb[(size_t) s * 320 + i ] = (float) sin(1000.0 * t * div); - temb[(size_t) s * 320 + i + 160] = (float) cos(1000.0 * t * div); - } - } - set_input_f32("inp_temb", temb); - } break; - case PROJECTOR_TYPE_GEMMA3: - case PROJECTOR_TYPE_GEMMA3NV: - case PROJECTOR_TYPE_IDEFICS3: - case PROJECTOR_TYPE_COHERE2V: - case PROJECTOR_TYPE_INTERNVL: - case PROJECTOR_TYPE_NEMOTRON_V2_VL: - case PROJECTOR_TYPE_NEMOTRON_OMNI: - case PROJECTOR_TYPE_QWEN2A: - case PROJECTOR_TYPE_QWEN3A: - case PROJECTOR_TYPE_GLMA: - case PROJECTOR_TYPE_ULTRAVOX: - case PROJECTOR_TYPE_LFM2: - case PROJECTOR_TYPE_D1OMNI_V: - case PROJECTOR_TYPE_VOXTRAL: - case PROJECTOR_TYPE_MERALION: - case PROJECTOR_TYPE_MUSIC_FLAMINGO: - case PROJECTOR_TYPE_JANUS_PRO: - case PROJECTOR_TYPE_PHI4: - case PROJECTOR_TYPE_COGVLM: - case PROJECTOR_TYPE_YASA2: - case PROJECTOR_TYPE_GEMMA4UA: - case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: - case PROJECTOR_TYPE_QWEN3TTS_SPKENC: - { - // do nothing - } break; - case PROJECTOR_TYPE_CHATTERBOX_SPKENC: - { - // the ceil-mode seg pooling divides by the full segment length, rescale the last partial segment - const int T1 = (imgs.entries[0].nx() - 1) / 2 + 1; - const int S = (T1 + 99) / 100; - std::vector segfix((size_t) S, 1.0f); - const int last = T1 - (S - 1) * 100; - segfix[(size_t) S - 1] = 100.0f / (float) last; - set_input_f32("inp_segfix", segfix); - } break; - case PROJECTOR_TYPE_LFM3A_GEN: - { - const int64_t n_cb = model.gen_code_embd_w->ne[2]; - const int64_t n_vocab = model.gen_code_embd_w->ne[1]; - if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) { - GGML_ASSERT(params->codes != nullptr); - // codes index the FSQ table via ggml_get_rows, the end-of-audio code has no waveform - const int64_t n_codes = model.c2w.quant_first_cb_w->ne[1]; - for (int32_t c : *params->codes) { - if (c < 0 || c >= n_codes) { - LOG_ERR("%s: code out of range (%d, codebook size %lld)\n", __func__, c, (long long) n_codes); - return false; - } - } - std::vector codes = *params->codes; - set_input_i32("inp_codes", codes); - set_gen_state_in(); - // the "pos" slot leads the state - float base = 0.0f; - ggml_backend_tensor_get(params->gen_state->slots[0], &base, 0, sizeof(float)); - set_lfm3a_window_inputs((int) base, (int) ggml_nelements(get_inp_tensor("positions")), hparams.wav_tfm_swa); - } else { - // rows of the merged per-codebook tables; a masked codebook (-1) takes row 0, the mask drops it - std::vector codes(n_cb, -1); - if (params->codes) { - GGML_ASSERT(params->codes->size() == (size_t) n_cb); - codes = *params->codes; - } - std::vector mask(n_cb, 0.0f); - for (int64_t i = 0; i < n_cb; i++) { - if (codes[i] >= n_vocab) { - LOG_ERR("%s: code out of range (%d, vocab %lld)\n", __func__, codes[i], (long long) n_vocab); - return false; - } - if (codes[i] < 0) { - codes[i] = 0; - } else { - mask[i] = 1.0f; - } - codes[i] += (int32_t) (i * n_vocab); - } - set_input_i32("inp_codes", codes); - if (ggml_graph_get_tensor(gf, "inp_mask") != nullptr) { - set_input_f32("inp_mask", mask); - } - } - } break; - case PROJECTOR_TYPE_DRAMA3_GEN: - { - const int64_t n_cb = model.gen_code_embd_w->ne[2]; - const int64_t n_vocab = model.gen_code_embd_w->ne[1]; - if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) { - // frame-major to group-major, clamp: codec residual codebooks are smaller than the tower vocab - GGML_ASSERT(params->codes != nullptr && params->codes->size() % n_cb == 0); - const int64_t n_frames = (int64_t) params->codes->size() / n_cb; - const int64_t n_first = model.c2w.quant_first_cb_w->ne[1]; - const int64_t n_rest = model.c2w.quant_rest_cb_w->ne[1]; - std::vector codes(params->codes->size()); - for (int64_t f = 0; f < n_frames; f++) { - for (int64_t g = 0; g < n_cb; g++) { - const int64_t n = g == 0 ? n_first : n_rest; - codes[g * n_frames + f] = std::clamp((*params->codes)[f * n_cb + g], 0, (int32_t) n - 1); - } - } - set_input_i32("inp_codes", codes); - set_gen_state_in(); - } else { - // code0 < 0: next row is text (prompt), the tower only fills its KV - // codes: input frame codes for a speech row, empty for a text row - const bool next_speech = params->code0 >= 0; - const bool cur_speech = params->codes && (int64_t) params->codes->size() == n_cb; - if (params->code0 >= n_vocab) { - LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) n_vocab); - return false; - } - std::vector code0 = { std::max(params->code0, 0) }; - set_input_i32("inp_code0", code0); - - std::vector extra(n_cb - 2); - for (int64_t g = 2; g < n_cb; g++) { - const int32_t c = cur_speech ? std::clamp((*params->codes)[g], 0, (int32_t) n_vocab - 1) : 0; - extra[g - 2] = c + (int32_t) (g * n_vocab); - } - set_input_i32("inp_extra", extra); - set_input_f32("inp_scale", { next_speech ? 1.0f : 0.0f, cur_speech ? 1.0f : 0.0f }); - - float pos = 0.0f; - ggml_backend_tensor_get(params->gen_state->slots[0], &pos, 0, sizeof(float)); - std::vector inp_pos = { (int32_t) pos }; - set_input_i32("inp_pos", inp_pos); - - std::uniform_real_distribution dist(0.0f, 1.0f); - std::vector r(model.gen_code_head_w->ne[2]); - for (auto & v : r) { - v = dist(params->gen_state->rng); - } - set_input_f32("inp_rand", r); - } - } break; - case PROJECTOR_TYPE_QWEN3TTS_GEN: - { - if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) { - GGML_ASSERT(params->codes != nullptr); - - // frame-major input to group-major, rear-padded with code 0 up to one window - const int64_t n_codes = model.gen_code_head_w->ne[2] + 1; - const int64_t n_frames_w = hparams.wav_tfm_swa; - const int64_t n_frames = (int64_t) params->codes->size() / n_codes; - GGML_ASSERT(n_frames > 0 && n_frames <= n_frames_w); - - // codes are used as ggml_get_rows indices, so check them against the codebook vocab - const int64_t vocab_first = model.c2w.quant_first_cb_w->ne[1]; - const int64_t vocab_rest = model.c2w.quant_rest_cb_w->ne[1]; - for (int64_t f = 0; f < n_frames; f++) { - for (int64_t g = 0; g < n_codes; g++) { - const int32_t c = (*params->codes)[f * n_codes + g]; - const int64_t vocab = (g == 0) ? vocab_first : vocab_rest; - if (c < 0 || (int64_t) c >= vocab) { - LOG_ERR("%s: code out of range (frame %lld, group %lld, code %d, vocab %lld)\n", - __func__, (long long) f, (long long) g, c, (long long) vocab); - return false; - } - } - } - - std::vector codes(n_frames_w * n_codes, 0); - for (int64_t f = 0; f < n_frames; f++) { - for (int64_t g = 0; g < n_codes; g++) { - codes[g * n_frames_w + f] = (*params->codes)[f * n_codes + g]; - } - } - set_input_i32("inp_codes", codes); - set_gen_state_in(); - } else { - // code0 indexes gen_code_out_embd_w via ggml_get_rows; bound it - const int64_t vocab0 = model.gen_code_out_embd_w->ne[1]; - if (params->code0 < 0 || (int64_t) params->code0 >= vocab0) { - LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) vocab0); - return false; - } - std::vector code0 = { params->code0 }; - set_input_i32("inp_code0", code0); - - // one uniform(0,1) draw per codebook, used by do_sampling() - std::uniform_real_distribution dist(0.0f, 1.0f); - const int64_t n_acoustic = model.gen_code_head_w->ne[2]; - for (int64_t g = 0; g < n_acoustic; g++) { - std::vector r = { dist(params->gen_state->rng) }; - set_input_f32(("inp_rand_" + std::to_string(g)).c_str(), r); - } - } - } break; - case PROJECTOR_TYPE_HUNYUANVL: - { - // Compute the HunyuanVL 2D position embedding on CPU (with the - // custom sf=(target+0.1)/n_grid bilinear sampling that the - // reference implementation uses) and upload it to the graph - // input declared in clip_graph_hunyuanvl::build(). - GGML_ASSERT(model.position_embeddings != nullptr); - ggml_tensor * src_t = model.position_embeddings; - const int64_t n_embd = src_t->ne[0]; - const int64_t n_pos = src_t->ne[1]; // = n_grid * n_grid - const int n_grid = (int)std::lround(std::sqrt((double)n_pos)); - GGML_ASSERT((int64_t)n_grid * n_grid == n_pos); - const int out_w = pos_w; // pw - const int out_h = pos_h; // ph - - // Pull weight to host. - std::vector src(n_embd * n_pos); - ggml_backend_tensor_get(src_t, src.data(), 0, ggml_nbytes(src_t)); - - // Output layout matches ggml_new_tensor_2d(F32, n_embd, out_h*out_w): - // ne[0] = n_embd (fastest), ne[1] = out_h*out_w - // dst[(y*out_w + x) * n_embd + c] - std::vector dst((size_t)n_embd * out_h * out_w); - - const float sx = (float)(out_w + 0.1f) / (float)n_grid; - const float sy = (float)(out_h + 0.1f) / (float)n_grid; - - for (int y = 0; y < out_h; ++y) { - // Match ggml_compute_forward_upscale_f32 pixel-center - // convention (align_corners=False): src_y = (y+0.5)/sy - 0.5. - const float fy = ((float)y + 0.5f) / sy - 0.5f; - int y0 = (int)std::floor(fy); - int y1 = y0 + 1; - y0 = std::clamp(y0, 0, n_grid - 1); - y1 = std::clamp(y1, 0, n_grid - 1); - float wy1 = std::clamp(fy - (float)y0, 0.0f, 1.0f); - const float wy0 = 1.0f - wy1; - for (int x = 0; x < out_w; ++x) { - const float fx = ((float)x + 0.5f) / sx - 0.5f; - int x0 = (int)std::floor(fx); - int x1 = x0 + 1; - x0 = std::clamp(x0, 0, n_grid - 1); - x1 = std::clamp(x1, 0, n_grid - 1); - float wx1 = std::clamp(fx - (float)x0, 0.0f, 1.0f); - const float wx0 = 1.0f - wx1; - - const float w00 = wy0 * wx0; - const float w01 = wy0 * wx1; - const float w10 = wy1 * wx0; - const float w11 = wy1 * wx1; - - const float * s00 = &src[((size_t)y0 * n_grid + x0) * n_embd]; - const float * s01 = &src[((size_t)y0 * n_grid + x1) * n_embd]; - const float * s10 = &src[((size_t)y1 * n_grid + x0) * n_embd]; - const float * s11 = &src[((size_t)y1 * n_grid + x1) * n_embd]; - float * d = &dst[((size_t)y * out_w + x) * n_embd]; - for (int c = 0; c < n_embd; ++c) { - d[c] = w00 * s00[c] + w01 * s01[c] + w10 * s10[c] + w11 * s11[c]; - } - } - } - - set_input_f32("hunyuanvl_pos_embd", dst); - } break; - case PROJECTOR_TYPE_LLAMA4: - { - // set the 2D positions - int n_patches_per_col = image_size_width / patch_size; - std::vector pos_data(num_patches + 1, 0); // +1 for the [CLS] token - // last pos is always kept 0, it's for CLS - // dimension H - for (int i = 0; i < num_patches; i++) { - pos_data[i] = (i / n_patches_per_col) + 1; - } - set_input_i32("pos_h", pos_data); - // dimension W - for (int i = 0; i < num_patches; i++) { - pos_data[i] = (i % n_patches_per_col) + 1; - } - set_input_i32("pos_w", pos_data); - } break; - case PROJECTOR_TYPE_DOTS3NOTE_A: - { - GGML_ASSERT(imgs.entries.size() == 1); - const int n_pos = (imgs.entries.front().nx() + 7) / 8; // 3x stride-2 conv2d - std::vector positions(n_pos); - for (int i = 0; i < n_pos; i++) { - positions[i] = i; - } - set_input_i32("positions", positions); - } break; - case PROJECTOR_TYPE_GEMMA4A: - { - GGML_ASSERT(imgs.entries.size() == 1); - const auto & img0 = imgs.entries.front(); - // Compute n_pos matching SSCP output: two stride-2 convs - int n_pos = img0.nx(); - for (int i = 0; i < 2; i++) { n_pos = (n_pos - 1) / 2 + 1; } - - // Chunked local attention: blocked causal mask and RPE - const int chunk_size = 12; - const int max_past = 12; - const int context_size = chunk_size + max_past; - const int num_blocks = (n_pos + chunk_size - 1) / chunk_size; - - // Blocked causal attention mask: [context_size, chunk_size, num_blocks] - { - std::vector mask(context_size * chunk_size * num_blocks, -1e9f); - for (int b = 0; b < num_blocks; b++) { - for (int q = 0; q < chunk_size; q++) { - int gq = b * chunk_size + q; - for (int k = 0; k < context_size; k++) { - int gk = b * chunk_size - max_past + k; - if (gq < n_pos && gk >= 0 && gk < n_pos && gk <= gq && (gq - gk) < max_past) { - mask[k + q * context_size + b * context_size * chunk_size] = 0.0f; - } - } - } - } - set_input_f32("kq_mask", mask); - } - - // Sinusoidal RPE: 13 positions [12, 11, ..., 0] - { - const int n_embd = ctx->model.hparams.n_embd; - const int num_timescales = n_embd / 2; - const float log_timescale_increment = logf(10000.0f) / std::max(num_timescales - 1, 1); - const int rpe_len = max_past + 1; - std::vector pos_emb(n_embd * rpe_len, 0.0f); - for (int p = 0; p < rpe_len; p++) { - float position = (float)(max_past - p); - for (int i = 0; i < num_timescales; i++) { - float inv_ts = expf(-(float)i * log_timescale_increment); - float scaled = position * inv_ts; - pos_emb[p * n_embd + i] = sinf(scaled); - pos_emb[p * n_embd + i + num_timescales] = cosf(scaled); - } - } - set_input_f32("pos_emb", pos_emb); - } - } break; - case PROJECTOR_TYPE_MIMO_AUDIO: - { - GGML_ASSERT(imgs.entries.size() == 1); - const int n_frames = imgs.entries.front().nx(); - const int n_pos = (n_frames - 1) / 2 + 1; // matches conv1(s=1)+conv2(s=2) output length - - std::vector positions(n_pos); - for (int i = 0; i < n_pos; i++) { - positions[i] = i; - } - set_input_i32("mimo_audio_positions", positions); - - const int window = hparams.attn_window_size; - GGML_ASSERT(window > 0); - - const float neg_inf = std::numeric_limits::lowest(); - std::vector full_mask((size_t) n_pos * n_pos); - std::vector window_mask((size_t) n_pos * n_pos); - for (int q = 0; q < n_pos; q++) { - for (int k = 0; k < n_pos; k++) { - const bool causal_ok = k <= q; - full_mask[(size_t) q * n_pos + k] = causal_ok ? 0.0f : neg_inf; - window_mask[(size_t) q * n_pos + k] = (causal_ok && (q - k) <= window) ? 0.0f : neg_inf; - } - } - set_input_attn_mask(full_mask, 0); - set_input_attn_mask(window_mask, 1); - - // input_local_transformer: block-diagonal mask + in-group positions - { - const int n_pos_ds = (n_pos - 2) / 2 + 1; // matches downsample conv (k=2,s=2,p=0) - const int group_size = hparams.audio_local_group_size; - GGML_ASSERT(group_size > 0); - const int n_groups = (n_pos_ds + group_size - 1) / group_size; - const int n_padded = n_groups * group_size; - - std::vector local_positions(n_padded); - for (int i = 0; i < n_padded; i++) { - local_positions[i] = i % group_size; - } - set_input_i32("mimo_audio_local_positions", local_positions); - - std::vector local_mask((size_t) n_padded * n_padded); - for (int q = 0; q < n_padded; q++) { - for (int k = 0; k < n_padded; k++) { - const bool same_group = (q / group_size) == (k / group_size); - local_mask[(size_t) q * n_padded + k] = same_group ? 0.0f : neg_inf; - } - } - set_input_attn_mask(local_mask, 2); - } - } break; - case PROJECTOR_TYPE_LFM2A: - case PROJECTOR_TYPE_D1OMNI_A: - { - GGML_ASSERT(imgs.entries.size() == 1); - const auto n_frames = clip_n_output_tokens(ctx, &imgs.entries.front()); - - auto d_model = 512; - auto seq_len = n_frames * 2 - 1; - std::vector pos_emb(d_model*seq_len); - std::vector inv_freq(d_model / 2); - for (size_t i = 0; i < inv_freq.size(); ++i) { - inv_freq[i] = std::exp(-(std::log(10000.0) / (float)d_model) * (2.0f * (float)(i))); - } - for (int64_t pos = 0; pos < seq_len; ++pos) { - for (size_t i = 0; i < inv_freq.size(); ++i) { - const float ang = (n_frames - pos - 1) * inv_freq[i]; - pos_emb[pos*d_model + 2*i + 0] = sinf(ang); // even - pos_emb[pos*d_model + 2*i + 1] = cosf(ang); // odd - } - } - set_input_f32("pos_emb", pos_emb); - } break; - case PROJECTOR_TYPE_LFM3A: - { - GGML_ASSERT(imgs.entries.size() == 1); - const int n_pos = clip_n_output_tokens(ctx, &imgs.entries.front()); - - // the "pos" slot leads the state; without a stream the state is zero, the clip is encoded from its start - int base = 0; - if (params->enc_state != nullptr) { - const auto & slots = params->enc_state->slots; - float base_f = 0.0f; - ggml_backend_tensor_get(slots[0], &base_f, 0, sizeof(float)); - base = (int) base_f; - size_t i = 0; - for (const auto & slot : list_lfm3a_state_slots(hparams, model)) { - ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str())); - } - } else { - for (const auto & slot : list_lfm3a_state_slots(hparams, model)) { - ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str()); - ggml_backend_tensor_memset(t, 0, 0, ggml_nbytes(t)); - } - } - - set_lfm3a_window_inputs(base, n_pos, hparams.attn_window_size); - } break; - case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: - { - GGML_ASSERT(imgs.entries.size() == 1); - std::vector positions(ggml_nelements(get_inp_tensor("positions"))); - for (size_t i = 0; i < positions.size(); i++) { - positions[i] = (int32_t) i; - } - set_input_i32("positions", positions); - GGML_ASSERT(params->enc_state != nullptr); - set_input_f32("inp_norm", params->enc_state->mel_norm); - if (ggml_graph_get_tensor(gf, "spk_prev") != nullptr) { - set_input_f32("spk_prev", params->enc_state->spk_prev); - } - } break; - case PROJECTOR_TYPE_PARAKEET: - { - GGML_ASSERT(imgs.entries.size() == 1); - struct ggml_tensor * attn_mask = ggml_graph_get_tensor(gf, "attn_mask"); - const int n_q = attn_mask->ne[1]; - const int n_k = attn_mask->ne[0]; - const int n_frames = imgs.entries.front().nx(); - const int n_tokens_real = (n_frames + hparams.subsampling_factor-1) / hparams.subsampling_factor; - const float mask_value = -1e30f; - - std::vector mask_data(n_q * n_k); - if (n_k == n_q) { - // full attention: mask keys that are padding - for (int q = 0; q < n_q; ++q) { - for (int k = 0; k < n_k; ++k) { - mask_data[q * n_k + k] = (k >= n_tokens_real) ? mask_value : 0.0f; - } - } - } else { - // local attention: mask keys outside the valid window - const int att_left = n_k / 2; - for (int q = 0; q < n_q; ++q) { - for (int k = 0; k < n_k; ++k) { - const int key = q - att_left + k; - mask_data[q * n_k + k] = (key >= 0 && key < n_tokens_real) ? 0.0f : mask_value; - } - } - } - set_input_f32(attn_mask->name, mask_data); - - // local attention skew mask: zeroes out the probs that were - // computed for keys outside the valid sliding window. - if (struct ggml_tensor * local_mask = ggml_graph_get_tensor(gf, "local_mask")) { - const int lm_k = local_mask->ne[0]; - const int lm_q = local_mask->ne[1]; - const int window_size = lm_k - lm_q + 1; - std::vector lm_data(lm_q * lm_k); - for (int q = 0; q < lm_q; ++q) { - for (int k = 0; k < lm_k; ++k) { - const int rel = k - q; - lm_data[q * lm_k + k] = (rel >= 0 && rel < window_size) ? 1.0f : 0.0f; - } - } - set_input_f32(local_mask->name, lm_data); - } - - // Generate rotation frequencies for relative positional encoding. - { - const int n_state = hparams.n_embd; - const int d_half = n_state / 2; - const float log_10000 = logf(10000.0f); - std::vector freqs(d_half); - for (int k = 0; k < d_half; ++k) { - freqs[k] = expf(-(float(k * 2) * log_10000 / float(n_state))); - } - set_input_f32("pos_freqs", freqs); - } - - // Generate relative positional distance values which scaled by - // the frequency to produce the angles for sin/cos. - { - // window_size is only known after graph construction since it depends on - // n_time from the conv output, so we read it back from the graph tensor. - struct ggml_tensor * rel_pos = ggml_graph_get_tensor(gf, "rel_positions"); - const int window_size = rel_pos->ne[1]; - std::vector pos(window_size); - // local attention: window is fixed at [att_left, att_right] - // full attention: window covers the full sequence, centered - if (ggml_graph_get_tensor(gf, "local_mask")) { - const int att_left = window_size / 2; - for (int t = 0; t < window_size; ++t) { - pos[t] = float(att_left - t); - } - } else { - const int n_time = (window_size + 1) / 2; - for (int t = 0; t < window_size; ++t) { - pos[t] = float(n_time - 1 - t); - } - } - set_input_f32(rel_pos->name, pos); - } - } break; - case PROJECTOR_TYPE_GRANITE_SPEECH: - { - const int context_size = ctx->model.hparams.audio_chunk_size; - const int max_pos_emb = ctx->model.hparams.audio_max_pos_emb; - - std::vector dists((size_t) context_size * (size_t) context_size); - for (int i = 0; i < context_size; i++) { - for (int j = 0; j < context_size; j++) { - int d = i - j; - if (d < -context_size) d = -context_size; - if (d > context_size) d = context_size; - dists[(size_t) i * (size_t) context_size + (size_t) j] = d + max_pos_emb; - } - } - set_input_i32("attn_dists", dists); - - const int n_frames = image_size_width; - const int remainder = n_frames % context_size; - if (remainder > 0) { - const int num_blocks = (n_frames + context_size - 1) / context_size; - std::vector mask((size_t) context_size * (size_t) context_size * (size_t) num_blocks, 0.0f); - const float neg_inf = -INFINITY; - const size_t last_block_offset = (size_t) (num_blocks - 1) * (size_t) context_size * (size_t) context_size; - for (int q = 0; q < context_size; q++) { - for (int k = 0; k < context_size; k++) { - if (q >= remainder || k >= remainder) { - mask[last_block_offset + (size_t) q * (size_t) context_size + (size_t) k] = neg_inf; - } - } - } - set_input_f32("attn_mask", mask); - } - } break; - case PROJECTOR_TYPE_GRANITE4_VISION: - { - // Granite Vision 4.1 uses precomputed permutation index - // tensors to express the _win / _unwin / spatial sampling - // reshapes as ggml_get_rows gathers. The names are set - // by g4v_gather() in models/granite4-vision.cpp. - const int patch_size = model.hparams.patch_size; - const int image_side = imgs.entries.front().nx() / patch_size; - const int window_side = hparams.downsample_window_side; - const int query_side = hparams.downsample_query_side; - const int n = image_side / window_side; - const int new_side = n * query_side; - - // Builds the raster→window permutation indices for a - // (side, side) grid split into (n × n) windows of (win × win) - // tokens each. dst[w * win*win + p] = source raster index. - auto make_win_idx = [](int side, int win) { - const int nn = side / win; - std::vector idx(static_cast(side) * side); - for (int wy = 0; wy < nn; ++wy) { - for (int wx = 0; wx < nn; ++wx) { - for (int iy = 0; iy < win; ++iy) { - for (int ix = 0; ix < win; ++ix) { - const int w = wy * nn + wx; - const int p = iy * win + ix; - const int y = wy * win + iy; - const int x = wx * win + ix; - idx[static_cast(w) * (win*win) + p] = y * side + x; - } - } - } - } - return idx; - }; - - auto make_unwin_idx = [&](int side, int win) { - const std::vector fwd = make_win_idx(side, win); - std::vector inv(fwd.size()); - for (size_t i = 0; i < fwd.size(); ++i) { - inv[fwd[i]] = static_cast(i); - } - return inv; - }; - - auto make_spatial_idx = [](int side, int offset) { - const int off_y = (offset >> 1) & 1; - const int off_x = offset & 1; - const int new_s = side / 2; - std::vector idx(static_cast(new_s) * new_s); - for (int y = 0; y < new_s; ++y) { - for (int x = 0; x < new_s; ++x) { - idx[y * new_s + x] = (y * 2 + off_y) * side + (x * 2 + off_x); - } - } - return idx; - }; - - // the same permutation is applied to every tile of the stacked image - auto upload = [&](const std::string & name, const std::vector & idx) { - ggml_tensor * t = ggml_graph_get_tensor(gf, name.c_str()); - GGML_ASSERT(t); - GGML_ASSERT(ggml_nelements(t) % (int64_t) idx.size() == 0); - const int n_rep = ggml_nelements(t) / idx.size(); - std::vector buf; - buf.reserve(idx.size() * n_rep); - for (int i = 0; i < n_rep; ++i) { - buf.insert(buf.end(), idx.begin(), idx.end()); - } - ggml_backend_tensor_set(t, buf.data(), 0, ggml_nbytes(t)); - }; - - // Stage 1b only uses block 0's permutations; future stages - // will upload all blocks. - for (size_t bid = 0; bid < hparams.feature_layers.size(); ++bid) { - const std::string prefix = "g4v_blk" + std::to_string(bid) + "_"; - upload(prefix + "win_idx", make_win_idx(image_side, window_side)); - upload(prefix + "qwin_idx", make_win_idx(new_side, query_side)); - upload(prefix + "unwin_idx", make_unwin_idx(new_side, query_side)); - const auto spatial_offset = hparams.proj_spatial_offsets[bid]; - if (spatial_offset >= 0) { - upload(prefix + "spatial_idx", make_spatial_idx(image_side,spatial_offset)); - } - } - } break; - default: - GGML_ABORT("Unknown projector type"); - } - - clip_backend_cpu_set_n_threads(ctx->backend_cpu, params->n_threads); - - auto status = ggml_backend_sched_graph_compute(ctx->sched.get(), gf); - if (status != GGML_STATUS_SUCCESS) { - LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status); - return false; - } - - if (embeddings != nullptr) { - // sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one) - const int n_tokens_out = embeddings->ne[1]; - const int expected_n_tokens_out = clip_n_output_tokens(ctx, &imgs.entries[0]); - if (n_tokens_out != expected_n_tokens_out) { - LOG_ERR("%s: expected output %d tokens, got %d\n", __func__, expected_n_tokens_out, n_tokens_out); - GGML_ABORT("Invalid number of output tokens"); - } - - LOG_DBG("%s: output embedding shape [%d, %d, %d]\n", __func__, - (int)embeddings->ne[0], (int)embeddings->ne[1], (int)embeddings->ne[2]); - - // copy output to user buffer if provided - // if output is empty, skip the copy - auto & out_batch_embd = *params->out_embd; - if (!out_batch_embd.empty()) { - if (out_batch_embd.size() != (size_t)ggml_nelements(embeddings)) { - LOG_ERR("%s: output buffer has %zu elements but expected %zu\n", __func__, out_batch_embd.size(), (size_t)ggml_nelements(embeddings)); - GGML_ABORT("Output buffer size mismatch"); - } - ggml_backend_tensor_get(embeddings, out_batch_embd.data(), 0, ggml_nbytes(embeddings)); - } else { - LOG_WRN("%s: output buffer is empty, skipping copy\n", __func__); - } - } - - // - // for audio gen models - // - - if (ctx->proj_type() == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == CLIP_GEN_PROCESS_TOKENIZE) { - ggml_tensor * fsq = ggml_graph_get_tensor(gf, "out_fsq"); - GGML_ASSERT(fsq != nullptr && params->out_codes); - const int n_tok = (int) fsq->ne[1]; - std::vector h((size_t) 8 * n_tok); - ggml_backend_tensor_get(fsq, h.data(), 0, ggml_nbytes(fsq)); - - // fsq round to base 3: h in (-1, 1) maps to digits {0, 1, 2} - params->out_codes->resize(n_tok); - for (int t = 0; t < n_tok; t++) { - int32_t code = 0; - for (int i = 7; i >= 0; i--) { - const int32_t d = (int32_t) roundf(h[(size_t) t * 8 + i] * 0.9990000128746033f) + 1; - code = code * 3 + d; - } - (*params->out_codes)[(size_t) t] = code; - } - - // speech table rows of the codes, the multilingual variant adds its learned positions as its perceiver expects - if (params->out_code_embd) { - const auto & tensors = ctx->model.cbx_tensors; - auto tab_it = tensors.find("a.gen.code.out_embd.weight"); - GGML_ASSERT(tab_it != tensors.end()); - ggml_tensor * tab = tab_it->second; - const int n_e = (int) tab->ne[0]; - auto pos_it = tensors.find("a.gen.t3.speech_pos_emb"); - ggml_tensor * pos = pos_it == tensors.end() ? nullptr : pos_it->second; - if ((tab->type != GGML_TYPE_F16 && tab->type != GGML_TYPE_F32) || - (pos && (pos->type != GGML_TYPE_F32 || pos->ne[0] != n_e || pos->ne[1] < n_tok))) { - LOG_ERR("%s: unexpected speech table or positions\n", __func__); - return false; - } - - params->out_code_embd->resize((size_t) n_tok * n_e); - std::vector h16(n_e); - std::vector pr(n_e); - for (int t = 0; t < n_tok; t++) { - float * dst = params->out_code_embd->data() + (size_t) t * n_e; - if ((*params->out_codes)[(size_t) t] >= tab->ne[1]) { - LOG_ERR("%s: speech code beyond the speech table\n", __func__); - return false; - } - const size_t r = (size_t) (*params->out_codes)[(size_t) t] * n_e; - if (tab->type == GGML_TYPE_F16) { - ggml_backend_tensor_get(tab, h16.data(), r * sizeof(ggml_fp16_t), (size_t) n_e * sizeof(ggml_fp16_t)); - for (int j = 0; j < n_e; j++) { - dst[j] = ggml_fp16_to_fp32(h16[j]); - } - } else { - ggml_backend_tensor_get(tab, dst, r * sizeof(float), (size_t) n_e * sizeof(float)); - } - if (pos) { - ggml_backend_tensor_get(pos, pr.data(), (size_t) t * n_e * sizeof(float), (size_t) n_e * sizeof(float)); - for (int j = 0; j < n_e; j++) { - dst[j] += pr[j]; - } - } - } - } - return true; - } - - if (ctx->proj_type() == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == CLIP_GEN_PROCESS_TTS_VOCODE) { - ggml_tensor * sp = ggml_graph_get_tensor(gf, "out_spec"); - GGML_ASSERT(sp != nullptr && params->out_spec); - params->out_spec->resize(ggml_nelements(sp)); - ggml_backend_tensor_get(sp, params->out_spec->data(), 0, ggml_nbytes(sp)); - return true; - } - - if (ctx->proj_type() == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == CLIP_GEN_PROCESS_TTS) { - ggml_tensor * mel = ggml_graph_get_tensor(gf, "out_mel"); - GGML_ASSERT(mel != nullptr); - std::vector mel_data(ggml_nelements(mel)); - ggml_backend_tensor_get(mel, mel_data.data(), 0, ggml_nbytes(mel)); - - // hift: f0 -> harmonic source -> source stft on the host, then the vocoder graph and the istft - ggml_tensor * f0_t = ggml_graph_get_tensor(gf, "out_f0"); - GGML_ASSERT(f0_t != nullptr); - const int n_mel_out = (int) ggml_nelements(f0_t); - std::vector f0(n_mel_out); - ggml_backend_tensor_get(f0_t, f0.data(), 0, ggml_nbytes(f0_t)); - - // SineGen: f0 upsampled x480, 9 harmonics with cumulative phase, uv gating and noise, merged by l_linear + tanh - const int ups_total = 480; - const double sr = 24000.0; - const int64_t n_wav = (int64_t) n_mel_out * ups_total; - std::vector lw(9); - float lb = 0.0f; - if (clip_cbx_read_tensor(ctx, "a.gen.hift.m_source.l_linear.weight", lw.data(), lw.size()) != 9 || - clip_cbx_read_tensor(ctx, "a.gen.hift.m_source.l_linear.bias", &lb, 1) != 1) { - LOG_ERR("%s: unexpected hift source module\n", __func__); - return false; - } - - std::mt19937 & srng = params->gen_state->rng; - std::uniform_real_distribution ud(-M_PI, M_PI); - std::normal_distribution snd(0.0f, 1.0f); - double phase[9]; - for (int h = 0; h < 9; h++) { - phase[h] = h == 0 ? 0.0 : ud(srng); - } - std::vector src((size_t) n_wav); - double cum[9] = {0.0}; - for (int64_t t = 0; t < n_wav; t++) { - const float f = f0[(size_t) (t / ups_total)]; - const float uv = f > 10.0f ? 1.0f : 0.0f; // nsf_voiced_threshold - const float namp = uv * 0.003f + (1.0f - uv) * 0.1f / 3.0f; - float merged = lb; - for (int h = 0; h < 9; h++) { - cum[h] += (double) f * (h + 1) / sr; - cum[h] -= floor(cum[h]); - float sine = 0.1f * (float) sin(2.0 * M_PI * cum[h] + phase[h]); - sine = sine * uv + namp * snd(srng); - merged += lw[(size_t) h] * sine; - } - src[(size_t) t] = tanhf(merged); - } - - // stft of the source: n_fft 16, hop 4, hann window, centered - const int n_fft = 16, hop = 4, n_bins = 9; - std::vector win(n_fft); - for (int i = 0; i < n_fft; i++) win[(size_t) i] = 0.5f - 0.5f * cosf(2.0f * (float) M_PI * i / n_fft); - const int n_stft = (int) (n_wav / hop) + 1; - std::vector sstft((size_t) n_stft * 18); - for (int fr = 0; fr < n_stft; fr++) { - const int64_t c0 = (int64_t) fr * hop - n_fft / 2; // centered, reflect padded - for (int k = 0; k < n_bins; k++) { - double re = 0.0, im = 0.0; - for (int i = 0; i < n_fft; i++) { - int64_t idx = c0 + i; - if (idx < 0) idx = -idx; - if (idx >= n_wav) idx = 2 * (n_wav - 1) - idx; - const double v = (double) src[(size_t) idx] * win[(size_t) i]; - const double a = 2.0 * M_PI * k * i / n_fft; - re += v * cos(a); - im -= v * sin(a); - } - sstft[(size_t) fr * 18 + k ] = (float) re; - sstft[(size_t) fr * 18 + 9 + k] = (float) im; - } - } - - // vocoder graph on mel + source stft - std::vector spec; - { - clip_encode_params vp = *params; - vp.gen_process = CLIP_GEN_PROCESS_TTS_VOCODE; - vp.mel_in = &mel_data; - vp.sstft_in = &sstft; - vp.vocode_n_mel = n_mel_out; - vp.vocode_n_stft = n_stft; - vp.out_audio = nullptr; - vp.out_spec = &spec; - if (!clip_encode(ctx, &vp)) { - LOG_ERR("%s: vocoder stage failed\n", __func__); - return false; - } - } - - // istft: mag = clipped exp, phase = sin, hann overlap-add - const int n_frames_out = (int) (spec.size() / 18); - const int64_t n_out = (int64_t) (n_frames_out - 1) * hop; - std::vector acc((size_t) n_out + n_fft, 0.0); - std::vector wsum((size_t) n_out + n_fft, 0.0); - for (int fr = 0; fr < n_frames_out; fr++) { - double frame[16]; - for (int i = 0; i < n_fft; i++) { - double v = 0.0; - for (int k = 0; k < n_bins; k++) { - const double mag = fmin(exp((double) spec[(size_t) fr * 18 + k]), 1e2); - const double ph = sin((double) spec[(size_t) fr * 18 + 9 + k]); - const double re = mag * cos(ph), im = mag * sin(ph); - const double a = 2.0 * M_PI * k * i / n_fft; - const double w = (k == 0 || k == n_fft / 2) ? 1.0 : 2.0; - v += w * (re * cos(a) - im * sin(a)); - } - frame[i] = v / n_fft; - } - const int64_t o = (int64_t) fr * hop; - for (int i = 0; i < n_fft; i++) { - acc [(size_t) (o + i)] += frame[i] * win[(size_t) i]; - wsum[(size_t) (o + i)] += (double) win[(size_t) i] * win[(size_t) i]; - } - } - GGML_ASSERT(params->out_audio); - auto & out_audio = *params->out_audio; - out_audio.resize((size_t) std::min(n_out - n_fft / 2, n_wav)); - for (size_t i = 0; i < out_audio.size(); i++) { - const size_t j = i + n_fft / 2; // drop the centering pad - const double v = wsum[j] > 1e-11 ? acc[j] / wsum[j] : 0.0; - out_audio[(size_t) i] = (float) fmax(-0.99, fmin(0.99, v)); - } - - // trim_fade of the reference: silence 20 ms then fade in 20 ms, hides the artifact of the prompt boundary - const size_t n_trim = 24000 / 50; - for (size_t i = 0; i < 2 * n_trim && i < out_audio.size(); i++) { - const double g = i < n_trim ? 0.0 - : (cos(M_PI * (1.0 - (double) (i - n_trim) / n_trim)) + 1.0) / 2.0; - out_audio[(size_t) i] *= (float) g; - } - return true; - } - - // optional outputs: a pipeline yields codes or feats, and not all have an eos head - if (params->out_codes != nullptr) { - ggml_tensor * codes = ggml_graph_get_tensor(gf, "out_codes"); - if (codes != nullptr) { - auto & out_codes = *params->out_codes; - out_codes.resize(ggml_nelements(codes)); - ggml_backend_tensor_get(codes, out_codes.data(), 0, ggml_nbytes(codes)); - } - } - if (params->out_feats != nullptr) { - ggml_tensor * feats = ggml_graph_get_tensor(gf, "out_feats"); - if (feats != nullptr) { - auto & out_feats = *params->out_feats; - out_feats.resize(ggml_nelements(feats)); - ggml_backend_tensor_get(feats, out_feats.data(), 0, ggml_nbytes(feats)); - } - } - if (params->out_is_eos != nullptr) { - ggml_tensor * eos = ggml_graph_get_tensor(gf, "out_eos_score"); - if (eos != nullptr) { - GGML_ASSERT(ggml_nelements(eos) == 1); - float score = 0.0f; - ggml_backend_tensor_get(eos, &score, 0, sizeof(float)); - *params->out_is_eos = score > hparams.gen_eos_threshold; - } - } - if (params->out_audio != nullptr) { - ggml_tensor * audio = ggml_graph_get_tensor(gf, "out_audio"); - if (audio == nullptr) { - GGML_ABORT("out_audio requested but graph has no \"out_audio\" tensor"); - } - auto & out_audio = *params->out_audio; - out_audio.resize(ggml_nelements(audio)); - ggml_backend_tensor_get(audio, out_audio.data(), 0, ggml_nbytes(audio)); - - if (model.proj_type == PROJECTOR_TYPE_SOPRANO) { - std::vector spectrum = std::move(out_audio); - clip_graph_soprano::decode_spectrum(model, spectrum, out_audio); - } - - // drop the tail audio that comes from the code-0 rear padding, drama3 has no padding - if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN) { - const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0; - const int64_t n_frames_w = hparams.wav_tfm_swa; - const int64_t n_frames = params->codes ? (int64_t) params->codes->size() / n_codes : n_frames_w; - if (n_frames < n_frames_w) { - const size_t hop = out_audio.size() / n_frames_w; - out_audio.resize((size_t) n_frames * hop); - } - } - } - if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_LFM3A) { - const auto & slots = params->enc_state->slots; - size_t i = 0; - for (const auto & slot : list_lfm3a_state_slots(hparams, model)) { - ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str()); - GGML_ASSERT(t != nullptr); - copy_gen_state_out(t, slots[i++]); - } - } else if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) { - clip_enc_state * st = params->enc_state; - ggml_tensor * chunk = ggml_graph_get_tensor(gf, "spk_chunk"); - ggml_tensor * preds = ggml_graph_get_tensor(gf, "spk_preds"); - GGML_ASSERT(chunk != nullptr && preds != nullptr); - st->spk_chunk.resize(ggml_nelements(chunk)); - st->spk_preds.resize(ggml_nelements(preds)); - ggml_backend_tensor_get(chunk, st->spk_chunk.data(), 0, ggml_nbytes(chunk)); - ggml_backend_tensor_get(preds, st->spk_preds.data(), 0, ggml_nbytes(preds)); - } else if (params->enc_state != nullptr) { - clip_enc_state * st = params->enc_state; - ggml_tensor * tail = ggml_graph_get_tensor(gf, "out_stem_tail"); - GGML_ASSERT(tail != nullptr); - st->stem_tail.resize(ggml_nelements(tail)); - ggml_backend_tensor_get(tail, st->stem_tail.data(), 0, ggml_nbytes(tail)); - - const int n_new = (imgs.entries[0].nx() - 2) / 2; - for (int i = 0; i < n_new; i++) { - st->slot_pos[(st->n_past + i) % st->n_ctx] = st->n_past + i; - } - st->n_past += n_new; - } - - if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) { - const auto & slots = params->gen_state->slots; - size_t i = 0; - for (const auto & slot : list_gen_state_slots(hparams, model)) { - if (slot.in_place) { - i++; - continue; - } - ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str()); - if (t == nullptr) { - GGML_ABORT("graph has no \"state_out_%s\" tensor", slot.name.c_str()); - } - GGML_ASSERT(i < slots.size()); - copy_gen_state_out(t, slots[i++]); - } - } - - // - // Debug: dump final embeddings if MTMD_DEBUG_EMBEDDINGS is set - // - - if (ctx->debug_output_embeddings && embeddings != nullptr) { - const int64_t n_embd = embeddings->ne[0]; - const int64_t n_tokens = embeddings->ne[1]; - std::vector emb_data(ggml_nelements(embeddings)); - ggml_backend_tensor_get(embeddings, emb_data.data(), 0, ggml_nbytes(embeddings)); - - LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n"); - LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens); - - // TEMP debugging (parity validation), will be removed before merge - // when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data] - const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS"); - if (dump_path && strcmp(dump_path, "1") != 0) { - FILE * f = fopen(dump_path, "wb"); - if (f) { - const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd }; - fwrite(hdr, sizeof(hdr), 1, f); - fwrite(emb_data.data(), sizeof(float), emb_data.size(), f); - fclose(f); - } - } - - // Print first few values of first token - LOG_INF("Token 0 (first 16 values): "); - for (int i = 0; i < std::min((int64_t)16, n_embd); i++) { - LOG_INF("%.6f ", emb_data[i]); - } - LOG_INF("\n"); - - // Print last few values of first token - if (n_embd > 16) { - LOG_INF("Token 0 (last 16 values): "); - for (int64_t i = n_embd - 16; i < n_embd; i++) { - LOG_INF("%.6f ", emb_data[i]); - } - LOG_INF("\n"); - } - - // Compute and print statistics - float sum = 0.0f, sum_sq = 0.0f, min_val = emb_data[0], max_val = emb_data[0]; - for (size_t i = 0; i < emb_data.size(); i++) { - sum += emb_data[i]; - sum_sq += emb_data[i] * emb_data[i]; - min_val = std::min(min_val, emb_data[i]); - max_val = std::max(max_val, emb_data[i]); - } - float mean = sum / emb_data.size(); - float variance = (sum_sq / emb_data.size()) - (mean * mean); - LOG_INF("Stats: mean=%.6f, std=%.6f, min=%.6f, max=%.6f, sum=%.6f\n", - mean, sqrtf(variance), min_val, max_val, sum); - LOG_INF("=== END MTMD_DEBUG_EMBEDDINGS ===\n\n"); - } - - return true; -} - -int clip_n_mmproj_embd(const struct clip_ctx * ctx) { - switch (ctx->model.proj_type) { - case PROJECTOR_TYPE_LDP: - return ctx->model.mm_model_block_1_block_2_1_b->ne[0]; - case PROJECTOR_TYPE_LDPV2: - return ctx->model.mm_model_peg_0_b->ne[0]; + return mm_model_peg_0_b->ne[0]; case PROJECTOR_TYPE_MLP: case PROJECTOR_TYPE_PHI4: case PROJECTOR_TYPE_PIXTRAL: @@ -8173,21 +4209,21 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_DOTS_OCR: case PROJECTOR_TYPE_DOTS3NOTE_V: case PROJECTOR_TYPE_DOTS3NOTE_A: - return ctx->model.mm_2_w->ne[1]; + return mm_2_w->ne[1]; case PROJECTOR_TYPE_MLP_NORM: - return ctx->model.mm_3_b->ne[0]; + return mm_3_b->ne[0]; case PROJECTOR_TYPE_MINICPMV: - return ctx->model.mm_model_proj->ne[0]; + return mm_model_proj->ne[0]; case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: case PROJECTOR_TYPE_MINICPM_OCR_1_0: - return ctx->model.mm_ffn_down_w->ne[1]; + return mm_ffn_down_w->ne[1]; case PROJECTOR_TYPE_GLM_EDGE: - return ctx->model.mm_model_mlp_3_w->ne[1]; + return mm_model_mlp_3_w->ne[1]; case PROJECTOR_TYPE_MINIMAX_M3: - return ctx->model.mm_merger_fc2_b->ne[0]; + return mm_merger_fc2_b->ne[0]; case PROJECTOR_TYPE_MUSE_GLIMMER: - return ctx->model.mm_2_w->ne[1]; + return mm_2_w->ne[1]; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_MAGEVL: @@ -8195,47 +4231,47 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_EXAONE4_5: case PROJECTOR_TYPE_JANUS_PRO: case PROJECTOR_TYPE_YOUTUVL: - return ctx->model.mm_1_b->ne[0]; + return mm_1_b->ne[0]; case PROJECTOR_TYPE_SENSENOVA_U1: - return ctx->model.hparams.projection_dim; + return hparams.projection_dim; case PROJECTOR_TYPE_QWEN3VL: // main path + deepstack paths - return ctx->model.mm_1_b->ne[0] * (1 + ctx->model.n_deepstack_layers); + return mm_1_b->ne[0] * (1 + n_deepstack_layers); case PROJECTOR_TYPE_LING3VL: - return ctx->model.mm_1_b->ne[0]; + return mm_1_b->ne[0]; case PROJECTOR_TYPE_MIMOVL: - return ctx->model.mm_1_w->ne[1]; + return mm_1_w->ne[1]; case PROJECTOR_TYPE_STEP3VL: - return ctx->model.mm_model_proj->ne[1]; + return mm_model_proj->ne[1]; case PROJECTOR_TYPE_GEMMA3: case PROJECTOR_TYPE_GEMMA3NV: - return ctx->model.mm_input_proj_w->ne[0]; + return mm_input_proj_w->ne[0]; case PROJECTOR_TYPE_GEMMA4V: case PROJECTOR_TYPE_GEMMA4UV: case PROJECTOR_TYPE_GEMMA4A: case PROJECTOR_TYPE_GEMMA4UA: - return ctx->model.mm_input_proj_w->ne[1]; + return mm_input_proj_w->ne[1]; case PROJECTOR_TYPE_IDEFICS3: case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: - return ctx->model.mm_fc_w->ne[1]; + return mm_fc_w->ne[1]; case PROJECTOR_TYPE_ULTRAVOX: case PROJECTOR_TYPE_VOXTRAL: case PROJECTOR_TYPE_VOXTRAL_RT: case PROJECTOR_TYPE_MUSIC_FLAMINGO: - return ctx->model.mm_2_w->ne[1]; + return mm_2_w->ne[1]; case PROJECTOR_TYPE_MERALION: - return ctx->model.mm_3_w->ne[1]; // out_proj output dim + return mm_3_w->ne[1]; // out_proj output dim case PROJECTOR_TYPE_INTERNVL: case PROJECTOR_TYPE_NEMOTRON_V2_VL: case PROJECTOR_TYPE_NEMOTRON_OMNI: - return ctx->model.mm_3_w->ne[1]; + return mm_3_w->ne[1]; case PROJECTOR_TYPE_LLAMA4: - return ctx->model.mm_model_proj->ne[1]; + return mm_model_proj->ne[1]; case PROJECTOR_TYPE_QWEN2A: case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: - return ctx->model.mm_fc_w->ne[1]; + return mm_fc_w->ne[1]; case PROJECTOR_TYPE_QWEN3A: - return ctx->model.mm_2_w->ne[1]; + return mm_2_w->ne[1]; case PROJECTOR_TYPE_GLMA: case PROJECTOR_TYPE_LFM2: case PROJECTOR_TYPE_D1OMNI_V: @@ -8246,74 +4282,58 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_DEEPSEEK4V: case PROJECTOR_TYPE_DEEPSEEK41V: case PROJECTOR_TYPE_COHERE2V: - return ctx->model.mm_2_w->ne[1]; + return mm_2_w->ne[1]; case PROJECTOR_TYPE_HUNYUANVL: - return ctx->model.mm_model_proj->ne[1]; + return mm_model_proj->ne[1]; case PROJECTOR_TYPE_COGVLM: - return ctx->model.mm_4h_to_h_w->ne[1]; + return mm_4h_to_h_w->ne[1]; case PROJECTOR_TYPE_DEEPSEEKOCR: case PROJECTOR_TYPE_DEEPSEEKOCR2: - return ctx->model.mm_fc_w->ne[1]; + return mm_fc_w->ne[1]; case PROJECTOR_TYPE_LFM2A: - return ctx->model.position_embeddings->ne[0]; + return position_embeddings->ne[0]; case PROJECTOR_TYPE_D1OMNI_A: case PROJECTOR_TYPE_LFM3A: - return ctx->model.mm_3_w->ne[1]; + return mm_3_w->ne[1]; case PROJECTOR_TYPE_GRANITE_SPEECH: - return ctx->model.qf_proj_blocks[0].qf_proj_linear_w->ne[1]; + return qf_proj_blocks[0].qf_proj_linear_w->ne[1]; case PROJECTOR_TYPE_GRANITE4_VISION: - return ctx->model.qf_proj_blocks.size() * ctx->model.hparams.projection_dim; + return qf_proj_blocks.size() * hparams.projection_dim; case PROJECTOR_TYPE_GLM4V: case PROJECTOR_TYPE_GLM5V: - return ctx->model.mm_ffn_down_w->ne[1]; + return mm_ffn_down_w->ne[1]; case PROJECTOR_TYPE_MIMO_AUDIO: - return ctx->model.mm_2_w->ne[1]; + return mm_2_w->ne[1]; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: - return ctx->model.mm_fc_w->ne[2]; + return mm_fc_w->ne[2]; case PROJECTOR_TYPE_QWEN3TTS_GEN: case PROJECTOR_TYPE_LFM3A_GEN: case PROJECTOR_TYPE_DRAMA3_GEN: - return ctx->model.gen_code_out_embd_w->ne[0]; + return gen_code_out_embd_w->ne[0]; case PROJECTOR_TYPE_POCKETTTS_SPKENC: - return ctx->model.spk_proj_w->ne[1]; + return spk_proj_w->ne[1]; case PROJECTOR_TYPE_POCKETTTS_GEN: - return ctx->model.gen_input_lin_w->ne[1]; + return gen_input_lin_w->ne[1]; case PROJECTOR_TYPE_CHATTERBOX: // talker width, carried by the speech table - return ctx->model.cbx_tensors.at("a.gen.code.out_embd.weight")->ne[0]; + return cbx_tensors.at("a.gen.code.out_embd.weight")->ne[0]; case PROJECTOR_TYPE_CHATTERBOX_SPKENC: // talker conditioning rows, the speaker projection carries their width - return ctx->model.cbx_tensors.at("a.cenc.spkr_enc.weight")->ne[1]; + return cbx_tensors.at("a.cenc.spkr_enc.weight")->ne[1]; case PROJECTOR_TYPE_SOPRANO: - return ctx->model.hparams.projection_dim; + return hparams.projection_dim; case PROJECTOR_TYPE_PARAKEET: - return ctx->model.mm_1_w->ne[1]; + return mm_1_w->ne[1]; default: GGML_ABORT("Unknown projector type"); } } -bool clip_is_llava(const struct clip_ctx * ctx) { - return ctx->model.hparams.has_llava_projector; -} - -bool clip_has_vision_encoder(const struct clip_ctx * ctx) { - return ctx->model.modality == CLIP_MODALITY_VISION; -} - -bool clip_has_audio_encoder(const struct clip_ctx * ctx) { - return ctx->model.modality == CLIP_MODALITY_AUDIO; -} - -bool clip_support_batch(const struct clip_ctx * ctx) { - return ctx->support_batch; -} - // TODO @ngxson : this is no longer correct with mtmd_batch API // this was only meant to be used by qwen-vl-based models, to fuse 2 input images into one (qwen-vl video support) // this logic should be refactored in near future to distinctly handle "merge frames" and "batching" -int clip_model_n_temporal_merge(const struct clip_ctx * ctx) { - switch (ctx->proj_type()) { +int mtmd_model::n_temporal_merge() const { + switch (proj_type) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: @@ -8325,31 +4345,3 @@ int clip_model_n_temporal_merge(const struct clip_ctx * ctx) { return 1; } } - -// -// API used internally with mtmd -// - -projector_type clip_get_projector_type(const struct clip_ctx * ctx) { - return ctx->proj_type(); -} - -const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx) { - return &ctx->model.hparams; -} - -std::map clip_get_mem_usage(const struct clip_ctx * ctx) { - std::map result = ctx->mem_usage; - for (auto & [dev, size] : ctx->mem_compute) { - result[dev] += size; - } - return result; -} - -// -// API for debugging -// - -void clip_set_debug_output_embeddings(clip_ctx * ctx, bool enable) { - ctx->debug_output_embeddings = enable; -} diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/mtmd-model.h similarity index 92% rename from tools/mtmd/clip-model.h rename to tools/mtmd/mtmd-model.h index f50fc40b9..0d975ba36 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/mtmd-model.h @@ -1,8 +1,8 @@ #pragma once #include "ggml.h" -#include "clip.h" -#include "clip-impl.h" +#include "ggml-backend.h" +#include "mtmd-impl.h" #include #include @@ -12,6 +12,25 @@ #include #include +enum mtmd_modality { + MTMD_MODALITY_VISION, + MTMD_MODALITY_AUDIO, + MTMD_MODALITY_GEN_AUDIO, +}; + +// gen-side voice reference (chatterbox): speech tokens, mel-rate features and the speaker vector +struct mtmd_gen_ref { + std::vector tokens; + std::vector feat; + std::vector spk; +}; + +// a predefined voice: backbone rows, plus the gen-side reference when the gen stack needs one +struct mtmd_gen_voice { + std::vector rows; + mtmd_gen_ref ref; +}; + enum ffn_op_type { FFN_GELU, FFN_GELU_ERF, @@ -50,7 +69,7 @@ enum pad_style { PAD_NEAREST, }; -struct clip_hparams { +struct mtmd_hparams { int32_t image_size = 0; int32_t patch_size = 0; int32_t n_embd = 0; @@ -74,7 +93,7 @@ struct clip_hparams { std::array image_pad_color = {0, 0, 0}; // (preprocessor) for llava-uhd style models - std::vector image_res_candidates; + std::vector image_res_candidates; int32_t preproc_min_tiles = 0; int32_t preproc_max_tiles = 0; int32_t max_slice_nums = 9; // llava-uhd slice cap; per-model, carried in the GGUF @@ -277,7 +296,7 @@ struct clip_hparams { } }; -struct clip_layer { +struct mtmd_layer { // layernorm 1 (or layer input norm, or pre-attention norm) ggml_tensor * ln_1_w = nullptr; ggml_tensor * ln_1_b = nullptr; @@ -484,13 +503,13 @@ struct qf_block { ggml_tensor * qf_proj_post_norm_w = nullptr; ggml_tensor * qf_proj_post_norm_b = nullptr; ggml_tensor * qf_proj_img_pos = nullptr; // Vision only - std::vector qf_proj_layers; + std::vector qf_proj_layers; }; // pocket-tts SEANet stack, used in both directions: // encoder = conv_in -> per stage (residual unit, strided conv) -> conv_out // decoder = conv_in -> per stage (strided convtr, residual unit) -> conv_out -struct clip_seanet { +struct mtmd_seanet { // one residual unit: ELU -> dilated conv -> ELU -> pointwise conv, added to the input struct stage { ggml_tensor * res_conv1_w = nullptr; @@ -509,7 +528,7 @@ struct clip_seanet { }; // pocket-tts flow-matching decoder (SimpleMLPAdaLN) -struct clip_flow_net { +struct mtmd_flow_net { // AdaLN res block: in_ln -> modulate -> Linear -> SiLU -> Linear, gated residual struct block { ggml_tensor * norm_w = nullptr; @@ -545,7 +564,7 @@ struct clip_flow_net { }; // qwen3tts code2wav: RVQ codes -> raw PCM -struct clip_code2wav { +struct mtmd_code2wav { // "upsample" stage: one ConvNeXt block plus the causal ConvTranspose1d before it struct upsample_block { ggml_tensor * conv_w = nullptr; // causal ConvTranspose1d, 2x @@ -599,7 +618,7 @@ struct clip_code2wav { ggml_tensor * tfm_out_proj_w = nullptr; ggml_tensor * tfm_out_proj_b = nullptr; ggml_tensor * tfm_output_norm_w = nullptr; - std::vector tfm_layers; // reuses the generic block fields (ln_1/attn/ln_2/ffn/ls_1/ls_2) + std::vector tfm_layers; // reuses the generic block fields (ln_1/attn/ln_2/ffn/ls_1/ls_2) std::vector upsample; @@ -612,7 +631,7 @@ struct clip_code2wav { ggml_tensor * dac_post_conv_b = nullptr; }; -struct clip_chatterbox { +struct mtmd_chatterbox { // espnet rel-pos conformer layer of the flow encoder struct enc_layer { ggml_tensor * norm_mha_w = nullptr; @@ -842,7 +861,7 @@ struct clip_chatterbox { }; // drama3 tower: extra tensors around the generic block -struct clip_drama3_layer { +struct mtmd_drama3_layer { ggml_tensor * enorm_w = nullptr; ggml_tensor * hnorm_w = nullptr; ggml_tensor * eh_proj_w = nullptr; // [code embd; hidden] -> hidden @@ -850,7 +869,7 @@ struct clip_drama3_layer { ggml_tensor * out_norm_w = nullptr; }; -struct clip_vocos { +struct mtmd_vocos { ggml_tensor * input_w = nullptr; ggml_tensor * input_b = nullptr; ggml_tensor * norm_w = nullptr; @@ -859,13 +878,23 @@ struct clip_vocos { ggml_tensor * output_norm_b = nullptr; ggml_tensor * output_w = nullptr; ggml_tensor * output_b = nullptr; - std::vector blocks; + std::vector blocks; }; -struct clip_model { - clip_modality modality = CLIP_MODALITY_VISION; +// the weights and hparams of one modality of an mmproj file, read-only once loaded +struct mtmd_model { + mtmd_modality modality = MTMD_MODALITY_VISION; projector_type proj_type = PROJECTOR_TYPE_MLP; - clip_hparams hparams; + mtmd_hparams hparams; + + // runtime requirements declared by the model, applied by mtmd_subcontext + int max_nodes = 8192; + bool flash_attn_auto = true; // false: MTMD_FLASH_ATTN_TYPE_AUTO resolves to disabled + + // weights + ggml_context_ptr ctx_data; + ggml_backend_buffer_ptr buf; + std::map mem_weights; // embeddings ggml_tensor * class_embedding = nullptr; @@ -888,7 +917,7 @@ struct clip_model { ggml_tensor * pre_ln_w = nullptr; ggml_tensor * pre_ln_b = nullptr; - std::vector layers; + std::vector layers; // one patch embedder per audio pyramid stage (muse-spark) struct audio_patch { @@ -898,7 +927,7 @@ struct clip_model { }; std::vector a_patches; - int32_t n_deepstack_layers = 0; // used by Qwen3-VL, calculated from clip_layer + int32_t n_deepstack_layers = 0; // used by Qwen3-VL, calculated from mtmd_layer ggml_tensor * post_ln_w; ggml_tensor * post_ln_b; @@ -1085,7 +1114,7 @@ struct clip_model { // mimo-v2.5: LLM-side connector (input_local_transformer, separate from the // audio_tokenizer's own encoder `layers`) - std::vector mm_a_local_layers; + std::vector mm_a_local_layers; ggml_tensor * mm_a_local_norm_w = nullptr; // qwen3a @@ -1116,18 +1145,18 @@ struct clip_model { ggml_tensor * gen_code_mask_embd = nullptr; // qwen3tts code2wav: RVQ codes -> raw PCM - clip_code2wav c2w; - std::vector drama3_layers; + mtmd_code2wav c2w; + std::vector drama3_layers; // pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path) - clip_seanet seanet; + mtmd_seanet seanet; // pocket-tts: voice latent -> backbone embd (speaker path) ggml_tensor * spk_proj_w = nullptr; ggml_tensor * downsample_w = nullptr; // pocket-tts: flow-matching decoder, backbone hidden state -> next latent - clip_flow_net flow; + mtmd_flow_net flow; ggml_tensor * gen_out_eos_w = nullptr; ggml_tensor * gen_out_eos_b = nullptr; ggml_tensor * gen_input_lin_w = nullptr; // latent -> backbone embd @@ -1135,18 +1164,18 @@ struct clip_model { ggml_tensor * gen_emb_std = nullptr; ggml_tensor * gen_quant_out_w = nullptr; // latent -> decoder dim ggml_tensor * gen_upsample_w = nullptr; // depthwise convtr, frame rate -> encoder frame rate - std::vector gen_tfm_layers; // mimi decoder_transformer + std::vector gen_tfm_layers; // mimi decoder_transformer // chatterbox flow encoder, cfm estimator, hift vocoder, s3 tokenizer and CAMPPlus speaker encoder - clip_chatterbox cbx; + mtmd_chatterbox cbx; - // chatterbox tensors read on the host, see clip_cbx_read_tensor() + // chatterbox tensors read on the host, see cbx_read_tensor() std::map cbx_tensors; // predefined voices, no rows if the voice is named in the prompt (lfm3a) - std::map gen_voices; + std::map gen_voices; // soprano: Vocos decoder, hidden states -> STFT coefficients - clip_vocos vocos; + mtmd_vocos vocos; // cogvlm ggml_tensor * mm_post_fc_norm_w = nullptr; @@ -1178,10 +1207,10 @@ struct clip_model { int32_t n_sam_layers = 12; // used by deepseek-ocr sam encoder - std::vector sam_layers; + std::vector sam_layers; // nemotron transcribe: speaker expert, Sortformer head, fusion - std::vector spk_layers; + std::vector spk_layers; ggml_tensor * spk_inp_proj_w = nullptr; ggml_tensor * spk_pre_ln_w = nullptr; ggml_tensor * spk_pre_ln_b = nullptr; @@ -1254,6 +1283,51 @@ struct clip_model { || proj_type == PROJECTOR_TYPE_VOXTRAL || proj_type == PROJECTOR_TYPE_MERALION; } + + // this should be equal to the embedding dimension of the text model + int n_mmproj_embd() const; + + int n_output_tokens(const mtmd_image_f32 * img) const; + + // for M-RoPE, this will be the number of token positions in X and Y directions + // for other models, X will be the total number of tokens and Y will be 1 + int n_output_tokens_x(const mtmd_image_f32 * img) const; + int n_output_tokens_y(const mtmd_image_f32 * img) const; + + // max number of output tokens per image, -1 if not dynamic size + int image_max_tokens() const; + + // TODO @ngxson : this is no longer correct with mtmd_batch API + int n_temporal_merge() const; + + // read a chatterbox tensor by name as F32, returns the element count (0 if not found), out may be null to query the size + size_t cbx_read_tensor(const char * name, float * out, size_t n_max) const; }; -const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx); +struct mtmd_model_params { + // custom value provided by user, -1 if not set + int32_t image_min_tokens = -1; + int32_t image_max_tokens = -1; + bool no_alloc = false; // only create the tensors, do not read their data + mtmd_progress_callback progress_callback = nullptr; + void * progress_callback_user_data = nullptr; +}; + +// the modalities of an mmproj file, a missing one is nullptr +struct mtmd_models { + std::unique_ptr v; + std::unique_ptr a; + std::unique_ptr gen_a; +}; + +// the weights are allocated in buft, throws on error +mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params, ggml_backend_buffer_type_t buft); + +struct mtmd_model_caps { + bool has_vision; + bool has_audio; + bool has_gen_audio; +}; + +// read the modalities of an mmproj file without loading its tensors +mtmd_model_caps mtmd_model_get_caps(const char * fname); diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp new file mode 100644 index 000000000..fa33d5887 --- /dev/null +++ b/tools/mtmd/mtmd-subcontext.cpp @@ -0,0 +1,2701 @@ +#include "mtmd-subcontext.h" +#include "mtmd-graph.h" +#include "models/models.h" +#include "mtmd-audio.h" + +#include "ggml.h" +#include "ggml-cpp.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "ggml-cpu.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// ggml_backend_cpu_set_n_threads(backend_cpu, n_threads); +static void mtmd_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) { + ggml_backend_dev_t dev = ggml_backend_get_device(backend_cpu); + ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; + if (reg) { + auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); + if (ggml_backend_set_n_threads_fn) { + ggml_backend_set_n_threads_fn(backend_cpu, n_threads); + } + } +} + +std::shared_ptr mtmd_threadpool_init(int n_threads) { + ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); + ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; + if (!reg || n_threads < 2) { + return nullptr; + } + auto * new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new"); + auto * free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free"); + if (!new_fn || !free_fn) { + return nullptr; + } + ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads); + tpp.poll = 0; // the pool is idle most of the time, don't take CPU away from the text model + ggml_threadpool_t threadpool = new_fn(&tpp); + if (!threadpool) { + throw std::runtime_error("failed to create threadpool"); + } + return std::shared_ptr(threadpool, free_fn); +} + +ggml_backend_dev_t mtmd_select_device(bool use_gpu, ggml_backend_dev_t device) { + if (!use_gpu) { + return nullptr; + } + if (device != nullptr) { + return device; + } + ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU); + return dev ? dev : ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU); +} + +mtmd_subcontext::mtmd_subcontext(const mtmd_model & model, const mtmd_subctx_params & params) : + model(model), + threadpool(params.threadpool), + max_nodes(model.max_nodes), + flash_attn_type(params.flash_attn_type) { + if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_AUTO && !model.flash_attn_auto) { + flash_attn_type = MTMD_FLASH_ATTN_TYPE_DISABLED; + } + + backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); + if (!backend_cpu) { + throw std::runtime_error("failed to initialize CPU backend"); + } + if (threadpool) { + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); + auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); + if (set_threadpool_fn) { + set_threadpool_fn(backend_cpu, threadpool.get()); + } + } + if (params.device != nullptr) { + backend = ggml_backend_dev_init(params.device, nullptr); + if (!backend) { + ggml_backend_free(backend_cpu); + throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n", + __func__, ggml_backend_dev_name(params.device))); + } + } + + if (backend) { + LOG_INF("%s: using %s backend\n", __func__, ggml_backend_name(backend)); + backend_ptrs.push_back(backend); + backend_buft.push_back(ggml_backend_get_default_buffer_type(backend)); + } else { + backend = backend_cpu; + LOG_INF("%s: using CPU backend\n", __func__); + } + + backend_ptrs.push_back(backend_cpu); + backend_buft.push_back(ggml_backend_get_default_buffer_type(backend_cpu)); + + sched.reset( + ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, true) + ); + + if (params.cb_eval != nullptr) { + ggml_backend_sched_set_eval_callback(sched.get(), params.cb_eval, params.cb_eval_user_data); + } + + debug_output_embeddings = std::getenv("MTMD_DEBUG_EMBEDDINGS") != nullptr; + + buf_compute_meta.resize(max_nodes * ggml_tensor_overhead() + ggml_graph_overhead_custom(max_nodes, false)); + + // TODO: fix warmup for audio gen + if (model.modality != MTMD_MODALITY_GEN_AUDIO) { + // check batching support + auto batch = get_dummy_batch(); + support_batch = mtmd_get_graph_builder(this, batch)->support_batch(); + + if (params.warmup) { + warmup(batch); + } + } +} + +mtmd_subcontext::~mtmd_subcontext() { + // the scheduler references the backends + sched.reset(); + ggml_backend_free(backend); + if (backend != backend_cpu) { + ggml_backend_free(backend_cpu); + } +} + +mtmd_image_f32_batch mtmd_subcontext::get_dummy_batch() const { + // create a fake batch + const auto & hparams = model.hparams; + mtmd_image_f32_batch batch; + mtmd_image_f32 img; + if (model.modality == MTMD_MODALITY_VISION) { + const int sz = hparams.warmup_image_size; + img.set_size({sz, sz}, false, false); + LOG_INF("%s: warmup with image size = %d x %d\n", __func__, sz, sz); + } else { + // GEMMA4UA uses n_mel_bins as a raw-waveform frame size (640), not a mel-bin count, + // so the [1, 256] bound only applies to FFT-based models. + const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA; + if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) { + throw std::runtime_error(string_format("%s: invalid n_mel_bins (%d), must be in [1, 256]\n", __func__, hparams.n_mel_bins)); + } + img.set_size({hparams.warmup_audio_size, hparams.n_mel_bins}, false, false); + LOG_INF("%s: warmup with audio size = %d\n", __func__, hparams.warmup_audio_size); + } + batch.entries.push_back(img); + return batch; +} + +void mtmd_subcontext::warmup(const mtmd_image_f32_batch & batch) { + support_info_graph info; + + if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_AUTO) { + // try to enable flash attention to see if it's supported + flash_attn_type = MTMD_FLASH_ATTN_TYPE_ENABLED; + info = reserve_compute_meta(batch); + if (!info.fattn && info.fattn_op) { + auto op = info.fattn_op; + LOG_WRN("%s: *****************************************************************\n", __func__); + LOG_WRN("%s: WARNING: flash attention not supported by %s, memory usage will increase\n", __func__, ggml_backend_name(backend)); + LOG_WRN("%s: op params: \n", __func__); + static auto print_shape = [](const char * fn, const char * name, ggml_tensor * t) { + LOG_WRN("%s: %s: type = %s, ne = [%d %d %d %d], nb = [%d %d %d %d]\n", fn, + name, ggml_type_name(t->type), + t->ne[0], t->ne[1], t->ne[2], t->ne[3], + t->nb[0], t->nb[1], t->nb[2], t->nb[3]); + }; + print_shape(__func__, " dst", op); + print_shape(__func__, "src0", op->src[0]); + print_shape(__func__, "src1", op->src[1]); + print_shape(__func__, "src2", op->src[2]); + LOG_WRN("%s: please report this on github as an issue\n", __func__); + LOG_WRN("%s: *****************************************************************\n", __func__); + flash_attn_type = MTMD_FLASH_ATTN_TYPE_DISABLED; + reserve_compute_meta(batch); + } + } else { + info = reserve_compute_meta(batch); + if (!info.fattn && flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) { + LOG_WRN("%s: flash attention is not supported by the current backend; falling back to CPU (performance will be degraded)\n", __func__); + } + } + + is_allocated = true; // mark buffers as allocated + + LOG_INF("%s: flash attention is %s\n", __func__, + (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled"); + + // print ops that are not supported by the GPU backend (if there is one) + if (backend && backend != backend_cpu) { + std::vector unsupported_ops; + for (const auto & op : info.ops) { + if (!op.is_accel) { + unsupported_ops.push_back(op); + } + } + if (!unsupported_ops.empty()) { + LOG_WRN("%s: *****************************************************************\n", __func__); + LOG_WRN("%s: WARNING: the mmproj graph uses unsupported operators by the backend\n", __func__); + LOG_WRN("%s: the performance will be suboptimal \n", __func__); + LOG_WRN("%s: list of unsupported ops (backend=%s):\n", __func__, ggml_backend_name(backend)); + for (const auto & op : unsupported_ops) { + LOG_WRN("%s: %16s: type = %s, ne = [%d %d %d %d]\n", __func__, + ggml_op_name(op.op->op), + ggml_type_name(op.op->type), + op.op->ne[0], op.op->ne[1], op.op->ne[2], op.op->ne[3]); + } + LOG_WRN("%s: flash attention is %s\n", __func__, + (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled"); + LOG_WRN("%s: please report this on github as an issue\n", __func__); + LOG_WRN("%s: ref: https://github.com/ggml-org/llama.cpp/pull/16837#issuecomment-3461676118\n", __func__); + LOG_WRN("%s: *****************************************************************\n", __func__); + } + } +} + +mtmd_subcontext::support_info_graph mtmd_subcontext::reserve_compute_meta(const mtmd_image_f32_batch & batch) { + ggml_cgraph * gf = mtmd_get_graph_builder(this, batch)->build(); + ggml_backend_sched_reserve(sched.get(), gf); + + mem_compute.clear(); + for (size_t i = 0; i < backend_ptrs.size(); ++i) { + ggml_backend_t backend = backend_ptrs[i]; + ggml_backend_buffer_type_t buft = backend_buft[i]; + size_t size = ggml_backend_sched_get_buffer_size(sched.get(), backend); + if (size > 1) { + LOG_INF("%s: %10s compute buffer size = %8.2f MiB\n", __func__, + ggml_backend_buft_name(buft), + size / 1024.0 / 1024.0); + } + mem_compute[ggml_backend_get_device(backend)] += size; + } + + const int n_splits = ggml_backend_sched_get_n_splits(sched.get()); + const int n_nodes = ggml_graph_n_nodes(gf); + + LOG_INF("%s: graph splits = %d, nodes = %d\n", __func__, n_splits, n_nodes); + + support_info_graph res { + /*.fattn = */ true, + /*.fattn_op = */ nullptr, + /*.ops = */ {}, + }; + + // check op support + for (int i = 0; i < ggml_graph_n_nodes(gf); i++) { + ggml_tensor * node = ggml_graph_node(gf, i); + res.ops.push_back({node, true}); + if (!ggml_backend_supports_op(backend, node)) { + res.ops.back().is_accel = false; + if (node->op == GGML_OP_FLASH_ATTN_EXT) { + res.fattn = false; + res.fattn_op = node; + } + } + } + + return res; +} + +// TODO: remove this and use ggml_mul_mat with GGML_PREC_F32 once all backends honor it for F32 x F32 +// ggml_mul_mat for 2D F32 tensors, always in F32 on CPU: some GPU backends round the inputs to F16 or TF32, which loses the quiet bins of a spectrum +static void mtmd_mul_mat_prec_op(ggml_tensor * dst, int ith, int nth, void * /*userdata*/) { + const ggml_tensor * a = dst->src[0]; + const ggml_tensor * b = dst->src[1]; + const int64_t n_k = a->ne[0]; + + for (int64_t n = ith; n < b->ne[1]; n += nth) { + const float * b_row = (const float *) ((const char *) b->data + n * b->nb[1]); + float * dst_row = (float *) ((char *) dst->data + n * dst->nb[1]); + for (int64_t m = 0; m < a->ne[1]; m++) { + const float * a_row = (const float *) ((const char *) a->data + m * a->nb[1]); + // independent partial sums, so that the compiler can vectorize the loop + float acc[16] = {}; + int64_t k = 0; + for (; k + 16 <= n_k; k += 16) { + for (int l = 0; l < 16; l++) { + acc[l] += a_row[k + l] * b_row[k + l]; + } + } + float sum = 0.0f; + for (; k < n_k; k++) { + sum += a_row[k] * b_row[k]; + } + for (int l = 0; l < 16; l++) { + sum += acc[l]; + } + dst_row[m] = sum; + } + } +} + +static ggml_tensor * mtmd_mul_mat_prec(ggml_context * ctx, ggml_tensor * a, ggml_tensor * b) { + GGML_ASSERT(a->type == GGML_TYPE_F32 && b->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_n_dims(a) <= 2 && ggml_n_dims(b) <= 2 && a->ne[0] == b->ne[0]); + GGML_ASSERT(a->nb[0] == sizeof(float) && b->nb[0] == sizeof(float)); + ggml_tensor * args[] = { a, b }; + return ggml_custom_4d(ctx, GGML_TYPE_F32, a->ne[1], b->ne[1], 1, 1, args, 2, mtmd_mul_mat_prec_op, GGML_N_TASKS_MAX, nullptr); +} + +bool mtmd_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out) { + ggml_backend_sched_t sched = config.sched; + ggml_backend_t backend_cpu = ggml_backend_sched_get_backend(sched, ggml_backend_sched_get_n_backends(sched) - 1); + mtmd_backend_cpu_set_n_threads(backend_cpu, std::max(1, config.n_threads)); + + const auto & cache = *config.cache; + const auto & filters = cache.filters.data; + const int64_t n_fft_bins = config.n_fft_bins; + const int64_t frame_size = (n_fft_bins - 1) * 2; + const int64_t n_mel = config.n_mel; + GGML_ASSERT(filters.size() == (size_t) (n_mel * n_fft_bins)); + GGML_ASSERT(cache.cos_vals.size() == (size_t) frame_size && cache.sin_vals.size() == (size_t) frame_size); + + // DFT matrix with the window folded in, the first n_fft_bins rows give the real part + std::vector dft((size_t) 2 * n_fft_bins * frame_size); + for (int64_t k = 0; k < n_fft_bins; k++) { + for (int64_t j = 0; j < frame_size; j++) { + const int64_t q = (j * k) % frame_size; + dft[k * frame_size + j] = window[j] * cache.cos_vals[q]; + dft[(k + n_fft_bins) * frame_size + j] = window[j] * cache.sin_vals[q]; + } + } + + // the spectrum is several times larger than the result, so long audio is processed in blocks + const int64_t n_frames_max = 8192; + + for (int64_t off = 0; off < n_frames; off += n_frames_max) { + const int64_t n_cur = std::min(n_frames_max, n_frames - off); + + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead() * 32 + ggml_graph_overhead(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx_ptr(ggml_init(params)); + ggml_context * ctx0 = ctx_ptr.get(); + + ggml_tensor * inp_samples = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, (n_cur - 1) * config.hop_length + frame_size); + ggml_tensor * inp_dft = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, frame_size, 2 * n_fft_bins); + ggml_tensor * inp_filters = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_fft_bins, n_mel); + ggml_set_input(inp_samples); + ggml_set_input(inp_dft); + ggml_set_input(inp_filters); + + ggml_tensor * cur = ggml_cont(ctx0, ggml_view_2d(ctx0, inp_samples, frame_size, n_cur, config.hop_length * ggml_element_size(inp_samples), 0)); + cur = mtmd_mul_mat_prec(ctx0, inp_dft, cur); // [2 * n_fft_bins, n_cur] + + cur = ggml_sqr(ctx0, cur); + cur = ggml_add(ctx0, + ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], 0), + ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], n_fft_bins * ggml_element_size(cur))); + if (config.use_magnitude) { + cur = ggml_sqrt(ctx0, cur); + } + + cur = mtmd_mul_mat_prec(ctx0, cur, inp_filters); // [n_cur, n_mel] + cur = config.mel_floor_add + ? ggml_scale_bias(ctx0, cur, 1.0f, config.mel_floor) + : ggml_clamp(ctx0, cur, config.mel_floor, FLT_MAX); + cur = ggml_log(ctx0, cur); + if (!config.use_natural_log) { + cur = ggml_scale(ctx0, cur, 1.0f / logf(10.0f)); + } + ggml_set_output(cur); + + ggml_cgraph * gf = ggml_new_graph(ctx0); + ggml_build_forward_expand(gf, cur); + + ggml_backend_sched_reset(sched); + if (!ggml_backend_sched_alloc_graph(sched, gf)) { + LOG_ERR("%s: failed to allocate compute graph\n", __func__); + return false; + } + + ggml_backend_tensor_set(inp_samples, samples + off * config.hop_length, 0, ggml_nbytes(inp_samples)); + ggml_backend_tensor_set(inp_dft, dft.data(), 0, ggml_nbytes(inp_dft)); + ggml_backend_tensor_set(inp_filters, filters.data(), 0, ggml_nbytes(inp_filters)); + + auto status = ggml_backend_sched_graph_compute(sched, gf); + if (status != GGML_STATUS_SUCCESS) { + LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status); + return false; + } + + for (int64_t j = 0; j < n_mel; j++) { + ggml_backend_tensor_get(cur, out + j * n_frames + off, j * cur->nb[1], n_cur * ggml_element_size(cur)); + } + } + + return true; +} + +bool mtmd_subcontext::encode_image(int n_threads, const mtmd_image_f32 * img, std::vector & out_vec) { + mtmd_image_f32_batch imgs; + mtmd_image_f32 img_copy = *img; + imgs.entries.push_back(std::move(img_copy)); + + return encode_batch(n_threads, &imgs, out_vec); +} + +bool mtmd_subcontext::encode_batch(int n_threads, const mtmd_image_f32_batch * imgs_c_ptr, std::vector & out_batch_embd) { + mtmd_subctx_encode_params params; + params.imgs = imgs_c_ptr; + params.n_threads = n_threads; + params.out_embd = &out_batch_embd; + + if (model.proj_type == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) { + return mtmd_graph_nemotron_transcribe::encode_windows(this, model, ¶ms); + } + return encode(¶ms); +} + +// persisted state slots of the gen-audio decoders and the streaming audio encoders +static std::vector list_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { + switch (model.proj_type) { + case PROJECTOR_TYPE_QWEN3TTS_GEN: return list_c2w_state_slots(hparams, model); + case PROJECTOR_TYPE_DRAMA3_GEN: return list_drama3_state_slots(hparams, model); + case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model); + case PROJECTOR_TYPE_LFM3A_GEN: return list_lfm3a_gen_state_slots(hparams, model); + default: return {}; + } +} + +// state_out can have another shape than its slot, only the byte size must match +static void copy_gen_state_out(ggml_tensor * src, ggml_tensor * dst) { + GGML_ASSERT(ggml_nbytes(src) == ggml_nbytes(dst)); + if (src->type == dst->type && ggml_are_same_shape(src, dst) && ggml_is_contiguous(src)) { + ggml_backend_tensor_copy(src, dst); + return; + } + std::vector tmp(ggml_nbytes(src)); + ggml_backend_tensor_get(src, tmp.data(), 0, tmp.size()); + ggml_backend_tensor_set(dst, tmp.data(), 0, tmp.size()); +} + +mtmd_subctx_gen_state * mtmd_subcontext::gen_state_init(uint32_t seed) { + auto st = std::make_unique(); + st->rng.seed(seed == UINT32_MAX ? std::random_device{}() : seed); + + const auto slots = list_gen_state_slots(model.hparams, model); + if (slots.empty()) { + return st.release(); + } + + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead() * slots.size(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + st->ctx_data.reset(ggml_init(params)); + for (const auto & slot : slots) { + ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1); + ggml_format_name(t, "gen_state_%s", slot.name.c_str()); + st->slots.push_back(t); + } + st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backend)); + if (!st->buf) { + LOG_ERR("%s: failed to allocate the gen state\n", __func__); + return nullptr; + } + // a zero state is the cold start + ggml_backend_buffer_clear(st->buf.get(), 0); + + return st.release(); +} + +mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) { + const auto & hparams = model.hparams; + if (model.proj_type == PROJECTOR_TYPE_LFM3A) { + // downsampler context, short-conv taps and attention window, see list_lfm3a_state_slots() + const auto slots = list_lfm3a_state_slots(hparams, model); + auto st = std::make_unique(); + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead() * slots.size(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + st->ctx_data.reset(ggml_init(params)); + for (const auto & slot : slots) { + ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1); + ggml_format_name(t, "enc_state_%s", slot.name.c_str()); + st->slots.push_back(t); + } + st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backend)); + if (!st->buf) { + LOG_ERR("%s: failed to allocate the encoder state\n", __func__); + return nullptr; + } + // a zero state is the cold start + ggml_backend_buffer_clear(st->buf.get(), 0); + return st.release(); + } + if (model.proj_type != PROJECTOR_TYPE_VOXTRAL_RT || hparams.attn_window_size <= 0) { + LOG_ERR("%s: the encoder does not support streaming input\n", __func__); + return nullptr; + } + if (n_frames_max <= 0) { + LOG_ERR("%s: invalid n_frames_max %d\n", __func__, n_frames_max); + return nullptr; + } + + const int n_layer = hparams.n_layer; + const int d_head = hparams.n_embd_head > 0 ? hparams.n_embd_head : hparams.n_embd / hparams.n_head; + const int n_embd_kv = hparams.n_head_kv * d_head; + + auto st = std::make_unique(); + st->n_ctx = hparams.attn_window_size + n_frames_max; + st->slot_pos.assign(st->n_ctx, -1); + + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead() * 2 * n_layer, + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + st->ctx_data.reset(ggml_init(params)); + for (int il = 0; il < n_layer; il++) { + ggml_tensor * k = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F16, n_embd_kv, st->n_ctx); + ggml_tensor * v = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F16, n_embd_kv, st->n_ctx); + ggml_format_name(k, "enc_state_k_%d", il); + ggml_format_name(v, "enc_state_v_%d", il); + st->k.push_back(k); + st->v.push_back(v); + } + st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backend)); + if (!st->buf) { + LOG_ERR("%s: failed to allocate the encoder state\n", __func__); + return nullptr; + } + // empty slots are masked out, but they must not hold NaN + ggml_backend_buffer_clear(st->buf.get(), 0); + + LOG_INF("%s: encoder state: %d slots, %.2f MiB\n", __func__, st->n_ctx, ggml_backend_buffer_get_size(st->buf.get()) / 1024.0 / 1024.0); + + return st.release(); +} + +bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { + const mtmd_image_f32_batch & imgs = *params->imgs; + int n_batch_cur = imgs.entries.size(); + + // [QWEN_VIDEO] for video models, the batch dimension is used as temporal dimension for merged frames + if (!support_batch && n_batch_cur > model.n_temporal_merge()) { + LOG_ERR("%s: batch size %d exceeds maximum supported batch/temporal-merge size %d\n", __func__, n_batch_cur, model.n_temporal_merge()); + return false; + } + + // if buffers are not allocated, we need to do a warmup run to allocate them + if (!is_allocated) { + warmup(*params->imgs); + } + + const bool needs_gen_state = params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE || + params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV || + params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS; + if (needs_gen_state && params->gen_state == nullptr) { + LOG_ERR("%s: audio gen requires a gen state\n", __func__); + return false; + } + if (model.proj_type == PROJECTOR_TYPE_DRAMA3_GEN && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE) { + float pos = 0.0f; + ggml_backend_tensor_get(params->gen_state->slots[0], &pos, 0, sizeof(float)); + if ((int) pos >= mtmd_graph_drama3_gen::n_pos_max) { + LOG_ERR("%s: the codebook tower holds at most %d rows\n", __func__, mtmd_graph_drama3_gen::n_pos_max); + return false; + } + } + + // build the inference graph + ggml_backend_sched_reset(sched.get()); + ggml_cgraph * gf = mtmd_get_graph_builder(this, imgs, params)->build(); + + // the last node is the embedding tensor, code2wav has no out_embd + // read it before scheduling: backends may reorder the nodes of graphs with several outputs (Metal) + ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr; + + if (!ggml_backend_sched_alloc_graph(sched.get(), gf)) { + LOG_ERR("%s: failed to allocate compute graph\n", __func__); + return false; + } + + // set inputs + const auto & hparams = model.hparams; + + const int image_size_width = imgs.entries[0].nx(); + const int image_size_height = imgs.entries[0].ny(); + + const int patch_size = hparams.patch_size; + const int num_patches = ((image_size_width / patch_size) * (image_size_height / patch_size)); + const int n_pos = num_patches + (model.class_embedding ? 1 : 0); + const int pos_w = image_size_width / patch_size; + const int pos_h = image_size_height / patch_size; + + + auto get_inp_tensor = [&gf](const char * name) { + ggml_tensor * inp = ggml_graph_get_tensor(gf, name); + if (inp == nullptr) { + GGML_ABORT("Failed to get tensor %s", name); + } + if (!(inp->flags & GGML_TENSOR_FLAG_INPUT)) { + GGML_ABORT("Tensor %s is not an input tensor", name); + } + return inp; + }; + + auto set_input_f32 = [&get_inp_tensor](const char * name, const std::vector & values) { + ggml_tensor * cur = get_inp_tensor(name); + GGML_ASSERT(cur->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size()); + ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); + }; + + // mask from build_inp_attn_mask(), f16 if flash attn is enabled + auto set_input_attn_mask = [&get_inp_tensor](const std::vector & values, int idx = 0) { + ggml_tensor * cur = get_inp_tensor(get_attn_mask_name(idx).c_str()); + GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size()); + if (cur->type == GGML_TYPE_F16) { + std::vector values_f16(values.size()); + ggml_fp32_to_fp16_row(values.data(), values_f16.data(), values.size()); + ggml_backend_tensor_set(cur, values_f16.data(), 0, ggml_nbytes(cur)); + } else { + GGML_ASSERT(cur->type == GGML_TYPE_F32); + ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); + } + }; + + auto set_input_i32 = [&get_inp_tensor](const char * name, std::vector & values) { + ggml_tensor * cur = get_inp_tensor(name); + GGML_ASSERT(cur->type == GGML_TYPE_I32); + GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size()); + ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); + }; + + // state of the previous call, zero on a cold start + auto set_gen_state_in = [&]() { + const auto & slots = params->gen_state->slots; + size_t i = 0; + for (const auto & slot : list_gen_state_slots(hparams, model)) { + GGML_ASSERT(i < slots.size()); + if (slot.in_place) { + i++; + continue; + } + ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str())); + } + }; + + // lfm3a encoder and codec: causal sliding window over [cached window | this call] + auto set_lfm3a_window_inputs = [&](int base, int n_pos, int window) { + std::vector positions(n_pos); + for (int i = 0; i < n_pos; i++) { + positions[i] = base + i; + } + set_input_i32("positions", positions); + + // key j holds position base - prefix + j + const int prefix = window - 1; + const int n_kv = prefix + n_pos; + std::vector mask((size_t) n_kv * n_pos, -INFINITY); + for (int q = 0; q < n_pos; q++) { + for (int j = std::max(0, q + prefix - window + 1); j <= q + prefix; j++) { + if (base - prefix + j >= 0) { + mask[(size_t) q * n_kv + j] = 0.0f; + } + } + } + set_input_attn_mask(mask); + }; + + // rope positions and attention mask of the mimi transformers (pocket-tts). + // the mask is causal with a sliding window, see _build_attention_mask() in the reference + auto set_pockettts_tfm_inputs = [&]() { + const int64_t n_pos = ggml_nelements(get_inp_tensor("inp_pos")); + GGML_ASSERT(n_pos > 0); + std::vector positions((size_t) n_pos); + for (int64_t i = 0; i < n_pos; i++) { + positions[(size_t) i] = (int32_t) i; + } + set_input_i32("inp_pos", positions); + + // the preprocessor truncates the waveform to keep this mask bounded + const int64_t max_pos = (int64_t) mtmd_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate / 120; + GGML_ASSERT(n_pos <= max_pos && "pocket-tts speaker reference too long for a dense mask"); + + const int64_t context = hparams.mimi_tfm_context; + std::vector mask((size_t) n_pos * n_pos, -INFINITY); + for (int64_t q = 0; q < n_pos; q++) { + for (int64_t k = 0; k < n_pos; k++) { + const int64_t delta = q - k; + if (delta >= 0 && delta < context) { + mask[(size_t) q * n_pos + k] = 0.0f; + } + } + } + set_input_attn_mask(mask); + }; + + // set input pixel values + if (!imgs.is_audio) { + size_t nelem = 0; + for (const auto & img : imgs.entries) { + nelem += img.nx() * img.ny() * 3; + } + std::vector inp_raw(nelem); + + // layout of data (note: the channel dim is unrolled to better visualize the layout): + // + // ┌──W──┐ + // │ H │ channel = R + // ├─────┤ │ + // │ H │ channel = G + // ├─────┤ │ + // │ H │ channel = B + // └─────┘ │ + // ──────┘ x B + + // IMPORTANT: [QWEN_VIDEO] the batch dim is currently used for temporal dim in Qwen-VL models + // All entries must have the same spatial size (enforced by can_batch_with() during merging) + { + const int nx = imgs.entries[0].nx(); + const int ny = imgs.entries[0].ny(); + const int n = nx * ny; + + for (int b = 0; b < n_batch_cur; b++) { + LOG_DBG("%s: copying image %d/%d to input buffer (nx=%d, ny=%d)\n", __func__, b+1, n_batch_cur, nx, ny); + const auto & buf = imgs.entries[b].get_ro_buf(); + float * batch_entry = inp_raw.data() + b * (3*n); + for (int y = 0; y < ny; y++) { + for (int x = 0; x < nx; x++) { + size_t base_src = 3*(y * nx + x); + size_t base_dst = y * nx + x; + batch_entry[ base_dst] = buf[base_src ]; + batch_entry[1*n + base_dst] = buf[base_src + 1]; + batch_entry[2*n + base_dst] = buf[base_src + 2]; + } + } + } + } + set_input_f32("inp_raw", inp_raw); + + } else if (params->gen_process != MTMD_SUBCTX_GEN_PROCESS_GEN_WAV && + params->gen_process != MTMD_SUBCTX_GEN_PROCESS_TTS && + params->gen_process != MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE && + ggml_graph_get_tensor(gf, "inp_raw") != nullptr) { + // audio input. GEN_WAV and the chatterbox TTS stages are not here: they take codes or feats, set in the switch below. + // a GEN_CODE graph may have no hidden-state input (lfm3a_gen feedback embedding) + GGML_ASSERT(imgs.entries.size() == 1); + + const auto & mel_inp = imgs.entries[0]; + const auto & buf = mel_inp.get_ro_buf(); + const int n_step = mel_inp.nx(); + const int n_mel = mel_inp.ny(); + GGML_ASSERT((size_t)n_step * n_mel == buf.size()); + + set_input_f32("inp_raw", buf); + } + + // set input per projector + switch (model.proj_type) { + case PROJECTOR_TYPE_VOXTRAL_RT: + { + mtmd_subctx_enc_state * st = params->enc_state; + if (!st) { + LOG_ERR("%s: this encoder needs a mtmd_subctx_enc_state\n", __func__); + return false; + } + const int n_window = hparams.attn_window_size; + const int n_new = (imgs.entries[0].nx() - 2) / 2; + GGML_ASSERT(n_new <= st->n_ctx - n_window); + + std::vector pos(n_new); + std::vector idx(n_new); + std::vector slot_pos = st->slot_pos; // including the frames written by this call + for (int i = 0; i < n_new; i++) { + pos[i] = st->n_past + i; + idx[i] = pos[i] % st->n_ctx; + slot_pos[idx[i]] = pos[i]; + } + set_input_i32("inp_pos", pos); + ggml_backend_tensor_set(get_inp_tensor("inp_kv_idx"), idx.data(), 0, idx.size() * sizeof(int64_t)); + + // each frame sees itself and the n_window - 1 frames before it + std::vector mask((size_t) st->n_ctx * n_new, -INFINITY); + for (int i = 0; i < n_new; i++) { + for (int s = 0; s < st->n_ctx; s++) { + const int32_t p = slot_pos[s]; + if (p >= 0 && p <= pos[i] && p > pos[i] - n_window) { + mask[(size_t) i * st->n_ctx + s] = 0.0f; + } + } + } + set_input_attn_mask(mask); + + std::vector tail = st->stem_tail; + tail.resize(hparams.n_embd, 0.0f); + set_input_f32("inp_stem_tail", tail); + } break; + case PROJECTOR_TYPE_MUSE_GLIMMER: + { + const int grid_w = pos_w; // image_size_width / patch_size + const int grid_h = pos_h; // image_size_height / patch_size + const int n_tok = grid_w * grid_h; + const int pgrid = (int) std::sqrt((double) model.position_embeddings->ne[1]); // 32 + const int f = hparams.n_merge; // downsample 2 + + // pixel patchify runs inside the graph via build_inp() (ggml_conv_2d); + // pos-emb bilinear interp via resize_position_embeddings(). + + // --- sparse window grouping (pgrid x pgrid windows) --- + const int win = pgrid; + const int nwin_h = (grid_h + win - 1) / win; + const int nwin_w = (grid_w + win - 1) / win; + std::vector sp_perm; sp_perm.reserve(n_tok); + std::vector sp_slens; + for (int wy = 0; wy < nwin_h; wy++) { + for (int wx = 0; wx < nwin_w; wx++) { + int cnt = 0; + for (int hh = 0; hh < win; hh++) { + for (int ww = 0; ww < win; ww++) { + const int gy = wy * win + hh; + const int gx = wx * win + ww; + if (gy < grid_h && gx < grid_w) { sp_perm.push_back(gy * grid_w + gx); cnt++; } + } + } + if (cnt > 0) sp_slens.push_back(cnt); + } + } + std::vector rpos_w(n_tok), rpos_h(n_tok), inv_perm(n_tok); + for (int i = 0; i < n_tok; i++) { + const int orig = sp_perm[i]; + rpos_w[i] = (orig % grid_w) + 1; // 1-indexed + rpos_h[i] = (orig / grid_w) + 1; + inv_perm[orig] = i; + } + set_input_i32("muse_glimmer_sp_perm", sp_perm); + set_input_i32("muse_glimmer_inv_perm", inv_perm); + set_input_i32("muse_glimmer_pos_w", rpos_w); + set_input_i32("muse_glimmer_pos_h", rpos_h); + + // block-diagonal window mask (permuted order) + std::vector sp_mask((size_t) n_tok * n_tok, -INFINITY); + { + int off = 0; + for (int s : sp_slens) { + for (int a = 0; a < s; a++) + for (int b = 0; b < s; b++) + sp_mask[(size_t) (off + a) * n_tok + (off + b)] = 0.0f; + off += s; + } + } + set_input_attn_mask(sp_mask); + + // pixel-shuffle gather (original order): f*f spatial neighbours grouped + std::vector dsp; dsp.reserve(n_tok); + for (int oy = 0; oy < grid_h / f; oy++) + for (int ox = 0; ox < grid_w / f; ox++) + for (int ry = 0; ry < f; ry++) + for (int rx = 0; rx < f; rx++) + dsp.push_back((oy * f + ry) * grid_w + (ox * f + rx)); + set_input_i32("muse_glimmer_ds_perm", dsp); + } break; + case PROJECTOR_TYPE_MINICPMV: + { + // inspired from siglip: + // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit + // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit/blob/d66538faeba44480d0bfaa42145eef26f9423199/modeling_siglip.py#L316 + std::vector positions(pos_h * pos_w); + // note: sized by the actual patch counts; a tall/wide image produces more + // than 1024 patches per side and a fixed [1024] array would be overrun + std::vector bucket_coords_h(pos_h); + std::vector bucket_coords_w(pos_w); + for (int i = 0; i < pos_h; i++){ + bucket_coords_h[i] = std::floor(70.0*i/pos_h); + } + for (int i = 0; i < pos_w; i++){ + bucket_coords_w[i] = std::floor(70.0*i/pos_w); + } + for (int i = 0, id = 0; i < pos_h; i++){ + for (int j = 0; j < pos_w; j++){ + positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j]; + } + } + set_input_i32("positions", positions); + + // inputs for resampler projector + // set the 2D positions (using float for sinusoidal embedding) + int n_patches_per_col = image_size_width / patch_size; + std::vector pos_data(n_pos); + // dimension H + for (int i = 0; i < n_pos; i++) { + pos_data[i] = static_cast(i / n_patches_per_col); + } + set_input_f32("pos_h", pos_data); + // dimension W + for (int i = 0; i < n_pos; i++) { + pos_data[i] = static_cast(i % n_patches_per_col); + } + set_input_f32("pos_w", pos_data); + // base frequency omega + const float base_freq = 10000.0f; + const int n_embd_proj = model.n_mmproj_embd(); + std::vector omega(n_embd_proj / 4); + for (int i = 0; i < n_embd_proj / 4; ++i) { + omega[i] = 1.0f / std::pow(base_freq, static_cast(i) / (n_embd_proj / 4)); + } + set_input_f32("omega", omega); + } break; + case PROJECTOR_TYPE_MINICPMV4_6: + case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: + { + const bool is_4x = hparams.n_merge == 2; + + // SigLIP position buckets (same as resampler path) + std::vector positions(pos_h * pos_w); + std::vector bucket_coords_h(pos_h); + std::vector bucket_coords_w(pos_w); + for (int i = 0; i < pos_h; i++){ + bucket_coords_h[i] = std::floor(70.0*i/pos_h); + } + for (int i = 0; i < pos_w; i++){ + bucket_coords_w[i] = std::floor(70.0*i/pos_w); + } + for (int i = 0, id = 0; i < pos_h; i++){ + for (int j = 0; j < pos_w; j++){ + positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j]; + } + } + set_input_i32("positions", positions); + + const int half_h = pos_h / 2; + const int half_w = pos_w / 2; + + auto make_ds_idx = [](int off_r, int off_c, int ds_h, int ds_w, int stride_w) { + std::vector idx(ds_h * ds_w); + for (int i = 0; i < ds_h; i++) { + for (int j = 0; j < ds_w; j++) { + idx[i * ds_w + j] = (2*i + off_r) * stride_w + (2*j + off_c); + } + } + return idx; + }; + + if (!is_4x) { + // window reorder indices for 2x2 windows + std::vector window_idx(n_pos); + std::vector inv_window_idx(n_pos); + { + int k = 0; + for (int wi = 0; wi < half_h; wi++) { + for (int wj = 0; wj < half_w; wj++) { + window_idx[k++] = (2*wi ) * pos_w + (2*wj ); + window_idx[k++] = (2*wi ) * pos_w + (2*wj + 1); + window_idx[k++] = (2*wi + 1) * pos_w + (2*wj ); + window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1); + } + } + for (int i = 0; i < n_pos; i++) { + inv_window_idx[window_idx[i]] = i; + } + } + set_input_i32("vit_merger_window_idx", window_idx); + set_input_i32("vit_merger_inv_window_idx", inv_window_idx); + + // block-diagonal attention mask: tokens in the same 4-token + // window attend to each other (mask = 0), all other positions + // are masked out (-inf). matches the window-major reorder above. + std::vector window_mask_data(n_pos * n_pos, std::numeric_limits::lowest()); + for (int wi = 0; wi < n_pos / 4; wi++) { + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f; + } + } + } + set_input_attn_mask(window_mask_data); + + // ViT merger 2x2 downsample indices + auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w); + auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w); + auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w); + auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w); + set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0); + set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1); + set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2); + set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3); + } + + const int merger_h = is_4x ? pos_h : half_h; + const int merger_w = is_4x ? pos_w : half_w; + auto m_ds_0 = make_ds_idx(0, 0, merger_h / 2, merger_w / 2, merger_w); + auto m_ds_1 = make_ds_idx(0, 1, merger_h / 2, merger_w / 2, merger_w); + auto m_ds_2 = make_ds_idx(1, 0, merger_h / 2, merger_w / 2, merger_w); + auto m_ds_3 = make_ds_idx(1, 1, merger_h / 2, merger_w / 2, merger_w); + set_input_i32("merger_ds_idx_0", m_ds_0); + set_input_i32("merger_ds_idx_1", m_ds_1); + set_input_i32("merger_ds_idx_2", m_ds_2); + set_input_i32("merger_ds_idx_3", m_ds_3); + } break; + case PROJECTOR_TYPE_LFM3VL: + { + // patches in raster-scan order (no interleave before encoder layers) + const int pw = image_size_width / patch_size; + const int ph = image_size_height / patch_size; + std::vector positions(n_pos * 4); + int ptr = 0; + for (int y = 0; y < ph; y++) { + for (int x = 0; x < pw; x++) { + positions[ ptr] = y; + positions[ num_patches + ptr] = x; + positions[2 * num_patches + ptr] = y; + positions[3 * num_patches + ptr] = x; + ptr++; + } + } + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_MAGEVL: + { + // t, h, w in 2x2 block order, t is always 0 for an image + const int merge_ratio = hparams.n_merge; + const int pw = image_size_width / patch_size / merge_ratio; + const int ph = image_size_height / patch_size / merge_ratio; + std::vector positions(n_pos * 4, 0); + int ptr = 0; + for (int ty = 0; ty < ph; ty++) { + for (int tx = 0; tx < pw; tx++) { + for (int dy = 0; dy < merge_ratio; dy++) { + for (int dx = 0; dx < merge_ratio; dx++) { + positions[ num_patches + ptr] = ty * merge_ratio + dy; + positions[2 * num_patches + ptr] = tx * merge_ratio + dx; + ptr++; + } + } + } + } + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_QWEN2VL: + case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_LING3VL: + case PROJECTOR_TYPE_GLM4V: + case PROJECTOR_TYPE_GLM5V: + { + const int merge_ratio = hparams.n_merge; + const int pw = image_size_width / patch_size / merge_ratio; + const int ph = image_size_height / patch_size / merge_ratio; + std::vector positions(n_pos * 4); + int ptr = 0; + for (int ty = 0; ty < ph; ty++) { + for (int tx = 0; tx < pw; tx++) { + for (int dy = 0; dy < merge_ratio; dy++) { + for (int dx = 0; dx < merge_ratio; dx++) { + const int y = ty * merge_ratio + dy; + const int x = tx * merge_ratio + dx; + positions[ ptr] = y; + positions[ num_patches + ptr] = x; + positions[2 * num_patches + ptr] = y; + positions[3 * num_patches + ptr] = x; + ptr++; + } + } + } + } + + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_STEP3VL: + { + std::vector pos_data(n_pos); + for (int i = 0; i < n_pos; i++) { + pos_data[i] = i / pos_w; + } + set_input_i32("pos_h", pos_data); + for (int i = 0; i < n_pos; i++) { + pos_data[i] = i % pos_w; + } + set_input_i32("pos_w", pos_data); + } break; + case PROJECTOR_TYPE_PADDLEOCR: + { + const int merge_ratio = hparams.n_merge; + const int pw = image_size_width / patch_size / merge_ratio; + const int ph = image_size_height / patch_size / merge_ratio; + std::vector positions(n_pos * 4); + int ptr = 0; + // NOTE: same as Qwen-VL, but x and y are swapped + for (int ty = 0; ty < ph; ty++) { + for (int dy = 0; dy < merge_ratio; dy++) { + for (int tx = 0; tx < pw; tx++) { + for (int dx = 0; dx < merge_ratio; dx++) { + const int y = ty * merge_ratio + dy; + const int x = tx * merge_ratio + dx; + positions[ ptr] = y; + positions[ num_patches + ptr] = x; + positions[2 * num_patches + ptr] = y; + positions[3 * num_patches + ptr] = x; + ptr++; + } + } + } + } + + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + const int n_merge = hparams.n_merge; + const int gh = image_size_height / patch_size; + const int gw = image_size_width / patch_size; + std::vector pos_h, pos_w; + pos_h.reserve(gh * gw); + pos_w.reserve(gh * gw); + for (int bh = 0; bh < gh / n_merge; bh++) + for (int bw = 0; bw < gw / n_merge; bw++) + for (int mh = 0; mh < n_merge; mh++) + for (int mw = 0; mw < n_merge; mw++) { + pos_h.push_back(bh * n_merge + mh); + pos_w.push_back(bw * n_merge + mw); + } + set_input_i32("minimax_pos_h", pos_h); + set_input_i32("minimax_pos_w", pos_w); + } break; + case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: + { + const int pw = image_size_width / patch_size; + const int ph = image_size_height / patch_size; + const int n_pos = ph * pw; + std::vector positions(n_pos * 4); + int ptr = 0; + + // flat layout: [h, w, h, w] for each patch + // patches are in raster order (matching conv2d output) + for (int y = 0; y < ph; y++) { + for (int x = 0; x < pw; x++) { + positions[ ptr] = y; + positions[ n_pos + ptr] = x; + positions[2*n_pos + ptr] = y; + positions[3*n_pos + ptr] = x; + ptr++; + } + } + + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_QWEN25VL: + case PROJECTOR_TYPE_EXAONE4_5: + case PROJECTOR_TYPE_YOUTUVL: + { + // pw * ph = number of tokens output by ViT after apply patch merger + // ipw * ipw = number of vision token been processed inside ViT + const bool use_window_attn = + (model.proj_type == PROJECTOR_TYPE_QWEN25VL || model.proj_type == PROJECTOR_TYPE_EXAONE4_5) + ? hparams.n_wa_pattern > 0 + : !hparams.wa_layer_indexes.empty(); + const int merge_ratio = 2; + const int pw = image_size_width / patch_size / merge_ratio; + const int ph = image_size_height / patch_size / merge_ratio; + const int ipw = image_size_width / patch_size; + const int iph = image_size_height / patch_size; + + std::vector idx (ph * pw); + std::vector inv_idx(ph * pw); + + if (use_window_attn) { + const int attn_window_size = hparams.attn_window_size > 0 ? hparams.attn_window_size : 112; + const int grid_window = attn_window_size / patch_size / merge_ratio; + int dst = 0; + // [num_vision_tokens, num_vision_tokens] attention mask tensor + std::vector mask(pow(ipw * iph, 2), std::numeric_limits::lowest()); + int mask_row = 0; + + for (int y = 0; y < ph; y += grid_window) { + for (int x = 0; x < pw; x += grid_window) { + const int win_h = std::min(grid_window, ph - y); + const int win_w = std::min(grid_window, pw - x); + const int dst_0 = dst; + // group all tokens belong to the same window togather (to a continue range) + for (int dy = 0; dy < win_h; dy++) { + for (int dx = 0; dx < win_w; dx++) { + const int src = (y + dy) * pw + (x + dx); + GGML_ASSERT(src < (int)idx.size()); + GGML_ASSERT(dst < (int)inv_idx.size()); + idx [src] = dst; + inv_idx[dst] = src; + dst++; + } + } + + for (int r=0; r < win_h * win_w * merge_ratio * merge_ratio; r++) { + int row_offset = mask_row * (ipw * iph); + std::fill( + mask.begin() + row_offset + (dst_0 * merge_ratio * merge_ratio), + mask.begin() + row_offset + (dst * merge_ratio * merge_ratio), + 0.0); + mask_row++; + } + } + } + + set_input_i32("window_idx", idx); + set_input_i32("inv_window_idx", inv_idx); + set_input_attn_mask(mask); + } else { + for (int i = 0; i < ph * pw; i++) { + idx[i] = i; + } + } + + const int mpow = merge_ratio * merge_ratio; + std::vector positions(n_pos * 4); + + int ptr = 0; + for (int ty = 0; ty < ph; ty++) { + for (int tx = 0; tx < pw; tx++) { + for (int dy = 0; dy < merge_ratio; dy++) { + for (int dx = 0; dx < merge_ratio; dx++) { + auto remap = idx[ptr / mpow]; + remap = (remap * mpow) + (ptr % mpow); + + const int y = ty * merge_ratio + dy; + const int x = tx * merge_ratio + dx; + positions[ remap] = y; + positions[ num_patches + remap] = x; + positions[2 * num_patches + remap] = y; + positions[3 * num_patches + remap] = x; + ptr++; + } + } + } + } + + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_MIMOVL: + { + const int merge = hparams.n_merge; // 2 + const int merge_unit = merge * merge; // 4 + const int patch = hparams.patch_size; // 16 + const int H = image_size_height / patch; + const int W = image_size_width / patch; + const int n_pos_full = H * W; + const int llm_h = H / merge; + const int llm_w = W / merge; + const int n_units = llm_h * llm_w; // n_pos / merge_unit + + // Row-major merge-tile-ordered (h, w) positions + std::vector pos_h_row(n_pos_full); + std::vector pos_w_row(n_pos_full); + { + int idx = 0; + for (int ty = 0; ty < llm_h; ty++) { + for (int tx = 0; tx < llm_w; tx++) { + for (int dy = 0; dy < merge; dy++) { + for (int dx = 0; dx < merge; dx++) { + pos_h_row[idx] = ty * merge + dy; + pos_w_row[idx] = tx * merge + dx; + idx++; + } + } + } + } + } + + // Col-major merge-unit permutation + std::vector idx_col(n_units); + for (int r = 0; r < llm_h; r++) { + for (int c = 0; c < llm_w; c++) { + int u_row = r * llm_w + c; + int u_col = c * llm_h + r; + idx_col[u_col] = (float) u_row; + } + } + + // Col-mode positions: permute pos_*_row by idx_col + std::vector pos_h_col(n_pos_full); + std::vector pos_w_col(n_pos_full); + for (int u = 0; u < n_units; u++) { + int src = (int) idx_col[u]; + for (int k = 0; k < merge_unit; k++) { + pos_h_col[u * merge_unit + k] = pos_h_row[src * merge_unit + k]; + pos_w_col[u * merge_unit + k] = pos_w_row[src * merge_unit + k]; + } + } + + // Pack into ggml_rope_multi VISION-mode layout. The non-CPU kernels + // only read slots 0 and 1, so pack h in slot 0, w in slot 1: + // positions[0..n_pos) = h + // positions[n_pos..2*n_pos) = w + // positions[2*n_pos..3*n_pos) = 0 + // positions[3*n_pos..4*n_pos) = 0 + std::vector positions_row(static_cast(n_pos_full) * 4, 0); + std::vector positions_col(static_cast(n_pos_full) * 4, 0); + for (int i = 0; i < n_pos_full; i++) { + positions_row[0 * n_pos_full + i] = pos_h_row[i]; + positions_row[1 * n_pos_full + i] = pos_w_row[i]; + positions_col[0 * n_pos_full + i] = pos_h_col[i]; + positions_col[1 * n_pos_full + i] = pos_w_col[i]; + } + + // Banded 1D sliding-window mask + const int window = hparams.attn_window_size; + GGML_ASSERT(window > 0); + std::vector mask(static_cast(n_pos_full) * n_pos_full, std::numeric_limits::lowest()); + for (int q = 0; q < n_pos_full; q++) { + int lo = std::max(0, q - window); + int hi = std::min(n_pos_full - 1, q + window); + for (int k = lo; k <= hi; k++) { + mask[static_cast(q) * n_pos_full + k] = 0.0f; + } + } + + set_input_i32("mimovl_positions_row", positions_row); + set_input_i32("mimovl_positions_col", positions_col); + set_input_f32("mimovl_idx_col", idx_col); + set_input_attn_mask(mask); + } break; + case PROJECTOR_TYPE_PIXTRAL: + case PROJECTOR_TYPE_KIMIVL: + case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_LIGHTONOCR: + { + // set the 2D positions + int n_patches_per_col = image_size_width / patch_size; + std::vector pos_data(n_pos); + // dimension H + for (int i = 0; i < n_pos; i++) { + pos_data[i] = i / n_patches_per_col; + } + set_input_i32("pos_h", pos_data); + // dimension W + for (int i = 0; i < n_pos; i++) { + pos_data[i] = i % n_patches_per_col; + } + set_input_i32("pos_w", pos_data); + } break; + case PROJECTOR_TYPE_SENSENOVA_U1: + { + const int n_patches_per_row = image_size_width / patch_size; + std::vector pos_data(n_pos); + for (int i = 0; i < n_pos; i++) { + pos_data[i] = i / n_patches_per_row; + } + set_input_i32("pos_y", pos_data); + for (int i = 0; i < n_pos; i++) { + pos_data[i] = i % n_patches_per_row; + } + set_input_i32("pos_x", pos_data); + } break; + case PROJECTOR_TYPE_DEEPSEEK4V: + { + // set the 2D positions (mrope layout, only the first 2 channels are used) + int n_patches_per_row = image_size_width / patch_size; + std::vector positions(n_pos * 4, 0); + for (int i = 0; i < n_pos; i++) { + positions[i] = i / n_patches_per_row; // row + positions[n_pos + i] = i % n_patches_per_row; // col + } + set_input_i32("positions", positions); + + // token block layout index (see mtmd_graph_deepseek4v::build) + // rows [0, n_grid) are the aligner output, the sentinels follow + const int n_merge = hparams.n_merge; + const int n_llm_w = MTMD_ALIGN(pos_w, n_merge) / n_merge; + const int n_llm_h = MTMD_ALIGN(pos_h, n_merge) / n_merge; + const int n_grid = n_llm_w * n_llm_h; + const int idx_start = n_grid; + const int idx_end = n_grid + 1; + const int idx_newline = n_grid + 2; + const int idx_pad = n_grid + 3; + + const int lead_pad = imgs.entries[0].lead_pad; + const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad); + + std::vector idx; + idx.reserve(bl.n_out); + for (int i = 0; i < lead_pad; i++) { + idx.push_back(idx_pad); + } + idx.push_back(idx_start); + // pairs of adjacent rows are interleaved column-wise ("N-layout") + // ref: build_image_block in inference/image_processor.py + for (int t = 0; t < bl.rows * bl.row_len; t++) { + const int g = t / (2 * bl.row_len); + const int rem = t % (2 * bl.row_len); + const int c = rem / 2; // column + const int r = 2 * g + rem % 2; // row + if (r >= n_llm_h) { + idx.push_back(idx_pad); + } else if (c == n_llm_w) { + idx.push_back(idx_newline); + } else { + idx.push_back(r * n_llm_w + c); + } + } + for (int i = 0; i < bl.pad_last; i++) { + idx.push_back(idx_pad); + } + idx.push_back(idx_end); + set_input_i32("layout_idx", idx); + } break; + case PROJECTOR_TYPE_DEEPSEEK41V: + { + // mrope layout, only the first 2 channels are used + const int n_patches_per_row = image_size_width / patch_size; + std::vector positions(n_pos * 4, 0); + for (int i = 0; i < n_pos; i++) { + positions[i] = i / n_patches_per_row; + positions[n_pos + i] = i % n_patches_per_row; + } + set_input_i32("positions", positions); + + // reading-order block: rows [0, n_grid) are the aligner output, then start / end / newline + const int n_merge = hparams.n_merge; + const int n_llm_w = MTMD_ALIGN(pos_w, n_merge) / n_merge; + const int n_llm_h = MTMD_ALIGN(pos_h, n_merge) / n_merge; + const int n_grid = n_llm_w * n_llm_h; + std::vector idx; + idx.reserve(dsv41_n_block_tokens(n_llm_w, n_llm_h)); + idx.push_back(n_grid); // IMAGE_START + for (int r = 0; r < n_llm_h; r++) { + for (int c = 0; c < n_llm_w; c++) { + idx.push_back(r * n_llm_w + c); + } + idx.push_back(n_grid + 2); // IMAGE_NEW_LINE + } + idx.push_back(n_grid + 1); // IMAGE_END + set_input_i32("layout_idx", idx); + } break; + case PROJECTOR_TYPE_GLM_EDGE: + { + // llava and other models + std::vector positions(n_pos); + for (int i = 0; i < n_pos; i++) { + positions[i] = i; + } + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_MLP: + case PROJECTOR_TYPE_MLP_NORM: + case PROJECTOR_TYPE_LDP: + case PROJECTOR_TYPE_LDPV2: + { + // llava and other models + std::vector positions(n_pos); + for (int i = 0; i < n_pos; i++) { + positions[i] = i; + } + set_input_i32("positions", positions); + + // The patches vector is used to get rows to index into the embeds with; + // we should skip dim 0 only if we have CLS to avoid going out of bounds + // when retrieving the rows. + int patch_offset = model.class_embedding ? 1 : 0; + std::vector patches(num_patches); + for (int i = 0; i < num_patches; i++) { + patches[i] = i + patch_offset; + } + set_input_i32("patches", patches); + } break; + case PROJECTOR_TYPE_POCKETTTS_SPKENC: + { + set_pockettts_tfm_inputs(); + } break; + case PROJECTOR_TYPE_POCKETTTS_GEN: + { + if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { + GGML_ASSERT(params->feats != nullptr); + set_input_f32("inp_feats", *params->feats); + // positions and mask are derived in-graph from the persisted counter + set_gen_state_in(); + } else { + // flow matching starts from gaussian noise, std = sqrt(temp) + ggml_tensor * t = get_inp_tensor("inp_noise"); + // Config.default_temperature, for a caller that does not set one + const float temp = params->temp > 0.0f ? params->temp : 0.7f; + std::normal_distribution dist(0.0f, std::sqrt(temp)); + std::vector noise(ggml_nelements(t)); + for (auto & v : noise) { + v = dist(params->gen_state->rng); + } + set_input_f32("inp_noise", noise); + } + } break; + case PROJECTOR_TYPE_SOPRANO: + { + set_input_f32("inp_feats", *params->feats); + } break; + case PROJECTOR_TYPE_GEMMA4V: + case PROJECTOR_TYPE_GEMMA4UV: + { + // set (col, row) patch positions for learned positional embedding + const int n_cols = image_size_width / patch_size; + std::vector pos_x(num_patches), pos_y(num_patches); + for (int i = 0; i < num_patches; i++) { + pos_x[i] = i % n_cols; + pos_y[i] = i / n_cols; + } + set_input_i32("pos_x", pos_x); + set_input_i32("pos_y", pos_y); + } break; + case PROJECTOR_TYPE_DEEPSEEKOCR: + case PROJECTOR_TYPE_DEEPSEEKOCR2: + { + GGML_ASSERT( + (pos_w == pos_h) // overview image + || (pos_h >= pos_w && pos_h % pos_w == 0) // tile images + ); + + const int window = hparams.attn_window_size; + const int pos = pos_w; + std::vector rel_pos_indices_local(window * window); + std::vector rel_pos_indices_global(pos * pos); + + for (int q = 0; q < window; q++) { + for (int k = 0; k < window; k++) { + rel_pos_indices_local[q * window + k] = q - k + window - 1; + } + } + + for (int q = 0; q < pos; q++) { + for (int k = 0; k < pos; k++) { + rel_pos_indices_global[q * pos + k] = q - k + pos - 1; + } + } + + set_input_i32("rel_pos_indices_local", rel_pos_indices_local); + set_input_i32("rel_pos_indices_global", rel_pos_indices_global); + + if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2) { + + // qwen2 encoder attention mask + + // num_image_tokens = num_patches / 16 + // 256 for 1024 global view + // 144 for 768 tile views + const int num_image_tokens = num_patches / 16; + const int seq_len = num_image_tokens * 2; + std::vector qwen2_mask(static_cast(seq_len) * seq_len, 0.0f); + + // attention mask layout + // +--------------+---------------+ + // | all 0 | all -inf | + // +--------------+---------------+ + // | all 0 | lower tri 0 | + // +--------------+---------------+ + for (int i = 0; i < seq_len; i++) { + for (int j = 0; j < seq_len; j++) { + const bool zero = i < num_image_tokens ? + j < num_image_tokens : + j < num_image_tokens || j <= i; + qwen2_mask[static_cast(i) * seq_len + j] = zero ? 0.0f : -1e9f; + } + } + set_input_attn_mask(qwen2_mask); + } + } break; + case PROJECTOR_TYPE_CHATTERBOX: + { + if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE) { + set_input_f32("inp_mel", *params->mel_in); + set_input_f32("inp_sstft", *params->sstft_in); + break; + } + if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) { + const int T1 = (imgs.entries[0].nx() - 1) / 2 + 1; + const int T2 = (T1 - 1) / 2 + 1; + std::vector pos(T2); + for (int i = 0; i < T2; i++) { + pos[(size_t) i] = i; + } + set_input_i32("inp_pos", pos); + break; + } + if (params->gen_process != MTMD_SUBCTX_GEN_PROCESS_TTS) { + break; + } + const int n_gen = (int) params->codes->size(); + GGML_ASSERT(params->ref); + const int n_prompt = (int) params->ref->tokens.size(); + std::vector tokens = params->ref->tokens; + tokens.resize((size_t) n_prompt + n_gen); + memcpy(tokens.data() + n_prompt, params->codes->data(), (size_t) n_gen * sizeof(int32_t)); + const int64_t n_codes = model.cbx.input_embedding_w->ne[1]; + for (int32_t t : tokens) { + if (t < 0 || t >= n_codes) { + LOG_ERR("%s: speech token out of range (%d, codebook size %lld)\n", __func__, t, (long long) n_codes); + return false; + } + } + const int T1 = n_prompt + n_gen; + const int T2 = 2 * T1; + set_input_i32("inp_tokens", tokens); + + // mel-rate reference conditioning of the voice + set_input_f32("inp_prompt_feat", params->ref->feat); + set_input_f32("inp_xvec", params->ref->spk); + + // espnet relative positional encoding, entry k holds the sinusoid of relative position (T-1) - k + auto fill_pos = [&](const char * name, int T) { + const int d = 512; + std::vector pos((size_t) (2 * T - 1) * d); + for (int k = 0; k < 2 * T - 1; k++) { + const double rel = (double) (T - 1 - k); + for (int i = 0; i < d / 2; i++) { + const double div = exp(-(double) (2 * i) * log(10000.0) / d); + pos[(size_t) k * d + 2 * i ] = (float) sin(rel * div); + pos[(size_t) k * d + 2 * i + 1] = (float) cos(rel * div); + } + } + set_input_f32(name, pos); + }; + fill_pos("inp_pos1", T1); + fill_pos("inp_pos2", T2); + + // noise and the time embeddings of the solver span points, matcha layout (sines then cosines, scale 1000) + std::vector noise((size_t) 80 * T2); + std::normal_distribution nd(0.0f, 1.0f); + for (auto & f : noise) f = nd(params->gen_state->rng); + set_input_f32("inp_noise", noise); + + const bool meanflow = model.cbx.time_embed_mixer_w != nullptr; + const int n_steps = meanflow ? 2 : 10; + std::vector temb((size_t) 320 * (n_steps + 1)); + for (int s = 0; s <= n_steps; s++) { + const double u = (double) s / n_steps; + const double t = meanflow ? u : 1.0 - cos(u * M_PI / 2.0); + for (int i = 0; i < 160; i++) { + const double div = exp(-(double) i * log(10000.0) / 159.0); + temb[(size_t) s * 320 + i ] = (float) sin(1000.0 * t * div); + temb[(size_t) s * 320 + i + 160] = (float) cos(1000.0 * t * div); + } + } + set_input_f32("inp_temb", temb); + } break; + case PROJECTOR_TYPE_GEMMA3: + case PROJECTOR_TYPE_GEMMA3NV: + case PROJECTOR_TYPE_IDEFICS3: + case PROJECTOR_TYPE_COHERE2V: + case PROJECTOR_TYPE_INTERNVL: + case PROJECTOR_TYPE_NEMOTRON_V2_VL: + case PROJECTOR_TYPE_NEMOTRON_OMNI: + case PROJECTOR_TYPE_QWEN2A: + case PROJECTOR_TYPE_QWEN3A: + case PROJECTOR_TYPE_GLMA: + case PROJECTOR_TYPE_ULTRAVOX: + case PROJECTOR_TYPE_LFM2: + case PROJECTOR_TYPE_D1OMNI_V: + case PROJECTOR_TYPE_VOXTRAL: + case PROJECTOR_TYPE_MERALION: + case PROJECTOR_TYPE_MUSIC_FLAMINGO: + case PROJECTOR_TYPE_JANUS_PRO: + case PROJECTOR_TYPE_PHI4: + case PROJECTOR_TYPE_COGVLM: + case PROJECTOR_TYPE_YASA2: + case PROJECTOR_TYPE_GEMMA4UA: + case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + // do nothing + } break; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + { + // the ceil-mode seg pooling divides by the full segment length, rescale the last partial segment + const int T1 = (imgs.entries[0].nx() - 1) / 2 + 1; + const int S = (T1 + 99) / 100; + std::vector segfix((size_t) S, 1.0f); + const int last = T1 - (S - 1) * 100; + segfix[(size_t) S - 1] = 100.0f / (float) last; + set_input_f32("inp_segfix", segfix); + } break; + case PROJECTOR_TYPE_LFM3A_GEN: + { + const int64_t n_cb = model.gen_code_embd_w->ne[2]; + const int64_t n_vocab = model.gen_code_embd_w->ne[1]; + if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { + GGML_ASSERT(params->codes != nullptr); + // codes index the FSQ table via ggml_get_rows, the end-of-audio code has no waveform + const int64_t n_codes = model.c2w.quant_first_cb_w->ne[1]; + for (int32_t c : *params->codes) { + if (c < 0 || c >= n_codes) { + LOG_ERR("%s: code out of range (%d, codebook size %lld)\n", __func__, c, (long long) n_codes); + return false; + } + } + std::vector codes = *params->codes; + set_input_i32("inp_codes", codes); + set_gen_state_in(); + // the "pos" slot leads the state + float base = 0.0f; + ggml_backend_tensor_get(params->gen_state->slots[0], &base, 0, sizeof(float)); + set_lfm3a_window_inputs((int) base, (int) ggml_nelements(get_inp_tensor("positions")), hparams.wav_tfm_swa); + } else { + // rows of the merged per-codebook tables; a masked codebook (-1) takes row 0, the mask drops it + std::vector codes(n_cb, -1); + if (params->codes) { + GGML_ASSERT(params->codes->size() == (size_t) n_cb); + codes = *params->codes; + } + std::vector mask(n_cb, 0.0f); + for (int64_t i = 0; i < n_cb; i++) { + if (codes[i] >= n_vocab) { + LOG_ERR("%s: code out of range (%d, vocab %lld)\n", __func__, codes[i], (long long) n_vocab); + return false; + } + if (codes[i] < 0) { + codes[i] = 0; + } else { + mask[i] = 1.0f; + } + codes[i] += (int32_t) (i * n_vocab); + } + set_input_i32("inp_codes", codes); + if (ggml_graph_get_tensor(gf, "inp_mask") != nullptr) { + set_input_f32("inp_mask", mask); + } + } + } break; + case PROJECTOR_TYPE_DRAMA3_GEN: + { + const int64_t n_cb = model.gen_code_embd_w->ne[2]; + const int64_t n_vocab = model.gen_code_embd_w->ne[1]; + if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { + // frame-major to group-major, clamp: codec residual codebooks are smaller than the tower vocab + GGML_ASSERT(params->codes != nullptr && params->codes->size() % n_cb == 0); + const int64_t n_frames = (int64_t) params->codes->size() / n_cb; + const int64_t n_first = model.c2w.quant_first_cb_w->ne[1]; + const int64_t n_rest = model.c2w.quant_rest_cb_w->ne[1]; + std::vector codes(params->codes->size()); + for (int64_t f = 0; f < n_frames; f++) { + for (int64_t g = 0; g < n_cb; g++) { + const int64_t n = g == 0 ? n_first : n_rest; + codes[g * n_frames + f] = std::clamp((*params->codes)[f * n_cb + g], 0, (int32_t) n - 1); + } + } + set_input_i32("inp_codes", codes); + set_gen_state_in(); + } else { + // code0 < 0: next row is text (prompt), the tower only fills its KV + // codes: input frame codes for a speech row, empty for a text row + const bool next_speech = params->code0 >= 0; + const bool cur_speech = params->codes && (int64_t) params->codes->size() == n_cb; + if (params->code0 >= n_vocab) { + LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) n_vocab); + return false; + } + std::vector code0 = { std::max(params->code0, 0) }; + set_input_i32("inp_code0", code0); + + std::vector extra(n_cb - 2); + for (int64_t g = 2; g < n_cb; g++) { + const int32_t c = cur_speech ? std::clamp((*params->codes)[g], 0, (int32_t) n_vocab - 1) : 0; + extra[g - 2] = c + (int32_t) (g * n_vocab); + } + set_input_i32("inp_extra", extra); + set_input_f32("inp_scale", { next_speech ? 1.0f : 0.0f, cur_speech ? 1.0f : 0.0f }); + + float pos = 0.0f; + ggml_backend_tensor_get(params->gen_state->slots[0], &pos, 0, sizeof(float)); + std::vector inp_pos = { (int32_t) pos }; + set_input_i32("inp_pos", inp_pos); + + std::uniform_real_distribution dist(0.0f, 1.0f); + std::vector r(model.gen_code_head_w->ne[2]); + for (auto & v : r) { + v = dist(params->gen_state->rng); + } + set_input_f32("inp_rand", r); + } + } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { + GGML_ASSERT(params->codes != nullptr); + + // frame-major input to group-major, rear-padded with code 0 up to one window + const int64_t n_codes = model.gen_code_head_w->ne[2] + 1; + const int64_t n_frames_w = hparams.wav_tfm_swa; + const int64_t n_frames = (int64_t) params->codes->size() / n_codes; + GGML_ASSERT(n_frames > 0 && n_frames <= n_frames_w); + + // codes are used as ggml_get_rows indices, so check them against the codebook vocab + const int64_t vocab_first = model.c2w.quant_first_cb_w->ne[1]; + const int64_t vocab_rest = model.c2w.quant_rest_cb_w->ne[1]; + for (int64_t f = 0; f < n_frames; f++) { + for (int64_t g = 0; g < n_codes; g++) { + const int32_t c = (*params->codes)[f * n_codes + g]; + const int64_t vocab = (g == 0) ? vocab_first : vocab_rest; + if (c < 0 || (int64_t) c >= vocab) { + LOG_ERR("%s: code out of range (frame %lld, group %lld, code %d, vocab %lld)\n", + __func__, (long long) f, (long long) g, c, (long long) vocab); + return false; + } + } + } + + std::vector codes(n_frames_w * n_codes, 0); + for (int64_t f = 0; f < n_frames; f++) { + for (int64_t g = 0; g < n_codes; g++) { + codes[g * n_frames_w + f] = (*params->codes)[f * n_codes + g]; + } + } + set_input_i32("inp_codes", codes); + set_gen_state_in(); + } else { + // code0 indexes gen_code_out_embd_w via ggml_get_rows; bound it + const int64_t vocab0 = model.gen_code_out_embd_w->ne[1]; + if (params->code0 < 0 || (int64_t) params->code0 >= vocab0) { + LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) vocab0); + return false; + } + std::vector code0 = { params->code0 }; + set_input_i32("inp_code0", code0); + + // one uniform(0,1) draw per codebook, used by do_sampling() + std::uniform_real_distribution dist(0.0f, 1.0f); + const int64_t n_acoustic = model.gen_code_head_w->ne[2]; + for (int64_t g = 0; g < n_acoustic; g++) { + std::vector r = { dist(params->gen_state->rng) }; + set_input_f32(("inp_rand_" + std::to_string(g)).c_str(), r); + } + } + } break; + case PROJECTOR_TYPE_HUNYUANVL: + { + // Compute the HunyuanVL 2D position embedding on CPU (with the + // custom sf=(target+0.1)/n_grid bilinear sampling that the + // reference implementation uses) and upload it to the graph + // input declared in mtmd_graph_hunyuanvl::build(). + GGML_ASSERT(model.position_embeddings != nullptr); + ggml_tensor * src_t = model.position_embeddings; + const int64_t n_embd = src_t->ne[0]; + const int64_t n_pos = src_t->ne[1]; // = n_grid * n_grid + const int n_grid = (int)std::lround(std::sqrt((double)n_pos)); + GGML_ASSERT((int64_t)n_grid * n_grid == n_pos); + const int out_w = pos_w; // pw + const int out_h = pos_h; // ph + + // Pull weight to host. + std::vector src(n_embd * n_pos); + ggml_backend_tensor_get(src_t, src.data(), 0, ggml_nbytes(src_t)); + + // Output layout matches ggml_new_tensor_2d(F32, n_embd, out_h*out_w): + // ne[0] = n_embd (fastest), ne[1] = out_h*out_w + // dst[(y*out_w + x) * n_embd + c] + std::vector dst((size_t)n_embd * out_h * out_w); + + const float sx = (float)(out_w + 0.1f) / (float)n_grid; + const float sy = (float)(out_h + 0.1f) / (float)n_grid; + + for (int y = 0; y < out_h; ++y) { + // Match ggml_compute_forward_upscale_f32 pixel-center + // convention (align_corners=False): src_y = (y+0.5)/sy - 0.5. + const float fy = ((float)y + 0.5f) / sy - 0.5f; + int y0 = (int)std::floor(fy); + int y1 = y0 + 1; + y0 = std::clamp(y0, 0, n_grid - 1); + y1 = std::clamp(y1, 0, n_grid - 1); + float wy1 = std::clamp(fy - (float)y0, 0.0f, 1.0f); + const float wy0 = 1.0f - wy1; + for (int x = 0; x < out_w; ++x) { + const float fx = ((float)x + 0.5f) / sx - 0.5f; + int x0 = (int)std::floor(fx); + int x1 = x0 + 1; + x0 = std::clamp(x0, 0, n_grid - 1); + x1 = std::clamp(x1, 0, n_grid - 1); + float wx1 = std::clamp(fx - (float)x0, 0.0f, 1.0f); + const float wx0 = 1.0f - wx1; + + const float w00 = wy0 * wx0; + const float w01 = wy0 * wx1; + const float w10 = wy1 * wx0; + const float w11 = wy1 * wx1; + + const float * s00 = &src[((size_t)y0 * n_grid + x0) * n_embd]; + const float * s01 = &src[((size_t)y0 * n_grid + x1) * n_embd]; + const float * s10 = &src[((size_t)y1 * n_grid + x0) * n_embd]; + const float * s11 = &src[((size_t)y1 * n_grid + x1) * n_embd]; + float * d = &dst[((size_t)y * out_w + x) * n_embd]; + for (int c = 0; c < n_embd; ++c) { + d[c] = w00 * s00[c] + w01 * s01[c] + w10 * s10[c] + w11 * s11[c]; + } + } + } + + set_input_f32("hunyuanvl_pos_embd", dst); + } break; + case PROJECTOR_TYPE_LLAMA4: + { + // set the 2D positions + int n_patches_per_col = image_size_width / patch_size; + std::vector pos_data(num_patches + 1, 0); // +1 for the [CLS] token + // last pos is always kept 0, it's for CLS + // dimension H + for (int i = 0; i < num_patches; i++) { + pos_data[i] = (i / n_patches_per_col) + 1; + } + set_input_i32("pos_h", pos_data); + // dimension W + for (int i = 0; i < num_patches; i++) { + pos_data[i] = (i % n_patches_per_col) + 1; + } + set_input_i32("pos_w", pos_data); + } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + GGML_ASSERT(imgs.entries.size() == 1); + const int n_pos = (imgs.entries.front().nx() + 7) / 8; // 3x stride-2 conv2d + std::vector positions(n_pos); + for (int i = 0; i < n_pos; i++) { + positions[i] = i; + } + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_GEMMA4A: + { + GGML_ASSERT(imgs.entries.size() == 1); + const auto & img0 = imgs.entries.front(); + // Compute n_pos matching SSCP output: two stride-2 convs + int n_pos = img0.nx(); + for (int i = 0; i < 2; i++) { n_pos = (n_pos - 1) / 2 + 1; } + + // Chunked local attention: blocked causal mask and RPE + const int chunk_size = 12; + const int max_past = 12; + const int context_size = chunk_size + max_past; + const int num_blocks = (n_pos + chunk_size - 1) / chunk_size; + + // Blocked causal attention mask: [context_size, chunk_size, num_blocks] + { + std::vector mask(context_size * chunk_size * num_blocks, -1e9f); + for (int b = 0; b < num_blocks; b++) { + for (int q = 0; q < chunk_size; q++) { + int gq = b * chunk_size + q; + for (int k = 0; k < context_size; k++) { + int gk = b * chunk_size - max_past + k; + if (gq < n_pos && gk >= 0 && gk < n_pos && gk <= gq && (gq - gk) < max_past) { + mask[k + q * context_size + b * context_size * chunk_size] = 0.0f; + } + } + } + } + set_input_f32("kq_mask", mask); + } + + // Sinusoidal RPE: 13 positions [12, 11, ..., 0] + { + const int n_embd = model.hparams.n_embd; + const int num_timescales = n_embd / 2; + const float log_timescale_increment = logf(10000.0f) / std::max(num_timescales - 1, 1); + const int rpe_len = max_past + 1; + std::vector pos_emb(n_embd * rpe_len, 0.0f); + for (int p = 0; p < rpe_len; p++) { + float position = (float)(max_past - p); + for (int i = 0; i < num_timescales; i++) { + float inv_ts = expf(-(float)i * log_timescale_increment); + float scaled = position * inv_ts; + pos_emb[p * n_embd + i] = sinf(scaled); + pos_emb[p * n_embd + i + num_timescales] = cosf(scaled); + } + } + set_input_f32("pos_emb", pos_emb); + } + } break; + case PROJECTOR_TYPE_MIMO_AUDIO: + { + GGML_ASSERT(imgs.entries.size() == 1); + const int n_frames = imgs.entries.front().nx(); + const int n_pos = (n_frames - 1) / 2 + 1; // matches conv1(s=1)+conv2(s=2) output length + + std::vector positions(n_pos); + for (int i = 0; i < n_pos; i++) { + positions[i] = i; + } + set_input_i32("mimo_audio_positions", positions); + + const int window = hparams.attn_window_size; + GGML_ASSERT(window > 0); + + const float neg_inf = std::numeric_limits::lowest(); + std::vector full_mask((size_t) n_pos * n_pos); + std::vector window_mask((size_t) n_pos * n_pos); + for (int q = 0; q < n_pos; q++) { + for (int k = 0; k < n_pos; k++) { + const bool causal_ok = k <= q; + full_mask[(size_t) q * n_pos + k] = causal_ok ? 0.0f : neg_inf; + window_mask[(size_t) q * n_pos + k] = (causal_ok && (q - k) <= window) ? 0.0f : neg_inf; + } + } + set_input_attn_mask(full_mask, 0); + set_input_attn_mask(window_mask, 1); + + // input_local_transformer: block-diagonal mask + in-group positions + { + const int n_pos_ds = (n_pos - 2) / 2 + 1; // matches downsample conv (k=2,s=2,p=0) + const int group_size = hparams.audio_local_group_size; + GGML_ASSERT(group_size > 0); + const int n_groups = (n_pos_ds + group_size - 1) / group_size; + const int n_padded = n_groups * group_size; + + std::vector local_positions(n_padded); + for (int i = 0; i < n_padded; i++) { + local_positions[i] = i % group_size; + } + set_input_i32("mimo_audio_local_positions", local_positions); + + std::vector local_mask((size_t) n_padded * n_padded); + for (int q = 0; q < n_padded; q++) { + for (int k = 0; k < n_padded; k++) { + const bool same_group = (q / group_size) == (k / group_size); + local_mask[(size_t) q * n_padded + k] = same_group ? 0.0f : neg_inf; + } + } + set_input_attn_mask(local_mask, 2); + } + } break; + case PROJECTOR_TYPE_LFM2A: + case PROJECTOR_TYPE_D1OMNI_A: + { + GGML_ASSERT(imgs.entries.size() == 1); + const auto n_frames = model.n_output_tokens(&imgs.entries.front()); + + auto d_model = 512; + auto seq_len = n_frames * 2 - 1; + std::vector pos_emb(d_model*seq_len); + std::vector inv_freq(d_model / 2); + for (size_t i = 0; i < inv_freq.size(); ++i) { + inv_freq[i] = std::exp(-(std::log(10000.0) / (float)d_model) * (2.0f * (float)(i))); + } + for (int64_t pos = 0; pos < seq_len; ++pos) { + for (size_t i = 0; i < inv_freq.size(); ++i) { + const float ang = (n_frames - pos - 1) * inv_freq[i]; + pos_emb[pos*d_model + 2*i + 0] = sinf(ang); // even + pos_emb[pos*d_model + 2*i + 1] = cosf(ang); // odd + } + } + set_input_f32("pos_emb", pos_emb); + } break; + case PROJECTOR_TYPE_LFM3A: + { + GGML_ASSERT(imgs.entries.size() == 1); + const int n_pos = model.n_output_tokens(&imgs.entries.front()); + + // the "pos" slot leads the state; without a stream the state is zero, the clip is encoded from its start + int base = 0; + if (params->enc_state != nullptr) { + const auto & slots = params->enc_state->slots; + float base_f = 0.0f; + ggml_backend_tensor_get(slots[0], &base_f, 0, sizeof(float)); + base = (int) base_f; + size_t i = 0; + for (const auto & slot : list_lfm3a_state_slots(hparams, model)) { + ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str())); + } + } else { + for (const auto & slot : list_lfm3a_state_slots(hparams, model)) { + ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str()); + ggml_backend_tensor_memset(t, 0, 0, ggml_nbytes(t)); + } + } + + set_lfm3a_window_inputs(base, n_pos, hparams.attn_window_size); + } break; + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: + { + GGML_ASSERT(imgs.entries.size() == 1); + std::vector positions(ggml_nelements(get_inp_tensor("positions"))); + for (size_t i = 0; i < positions.size(); i++) { + positions[i] = (int32_t) i; + } + set_input_i32("positions", positions); + GGML_ASSERT(params->enc_state != nullptr); + set_input_f32("inp_norm", params->enc_state->mel_norm); + if (ggml_graph_get_tensor(gf, "spk_prev") != nullptr) { + set_input_f32("spk_prev", params->enc_state->spk_prev); + } + } break; + case PROJECTOR_TYPE_PARAKEET: + { + GGML_ASSERT(imgs.entries.size() == 1); + struct ggml_tensor * attn_mask = ggml_graph_get_tensor(gf, "attn_mask"); + const int n_q = attn_mask->ne[1]; + const int n_k = attn_mask->ne[0]; + const int n_frames = imgs.entries.front().nx(); + const int n_tokens_real = (n_frames + hparams.subsampling_factor-1) / hparams.subsampling_factor; + const float mask_value = -1e30f; + + std::vector mask_data(n_q * n_k); + if (n_k == n_q) { + // full attention: mask keys that are padding + for (int q = 0; q < n_q; ++q) { + for (int k = 0; k < n_k; ++k) { + mask_data[q * n_k + k] = (k >= n_tokens_real) ? mask_value : 0.0f; + } + } + } else { + // local attention: mask keys outside the valid window + const int att_left = n_k / 2; + for (int q = 0; q < n_q; ++q) { + for (int k = 0; k < n_k; ++k) { + const int key = q - att_left + k; + mask_data[q * n_k + k] = (key >= 0 && key < n_tokens_real) ? 0.0f : mask_value; + } + } + } + set_input_f32(attn_mask->name, mask_data); + + // local attention skew mask: zeroes out the probs that were + // computed for keys outside the valid sliding window. + if (struct ggml_tensor * local_mask = ggml_graph_get_tensor(gf, "local_mask")) { + const int lm_k = local_mask->ne[0]; + const int lm_q = local_mask->ne[1]; + const int window_size = lm_k - lm_q + 1; + std::vector lm_data(lm_q * lm_k); + for (int q = 0; q < lm_q; ++q) { + for (int k = 0; k < lm_k; ++k) { + const int rel = k - q; + lm_data[q * lm_k + k] = (rel >= 0 && rel < window_size) ? 1.0f : 0.0f; + } + } + set_input_f32(local_mask->name, lm_data); + } + + // Generate rotation frequencies for relative positional encoding. + { + const int n_state = hparams.n_embd; + const int d_half = n_state / 2; + const float log_10000 = logf(10000.0f); + std::vector freqs(d_half); + for (int k = 0; k < d_half; ++k) { + freqs[k] = expf(-(float(k * 2) * log_10000 / float(n_state))); + } + set_input_f32("pos_freqs", freqs); + } + + // Generate relative positional distance values which scaled by + // the frequency to produce the angles for sin/cos. + { + // window_size is only known after graph construction since it depends on + // n_time from the conv output, so we read it back from the graph tensor. + struct ggml_tensor * rel_pos = ggml_graph_get_tensor(gf, "rel_positions"); + const int window_size = rel_pos->ne[1]; + std::vector pos(window_size); + // local attention: window is fixed at [att_left, att_right] + // full attention: window covers the full sequence, centered + if (ggml_graph_get_tensor(gf, "local_mask")) { + const int att_left = window_size / 2; + for (int t = 0; t < window_size; ++t) { + pos[t] = float(att_left - t); + } + } else { + const int n_time = (window_size + 1) / 2; + for (int t = 0; t < window_size; ++t) { + pos[t] = float(n_time - 1 - t); + } + } + set_input_f32(rel_pos->name, pos); + } + } break; + case PROJECTOR_TYPE_GRANITE_SPEECH: + { + const int context_size = model.hparams.audio_chunk_size; + const int max_pos_emb = model.hparams.audio_max_pos_emb; + + std::vector dists((size_t) context_size * (size_t) context_size); + for (int i = 0; i < context_size; i++) { + for (int j = 0; j < context_size; j++) { + int d = i - j; + if (d < -context_size) d = -context_size; + if (d > context_size) d = context_size; + dists[(size_t) i * (size_t) context_size + (size_t) j] = d + max_pos_emb; + } + } + set_input_i32("attn_dists", dists); + + const int n_frames = image_size_width; + const int remainder = n_frames % context_size; + if (remainder > 0) { + const int num_blocks = (n_frames + context_size - 1) / context_size; + std::vector mask((size_t) context_size * (size_t) context_size * (size_t) num_blocks, 0.0f); + const float neg_inf = -INFINITY; + const size_t last_block_offset = (size_t) (num_blocks - 1) * (size_t) context_size * (size_t) context_size; + for (int q = 0; q < context_size; q++) { + for (int k = 0; k < context_size; k++) { + if (q >= remainder || k >= remainder) { + mask[last_block_offset + (size_t) q * (size_t) context_size + (size_t) k] = neg_inf; + } + } + } + set_input_f32("attn_mask", mask); + } + } break; + case PROJECTOR_TYPE_GRANITE4_VISION: + { + // Granite Vision 4.1 uses precomputed permutation index + // tensors to express the _win / _unwin / spatial sampling + // reshapes as ggml_get_rows gathers. The names are set + // by g4v_gather() in models/granite4-vision.cpp. + const int patch_size = model.hparams.patch_size; + const int image_side = imgs.entries.front().nx() / patch_size; + const int window_side = hparams.downsample_window_side; + const int query_side = hparams.downsample_query_side; + const int n = image_side / window_side; + const int new_side = n * query_side; + + // Builds the raster→window permutation indices for a + // (side, side) grid split into (n × n) windows of (win × win) + // tokens each. dst[w * win*win + p] = source raster index. + auto make_win_idx = [](int side, int win) { + const int nn = side / win; + std::vector idx(static_cast(side) * side); + for (int wy = 0; wy < nn; ++wy) { + for (int wx = 0; wx < nn; ++wx) { + for (int iy = 0; iy < win; ++iy) { + for (int ix = 0; ix < win; ++ix) { + const int w = wy * nn + wx; + const int p = iy * win + ix; + const int y = wy * win + iy; + const int x = wx * win + ix; + idx[static_cast(w) * (win*win) + p] = y * side + x; + } + } + } + } + return idx; + }; + + auto make_unwin_idx = [&](int side, int win) { + const std::vector fwd = make_win_idx(side, win); + std::vector inv(fwd.size()); + for (size_t i = 0; i < fwd.size(); ++i) { + inv[fwd[i]] = static_cast(i); + } + return inv; + }; + + auto make_spatial_idx = [](int side, int offset) { + const int off_y = (offset >> 1) & 1; + const int off_x = offset & 1; + const int new_s = side / 2; + std::vector idx(static_cast(new_s) * new_s); + for (int y = 0; y < new_s; ++y) { + for (int x = 0; x < new_s; ++x) { + idx[y * new_s + x] = (y * 2 + off_y) * side + (x * 2 + off_x); + } + } + return idx; + }; + + // the same permutation is applied to every tile of the stacked image + auto upload = [&](const std::string & name, const std::vector & idx) { + ggml_tensor * t = ggml_graph_get_tensor(gf, name.c_str()); + GGML_ASSERT(t); + GGML_ASSERT(ggml_nelements(t) % (int64_t) idx.size() == 0); + const int n_rep = ggml_nelements(t) / idx.size(); + std::vector buf; + buf.reserve(idx.size() * n_rep); + for (int i = 0; i < n_rep; ++i) { + buf.insert(buf.end(), idx.begin(), idx.end()); + } + ggml_backend_tensor_set(t, buf.data(), 0, ggml_nbytes(t)); + }; + + // Stage 1b only uses block 0's permutations; future stages + // will upload all blocks. + for (size_t bid = 0; bid < hparams.feature_layers.size(); ++bid) { + const std::string prefix = "g4v_blk" + std::to_string(bid) + "_"; + upload(prefix + "win_idx", make_win_idx(image_side, window_side)); + upload(prefix + "qwin_idx", make_win_idx(new_side, query_side)); + upload(prefix + "unwin_idx", make_unwin_idx(new_side, query_side)); + const auto spatial_offset = hparams.proj_spatial_offsets[bid]; + if (spatial_offset >= 0) { + upload(prefix + "spatial_idx", make_spatial_idx(image_side,spatial_offset)); + } + } + } break; + default: + GGML_ABORT("Unknown projector type"); + } + + mtmd_backend_cpu_set_n_threads(backend_cpu, params->n_threads); + + auto status = ggml_backend_sched_graph_compute(sched.get(), gf); + if (status != GGML_STATUS_SUCCESS) { + LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status); + return false; + } + + if (embeddings != nullptr) { + // sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one) + const int n_tokens_out = embeddings->ne[1]; + const int expected_n_tokens_out = model.n_output_tokens(&imgs.entries[0]); + if (n_tokens_out != expected_n_tokens_out) { + LOG_ERR("%s: expected output %d tokens, got %d\n", __func__, expected_n_tokens_out, n_tokens_out); + GGML_ABORT("Invalid number of output tokens"); + } + + LOG_DBG("%s: output embedding shape [%d, %d, %d]\n", __func__, + (int)embeddings->ne[0], (int)embeddings->ne[1], (int)embeddings->ne[2]); + + // copy output to user buffer if provided + // if output is empty, skip the copy + auto & out_batch_embd = *params->out_embd; + if (!out_batch_embd.empty()) { + if (out_batch_embd.size() != (size_t)ggml_nelements(embeddings)) { + LOG_ERR("%s: output buffer has %zu elements but expected %zu\n", __func__, out_batch_embd.size(), (size_t)ggml_nelements(embeddings)); + GGML_ABORT("Output buffer size mismatch"); + } + ggml_backend_tensor_get(embeddings, out_batch_embd.data(), 0, ggml_nbytes(embeddings)); + } else { + LOG_WRN("%s: output buffer is empty, skipping copy\n", __func__); + } + } + + // + // for audio gen models + // + + if (model.proj_type == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) { + ggml_tensor * fsq = ggml_graph_get_tensor(gf, "out_fsq"); + GGML_ASSERT(fsq != nullptr && params->out_codes); + const int n_tok = (int) fsq->ne[1]; + std::vector h((size_t) 8 * n_tok); + ggml_backend_tensor_get(fsq, h.data(), 0, ggml_nbytes(fsq)); + + // fsq round to base 3: h in (-1, 1) maps to digits {0, 1, 2} + params->out_codes->resize(n_tok); + for (int t = 0; t < n_tok; t++) { + int32_t code = 0; + for (int i = 7; i >= 0; i--) { + const int32_t d = (int32_t) roundf(h[(size_t) t * 8 + i] * 0.9990000128746033f) + 1; + code = code * 3 + d; + } + (*params->out_codes)[(size_t) t] = code; + } + + // speech table rows of the codes, the multilingual variant adds its learned positions as its perceiver expects + if (params->out_code_embd) { + const auto & tensors = model.cbx_tensors; + auto tab_it = tensors.find("a.gen.code.out_embd.weight"); + GGML_ASSERT(tab_it != tensors.end()); + ggml_tensor * tab = tab_it->second; + const int n_e = (int) tab->ne[0]; + auto pos_it = tensors.find("a.gen.t3.speech_pos_emb"); + ggml_tensor * pos = pos_it == tensors.end() ? nullptr : pos_it->second; + if ((tab->type != GGML_TYPE_F16 && tab->type != GGML_TYPE_F32) || + (pos && (pos->type != GGML_TYPE_F32 || pos->ne[0] != n_e || pos->ne[1] < n_tok))) { + LOG_ERR("%s: unexpected speech table or positions\n", __func__); + return false; + } + + params->out_code_embd->resize((size_t) n_tok * n_e); + std::vector h16(n_e); + std::vector pr(n_e); + for (int t = 0; t < n_tok; t++) { + float * dst = params->out_code_embd->data() + (size_t) t * n_e; + if ((*params->out_codes)[(size_t) t] >= tab->ne[1]) { + LOG_ERR("%s: speech code beyond the speech table\n", __func__); + return false; + } + const size_t r = (size_t) (*params->out_codes)[(size_t) t] * n_e; + if (tab->type == GGML_TYPE_F16) { + ggml_backend_tensor_get(tab, h16.data(), r * sizeof(ggml_fp16_t), (size_t) n_e * sizeof(ggml_fp16_t)); + for (int j = 0; j < n_e; j++) { + dst[j] = ggml_fp16_to_fp32(h16[j]); + } + } else { + ggml_backend_tensor_get(tab, dst, r * sizeof(float), (size_t) n_e * sizeof(float)); + } + if (pos) { + ggml_backend_tensor_get(pos, pr.data(), (size_t) t * n_e * sizeof(float), (size_t) n_e * sizeof(float)); + for (int j = 0; j < n_e; j++) { + dst[j] += pr[j]; + } + } + } + } + return true; + } + + if (model.proj_type == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE) { + ggml_tensor * sp = ggml_graph_get_tensor(gf, "out_spec"); + GGML_ASSERT(sp != nullptr && params->out_spec); + params->out_spec->resize(ggml_nelements(sp)); + ggml_backend_tensor_get(sp, params->out_spec->data(), 0, ggml_nbytes(sp)); + return true; + } + + if (model.proj_type == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS) { + ggml_tensor * mel = ggml_graph_get_tensor(gf, "out_mel"); + GGML_ASSERT(mel != nullptr); + std::vector mel_data(ggml_nelements(mel)); + ggml_backend_tensor_get(mel, mel_data.data(), 0, ggml_nbytes(mel)); + + // hift: f0 -> harmonic source -> source stft on the host, then the vocoder graph and the istft + ggml_tensor * f0_t = ggml_graph_get_tensor(gf, "out_f0"); + GGML_ASSERT(f0_t != nullptr); + const int n_mel_out = (int) ggml_nelements(f0_t); + std::vector f0(n_mel_out); + ggml_backend_tensor_get(f0_t, f0.data(), 0, ggml_nbytes(f0_t)); + + // SineGen: f0 upsampled x480, 9 harmonics with cumulative phase, uv gating and noise, merged by l_linear + tanh + const int ups_total = 480; + const double sr = 24000.0; + const int64_t n_wav = (int64_t) n_mel_out * ups_total; + std::vector lw(9); + float lb = 0.0f; + if (model.cbx_read_tensor("a.gen.hift.m_source.l_linear.weight", lw.data(), lw.size()) != 9 || + model.cbx_read_tensor("a.gen.hift.m_source.l_linear.bias", &lb, 1) != 1) { + LOG_ERR("%s: unexpected hift source module\n", __func__); + return false; + } + + std::mt19937 & srng = params->gen_state->rng; + std::uniform_real_distribution ud(-M_PI, M_PI); + std::normal_distribution snd(0.0f, 1.0f); + double phase[9]; + for (int h = 0; h < 9; h++) { + phase[h] = h == 0 ? 0.0 : ud(srng); + } + std::vector src((size_t) n_wav); + double cum[9] = {0.0}; + for (int64_t t = 0; t < n_wav; t++) { + const float f = f0[(size_t) (t / ups_total)]; + const float uv = f > 10.0f ? 1.0f : 0.0f; // nsf_voiced_threshold + const float namp = uv * 0.003f + (1.0f - uv) * 0.1f / 3.0f; + float merged = lb; + for (int h = 0; h < 9; h++) { + cum[h] += (double) f * (h + 1) / sr; + cum[h] -= floor(cum[h]); + float sine = 0.1f * (float) sin(2.0 * M_PI * cum[h] + phase[h]); + sine = sine * uv + namp * snd(srng); + merged += lw[(size_t) h] * sine; + } + src[(size_t) t] = tanhf(merged); + } + + // stft of the source: n_fft 16, hop 4, hann window, centered + const int n_fft = 16, hop = 4, n_bins = 9; + std::vector win(n_fft); + for (int i = 0; i < n_fft; i++) win[(size_t) i] = 0.5f - 0.5f * cosf(2.0f * (float) M_PI * i / n_fft); + const int n_stft = (int) (n_wav / hop) + 1; + std::vector sstft((size_t) n_stft * 18); + for (int fr = 0; fr < n_stft; fr++) { + const int64_t c0 = (int64_t) fr * hop - n_fft / 2; // centered, reflect padded + for (int k = 0; k < n_bins; k++) { + double re = 0.0, im = 0.0; + for (int i = 0; i < n_fft; i++) { + int64_t idx = c0 + i; + if (idx < 0) idx = -idx; + if (idx >= n_wav) idx = 2 * (n_wav - 1) - idx; + const double v = (double) src[(size_t) idx] * win[(size_t) i]; + const double a = 2.0 * M_PI * k * i / n_fft; + re += v * cos(a); + im -= v * sin(a); + } + sstft[(size_t) fr * 18 + k ] = (float) re; + sstft[(size_t) fr * 18 + 9 + k] = (float) im; + } + } + + // vocoder graph on mel + source stft + std::vector spec; + { + mtmd_subctx_encode_params vp = *params; + vp.gen_process = MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE; + vp.mel_in = &mel_data; + vp.sstft_in = &sstft; + vp.vocode_n_mel = n_mel_out; + vp.vocode_n_stft = n_stft; + vp.out_audio = nullptr; + vp.out_spec = &spec; + if (!encode(&vp)) { + LOG_ERR("%s: vocoder stage failed\n", __func__); + return false; + } + } + + // istft: mag = clipped exp, phase = sin, hann overlap-add + const int n_frames_out = (int) (spec.size() / 18); + const int64_t n_out = (int64_t) (n_frames_out - 1) * hop; + std::vector acc((size_t) n_out + n_fft, 0.0); + std::vector wsum((size_t) n_out + n_fft, 0.0); + for (int fr = 0; fr < n_frames_out; fr++) { + double frame[16]; + for (int i = 0; i < n_fft; i++) { + double v = 0.0; + for (int k = 0; k < n_bins; k++) { + const double mag = fmin(exp((double) spec[(size_t) fr * 18 + k]), 1e2); + const double ph = sin((double) spec[(size_t) fr * 18 + 9 + k]); + const double re = mag * cos(ph), im = mag * sin(ph); + const double a = 2.0 * M_PI * k * i / n_fft; + const double w = (k == 0 || k == n_fft / 2) ? 1.0 : 2.0; + v += w * (re * cos(a) - im * sin(a)); + } + frame[i] = v / n_fft; + } + const int64_t o = (int64_t) fr * hop; + for (int i = 0; i < n_fft; i++) { + acc [(size_t) (o + i)] += frame[i] * win[(size_t) i]; + wsum[(size_t) (o + i)] += (double) win[(size_t) i] * win[(size_t) i]; + } + } + GGML_ASSERT(params->out_audio); + auto & out_audio = *params->out_audio; + out_audio.resize((size_t) std::min(n_out - n_fft / 2, n_wav)); + for (size_t i = 0; i < out_audio.size(); i++) { + const size_t j = i + n_fft / 2; // drop the centering pad + const double v = wsum[j] > 1e-11 ? acc[j] / wsum[j] : 0.0; + out_audio[(size_t) i] = (float) fmax(-0.99, fmin(0.99, v)); + } + + // trim_fade of the reference: silence 20 ms then fade in 20 ms, hides the artifact of the prompt boundary + const size_t n_trim = 24000 / 50; + for (size_t i = 0; i < 2 * n_trim && i < out_audio.size(); i++) { + const double g = i < n_trim ? 0.0 + : (cos(M_PI * (1.0 - (double) (i - n_trim) / n_trim)) + 1.0) / 2.0; + out_audio[(size_t) i] *= (float) g; + } + return true; + } + + // optional outputs: a pipeline yields codes or feats, and not all have an eos head + if (params->out_codes != nullptr) { + ggml_tensor * codes = ggml_graph_get_tensor(gf, "out_codes"); + if (codes != nullptr) { + auto & out_codes = *params->out_codes; + out_codes.resize(ggml_nelements(codes)); + ggml_backend_tensor_get(codes, out_codes.data(), 0, ggml_nbytes(codes)); + } + } + if (params->out_feats != nullptr) { + ggml_tensor * feats = ggml_graph_get_tensor(gf, "out_feats"); + if (feats != nullptr) { + auto & out_feats = *params->out_feats; + out_feats.resize(ggml_nelements(feats)); + ggml_backend_tensor_get(feats, out_feats.data(), 0, ggml_nbytes(feats)); + } + } + if (params->out_is_eos != nullptr) { + ggml_tensor * eos = ggml_graph_get_tensor(gf, "out_eos_score"); + if (eos != nullptr) { + GGML_ASSERT(ggml_nelements(eos) == 1); + float score = 0.0f; + ggml_backend_tensor_get(eos, &score, 0, sizeof(float)); + *params->out_is_eos = score > hparams.gen_eos_threshold; + } + } + if (params->out_audio != nullptr) { + ggml_tensor * audio = ggml_graph_get_tensor(gf, "out_audio"); + if (audio == nullptr) { + GGML_ABORT("out_audio requested but graph has no \"out_audio\" tensor"); + } + auto & out_audio = *params->out_audio; + out_audio.resize(ggml_nelements(audio)); + ggml_backend_tensor_get(audio, out_audio.data(), 0, ggml_nbytes(audio)); + + if (model.proj_type == PROJECTOR_TYPE_SOPRANO) { + std::vector spectrum = std::move(out_audio); + mtmd_graph_soprano::decode_spectrum(model, spectrum, out_audio); + } + + // drop the tail audio that comes from the code-0 rear padding, drama3 has no padding + if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN) { + const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0; + const int64_t n_frames_w = hparams.wav_tfm_swa; + const int64_t n_frames = params->codes ? (int64_t) params->codes->size() / n_codes : n_frames_w; + if (n_frames < n_frames_w) { + const size_t hop = out_audio.size() / n_frames_w; + out_audio.resize((size_t) n_frames * hop); + } + } + } + if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_LFM3A) { + const auto & slots = params->enc_state->slots; + size_t i = 0; + for (const auto & slot : list_lfm3a_state_slots(hparams, model)) { + ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str()); + GGML_ASSERT(t != nullptr); + copy_gen_state_out(t, slots[i++]); + } + } else if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) { + mtmd_subctx_enc_state * st = params->enc_state; + ggml_tensor * chunk = ggml_graph_get_tensor(gf, "spk_chunk"); + ggml_tensor * preds = ggml_graph_get_tensor(gf, "spk_preds"); + GGML_ASSERT(chunk != nullptr && preds != nullptr); + st->spk_chunk.resize(ggml_nelements(chunk)); + st->spk_preds.resize(ggml_nelements(preds)); + ggml_backend_tensor_get(chunk, st->spk_chunk.data(), 0, ggml_nbytes(chunk)); + ggml_backend_tensor_get(preds, st->spk_preds.data(), 0, ggml_nbytes(preds)); + } else if (params->enc_state != nullptr) { + mtmd_subctx_enc_state * st = params->enc_state; + ggml_tensor * tail = ggml_graph_get_tensor(gf, "out_stem_tail"); + GGML_ASSERT(tail != nullptr); + st->stem_tail.resize(ggml_nelements(tail)); + ggml_backend_tensor_get(tail, st->stem_tail.data(), 0, ggml_nbytes(tail)); + + const int n_new = (imgs.entries[0].nx() - 2) / 2; + for (int i = 0; i < n_new; i++) { + st->slot_pos[(st->n_past + i) % st->n_ctx] = st->n_past + i; + } + st->n_past += n_new; + } + + if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { + const auto & slots = params->gen_state->slots; + size_t i = 0; + for (const auto & slot : list_gen_state_slots(hparams, model)) { + if (slot.in_place) { + i++; + continue; + } + ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str()); + if (t == nullptr) { + GGML_ABORT("graph has no \"state_out_%s\" tensor", slot.name.c_str()); + } + GGML_ASSERT(i < slots.size()); + copy_gen_state_out(t, slots[i++]); + } + } + + // + // Debug: dump final embeddings if MTMD_DEBUG_EMBEDDINGS is set + // + + if (debug_output_embeddings && embeddings != nullptr) { + const int64_t n_embd = embeddings->ne[0]; + const int64_t n_tokens = embeddings->ne[1]; + std::vector emb_data(ggml_nelements(embeddings)); + ggml_backend_tensor_get(embeddings, emb_data.data(), 0, ggml_nbytes(embeddings)); + + LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n"); + LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens); + + // TEMP debugging (parity validation), will be removed before merge + // when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data] + const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS"); + if (dump_path && strcmp(dump_path, "1") != 0) { + FILE * f = fopen(dump_path, "wb"); + if (f) { + const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd }; + fwrite(hdr, sizeof(hdr), 1, f); + fwrite(emb_data.data(), sizeof(float), emb_data.size(), f); + fclose(f); + } + } + + // Print first few values of first token + LOG_INF("Token 0 (first 16 values): "); + for (int i = 0; i < std::min((int64_t)16, n_embd); i++) { + LOG_INF("%.6f ", emb_data[i]); + } + LOG_INF("\n"); + + // Print last few values of first token + if (n_embd > 16) { + LOG_INF("Token 0 (last 16 values): "); + for (int64_t i = n_embd - 16; i < n_embd; i++) { + LOG_INF("%.6f ", emb_data[i]); + } + LOG_INF("\n"); + } + + // Compute and print statistics + float sum = 0.0f, sum_sq = 0.0f, min_val = emb_data[0], max_val = emb_data[0]; + for (size_t i = 0; i < emb_data.size(); i++) { + sum += emb_data[i]; + sum_sq += emb_data[i] * emb_data[i]; + min_val = std::min(min_val, emb_data[i]); + max_val = std::max(max_val, emb_data[i]); + } + float mean = sum / emb_data.size(); + float variance = (sum_sq / emb_data.size()) - (mean * mean); + LOG_INF("Stats: mean=%.6f, std=%.6f, min=%.6f, max=%.6f, sum=%.6f\n", + mean, sqrtf(variance), min_val, max_val, sum); + LOG_INF("=== END MTMD_DEBUG_EMBEDDINGS ===\n\n"); + } + + return true; +} + +std::map mtmd_subcontext::get_mem_usage() const { + std::map result = model.mem_weights; + for (auto & [dev, size] : mem_compute) { + result[dev] += size; + } + return result; +} diff --git a/tools/mtmd/mtmd-subcontext.h b/tools/mtmd/mtmd-subcontext.h new file mode 100644 index 000000000..d757c9b6e --- /dev/null +++ b/tools/mtmd/mtmd-subcontext.h @@ -0,0 +1,202 @@ +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" +#include "ggml-cpp.h" +#include "mtmd-impl.h" +#include "mtmd-model.h" + +#include +#include + +#include +#include +#include +#include + +// !!! Internal header, to be used by mtmd only !!! + +// a subcontext runs the graphs of one mtmd_model (one modality) +// it owns the backends and the compute buffers, the weights are owned by the model + +enum mtmd_flash_attn_type { + MTMD_FLASH_ATTN_TYPE_AUTO = -1, + MTMD_FLASH_ATTN_TYPE_DISABLED = 0, + MTMD_FLASH_ATTN_TYPE_ENABLED = 1, +}; + +struct mtmd_subctx_params { + ggml_backend_dev_t device; // nullptr means CPU, see mtmd_select_device() + std::shared_ptr threadpool; // CPU threadpool shared by the subcontexts, can be nullptr + enum mtmd_flash_attn_type flash_attn_type; + bool warmup; + ggml_backend_sched_eval_callback cb_eval; + void * cb_eval_user_data; +}; + +// the device the weights and the graphs are placed on, nullptr means CPU +ggml_backend_dev_t mtmd_select_device(bool use_gpu, ggml_backend_dev_t device); + +// nullptr if n_threads < 2 or the CPU backend has no threadpool support +std::shared_ptr mtmd_threadpool_init(int n_threads); + +// compute the log-mel of n_frames frames, the waveform must already be padded and window must hold one whole frame +// out has the layout [n_frames, n_mel]; like the encoders, must not run at the same time as another pass on config.sched +struct mtmd_audio_mel_config; +bool mtmd_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out); + +enum mtmd_subctx_gen_process { + MTMD_SUBCTX_GEN_PROCESS_GEN_UNKNOWN, + MTMD_SUBCTX_GEN_PROCESS_GEN_CODE, // h_state to codes + MTMD_SUBCTX_GEN_PROCESS_GEN_WAV, // codes to raw PCM audio + MTMD_SUBCTX_GEN_PROCESS_TTS, // full utterance of codes to raw PCM audio + MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE, // internal: mel + source stft to istft input + MTMD_SUBCTX_GEN_PROCESS_TOKENIZE, // raw PCM audio to semantic speech tokens +}; + +// persistent state of a causal audio encoder, for streaming input (one per stream) +// it holds the sliding window of past keys/values, so each encode() call only takes the new frames +struct mtmd_subctx_enc_state { + int32_t n_ctx = 0; // ring slots: attention window + frames per call + int32_t n_past = 0; // absolute position of the next frame + + std::vector slot_pos; // absolute position held by each slot, -1 if empty + std::vector stem_tail; // last conv stem output frame, empty on a cold start + + ggml_context_ptr ctx_data; + ggml_backend_buffer_ptr buf; + std::vector k; // per layer [n_embd_kv, n_ctx] + std::vector v; + + // lfm3a: same order as list_lfm3a_state_slots(), F32 [ne0, ne1], updated in place + std::vector slots; + + // nemotron transcribe: the window's normalized log-mel, cache and FIFO rows in; its speaker rows and predictions out + std::vector mel_norm; + std::vector spk_prev; + std::vector spk_chunk, spk_preds; +}; + +// per-sequence state of audio generation: RNG and the slots kept between GEN_WAV calls +struct mtmd_subctx_gen_state { + std::mt19937 rng; + + ggml_context_ptr ctx_data; + ggml_backend_buffer_ptr buf; + std::vector slots; // same order as list_gen_state_slots(), F32 [ne0, ne1] + + // clears the slots and keeps the RNG stream + void reset() { + if (buf) { + ggml_backend_buffer_clear(buf.get(), 0); + } + } +}; + +struct mtmd_subctx_encode_params { + int n_threads = 1; + const mtmd_image_f32_batch * imgs = nullptr; + std::vector * out_embd = nullptr; + + // for audio gen, imgs has exactly one entry: hidden state from backbone (GEN_CODE) or unused (GEN_WAV) + mtmd_subctx_gen_process gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_UNKNOWN; + + // GEN_CODE: out_embd receives the embd to feed back to the backbone + int32_t code0 = 0; // semantic code sampled by the backbone + int32_t top_k = 50; + float top_p = 1.0f; + std::vector * out_codes = nullptr; // this frame's 16 sampled codes + std::vector * out_feats = nullptr; // continuous counterpart of out_codes + float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders + bool * out_is_eos = nullptr; + + // TOKENIZE: speech embedding rows of the produced codes + std::vector * out_code_embd = nullptr; + + // GEN_WAV + const std::vector * codes = nullptr; // this frame's 16 RVQ codes + const std::vector * feats = nullptr; // continuous counterpart of codes + std::vector * out_audio = nullptr; // decoded PCM samples, F32 + const mtmd_gen_ref * ref = nullptr; // TTS reference conditioning of the voice + // TTS_VOCODE internal stage inputs + const std::vector * mel_in = nullptr; + const std::vector * sstft_in = nullptr; + int vocode_n_mel = 0; + int vocode_n_stft = 0; + std::vector * out_spec = nullptr; + + // audio gen: state of one sequence, encode() updates it in place + struct mtmd_subctx_gen_state * gen_state = nullptr; + + // streaming audio input, updated in place by encode() + struct mtmd_subctx_enc_state * enc_state = nullptr; +}; + +struct mtmd_subcontext { + const mtmd_model & model; + + std::vector buf_compute_meta; + + std::vector backend_ptrs; + std::vector backend_buft; + + std::shared_ptr threadpool; // shared by the subcontexts of a model + + ggml_backend_t backend = nullptr; + ggml_backend_t backend_cpu = nullptr; + + int max_nodes = 8192; + ggml_backend_sched_ptr sched; + mtmd_flash_attn_type flash_attn_type = MTMD_FLASH_ATTN_TYPE_AUTO; + bool is_allocated = false; + + bool debug_output_embeddings = false; + + // for measuring memory usage + std::map mem_compute; + + bool support_batch = false; + + // the model must outlive the subcontext + mtmd_subcontext(const mtmd_model & model, const mtmd_subctx_params & params); + ~mtmd_subcontext(); + + // reserve the compute buffers for the given batch + void warmup(const mtmd_image_f32_batch & batch); + + bool encode(mtmd_subctx_encode_params * params); + + // TODO: remove encode_image() and always use batched version + bool encode_image(int n_threads, const mtmd_image_f32 * img, std::vector & out_vec); + bool encode_batch(int n_threads, const mtmd_image_f32_batch * imgs, std::vector & out_batch_embd); + + // n_frames_max is the maximum number of encoder frames per encode() call, free the result with delete + mtmd_subctx_enc_state * enc_state_init(int32_t n_frames_max); + + // seed UINT32_MAX means random, free the result with delete + mtmd_subctx_gen_state * gen_state_init(uint32_t seed); + + // weights and compute buffers + std::map get_mem_usage() const; + +private: + struct support_info_op { + ggml_tensor * op; + + // true if the op runs on the accelerated backend + bool is_accel = true; + }; + + struct support_info_graph { + // whether the backend supports flash attention + bool fattn = true; + ggml_tensor * fattn_op = nullptr; // for debugging + + std::vector ops; + }; + + mtmd_image_f32_batch get_dummy_batch() const; + + // only initialize backend buffers, but do not allocate them yet + support_info_graph reserve_compute_meta(const mtmd_image_f32_batch & batch); +}; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 538f90013..6b4fc4d0a 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1,5 +1,5 @@ -#include "clip.h" -#include "clip-impl.h" +#include "mtmd-subcontext.h" +#include "mtmd-impl.h" #include "mtmd.h" #include "mtmd-internal.h" #include "mtmd-audio.h" @@ -106,10 +106,10 @@ std::string mtmd_serialization::read() { return str; } -// only mtmd.cpp needs these, so they're implemented here rather than in clip-impl.h -void clip_image_f32::serialize(mtmd_serialization & ser) const { +// only mtmd.cpp needs these, so they're implemented here rather than in mtmd-impl.h +void mtmd_image_f32::serialize(mtmd_serialization & ser) const { // remember to bump MTMD_SERIALIZATION_VERSION if this is changed - // note: buf is intentionally NOT serialized; the loaded clip_image_f32 will always be a placeholder + // note: buf is intentionally NOT serialized; the loaded mtmd_image_f32 will always be a placeholder ser.write(add_viewsep); ser.write(add_newline); ser.write(lead_pad); @@ -121,16 +121,16 @@ void clip_image_f32::serialize(mtmd_serialization & ser) const { ser.write((int32_t)nx_); ser.write((int32_t)ny_); } -void clip_image_f32::deserialize(mtmd_serialization & ser) { +void mtmd_image_f32::deserialize(mtmd_serialization & ser) { add_viewsep = ser.read(); add_newline = ser.read(); lead_pad = ser.read(); if (ser.version >= 3) { const int32_t suffix_raw = ser.read(); - if (suffix_raw < 0 || suffix_raw >= CLIP_SUFFIX_COUNT) { + if (suffix_raw < 0 || suffix_raw >= MTMD_SUFFIX_COUNT) { throw std::runtime_error("invalid suffix type"); } - suffix_type = (clip_suffix_type)suffix_raw; + suffix_type = (mtmd_suffix_type)suffix_raw; anyres.grid_x = ser.read(); anyres.grid_y = ser.read(); anyres.orig_nx = ser.read(); @@ -141,7 +141,7 @@ void clip_image_f32::deserialize(mtmd_serialization & ser) { buf.clear(); // always a placeholder after loading } -void clip_image_f32_batch::serialize(mtmd_serialization & ser) const { +void mtmd_image_f32_batch::serialize(mtmd_serialization & ser) const { // remember to bump MTMD_SERIALIZATION_VERSION if this is changed ser.write(is_audio); ser.write(entries.size()); @@ -149,7 +149,7 @@ void clip_image_f32_batch::serialize(mtmd_serialization & ser) const { entry.serialize(ser); } } -void clip_image_f32_batch::deserialize(mtmd_serialization & ser) { +void mtmd_image_f32_batch::deserialize(mtmd_serialization & ser) { is_audio = ser.read(); uint64_t n = ser.read(); constexpr size_t min_entry_bytes = sizeof(uint8_t) * 2 + sizeof(int32_t) * 2; @@ -159,7 +159,7 @@ void clip_image_f32_batch::deserialize(mtmd_serialization & ser) { entries.clear(); entries.reserve(n); for (uint64_t i = 0; i < n; i++) { - clip_image_f32 entry; + mtmd_image_f32 entry; entry.deserialize(ser); entries.push_back(std::move(entry)); } @@ -176,7 +176,7 @@ struct mtmd_bitmap { std::string id; // optional user-defined id, for ex: can be set to image hash, useful for KV cache tracking bool is_audio = false; // true if the bitmap is audio bool is_voice = false; - std::shared_ptr gen_ref; // gen-side reference of a predefined voice, if any + std::shared_ptr gen_ref; // gen-side reference of a predefined voice, if any bool mergeable = false; // [QWEN_VIDEO] set only on frames of the same video bool is_video = false; // see mtmd_bitmap_set_is_video() @@ -202,13 +202,13 @@ struct mtmd_bitmap { } } - static mtmd_bitmap * init_voice(const std::string & name, const clip_gen_voice & voice, uint32_t n_embd) { + static mtmd_bitmap * init_voice(const std::string & name, const mtmd_gen_voice & voice, uint32_t n_embd) { const float * rows = voice.rows.data(); const uint32_t n_rows = voice.rows.size() / n_embd; auto * bitmap = new mtmd_bitmap(nullptr, n_embd, n_rows); bitmap->is_voice = true; if (!voice.ref.tokens.empty()) { - bitmap->gen_ref = std::make_shared(voice.ref); + bitmap->gen_ref = std::make_shared(voice.ref); } bitmap->id = name; bitmap->data.assign((const unsigned char *) rows, (const unsigned char *) (rows + (size_t) n_embd * n_rows)); @@ -268,7 +268,7 @@ struct mtmd_image_tokens { uint32_t n = 0; for (size_t k = 0; k < batch_f32.entries.size(); ++k) { const auto [gw, gh] = canvas_tile_grid(k); - n += gw * gh + (uint32_t) clip_suffix_rows(batch_f32.entries[k].suffix_type).size(); + n += gw * gh + (uint32_t) mtmd_suffix_rows(batch_f32.entries[k].suffix_type).size(); } return n; } @@ -284,7 +284,7 @@ struct mtmd_image_tokens { } return nx * ny * nz; } - clip_image_f32_batch batch_f32; // preprocessed image patches + mtmd_image_f32_batch batch_f32; // preprocessed image patches std::string id; // optional user-defined ID, useful for KV cache tracking // true if one of entries in batch_f32 is a placeholder @@ -354,16 +354,16 @@ using mtmd_audio_source_ptr = std::shared_ptr; // an audio track split across chunks (e.g. the soundtrack of a video): it is encoded once, as one clip, and every // chunk takes its rows of it struct mtmd_audio_track { - clip_image_f32_batch batch_f32; // the mel of the whole track + mtmd_image_f32_batch batch_f32; // the mel of the whole track mtmd_audio_source_ptr source; uint32_t n_tokens = 0; std::vector embd; // filled by the first encode - std::shared_ptr gen_ref; // gen-side reference of a predefined voice, if any + std::shared_ptr gen_ref; // gen-side reference of a predefined voice, if any }; struct mtmd_audio_tokens { uint32_t n_tokens = 0; // number of tokens - clip_image_f32_batch batch_f32; // preprocessed image patches + mtmd_image_f32_batch batch_f32; // preprocessed image patches std::string id; // optional user-defined ID, useful for KV cache tracking mtmd_audio_source_ptr source; // if set, batch_f32 has the shape of the mel of chunk source_idx @@ -554,13 +554,13 @@ const char * mtmd_default_marker() { return "<__media__>"; } -static clip_flash_attn_type mtmd_get_clip_flash_attn_type(enum llama_flash_attn_type flash_attn_type) { +static mtmd_flash_attn_type mtmd_to_flash_attn_type(enum llama_flash_attn_type flash_attn_type) { switch (flash_attn_type) { - case LLAMA_FLASH_ATTN_TYPE_AUTO: return CLIP_FLASH_ATTN_TYPE_AUTO; - case LLAMA_FLASH_ATTN_TYPE_DISABLED: return CLIP_FLASH_ATTN_TYPE_DISABLED; - case LLAMA_FLASH_ATTN_TYPE_ENABLED: return CLIP_FLASH_ATTN_TYPE_ENABLED; + case LLAMA_FLASH_ATTN_TYPE_AUTO: return MTMD_FLASH_ATTN_TYPE_AUTO; + case LLAMA_FLASH_ATTN_TYPE_DISABLED: return MTMD_FLASH_ATTN_TYPE_DISABLED; + case LLAMA_FLASH_ATTN_TYPE_ENABLED: return MTMD_FLASH_ATTN_TYPE_ENABLED; } - return CLIP_FLASH_ATTN_TYPE_AUTO; + return MTMD_FLASH_ATTN_TYPE_AUTO; } mtmd_context_params mtmd_context_params_default() { @@ -585,20 +585,23 @@ mtmd_context_params mtmd_context_params_default() { } // chatterbox multilingual has a perceiver in its conditioning encoder (speaker encoding context) -static bool cbx_is_mtl(clip_ctx * ctx_a) { - return clip_cbx_read_tensor(ctx_a, "a.cenc.perceiver.pre_attention_query", nullptr, 0) > 0; +static bool cbx_is_mtl(const mtmd_model & model) { + return model.cbx_read_tensor("a.cenc.perceiver.pre_attention_query", nullptr, 0) > 0; } struct mtmd_context { - struct clip_ctx * ctx_v; // vision - struct clip_ctx * ctx_a; // audio + // must outlive the subcontexts below + mtmd_models models; + + std::unique_ptr ctx_v; // vision + std::unique_ptr ctx_a; // audio std::vector out_embd; // image embedding vector // generation context - struct clip_ctx * ctx_gen_a; // audio + std::unique_ptr ctx_gen_a; // audio // gen-side reference data of the last encoded chunk (chatterbox), see mtmd_gen_state_set_ref() - clip_gen_ref gen_ref; + mtmd_gen_ref gen_ref; bool print_timings; int n_threads; @@ -683,34 +686,48 @@ struct mtmd_context { } } - clip_context_params ctx_clip_params { - /* use_gpu */ ctx_params.use_gpu, - /* device */ ctx_params.device, - /* flash_attn_type */ mtmd_get_clip_flash_attn_type(ctx_params.flash_attn_type), - /* image_min_tokens */ ctx_params.image_min_tokens, - /* image_max_tokens */ ctx_params.image_max_tokens, + ggml_backend_dev_t device = mtmd_select_device(ctx_params.use_gpu, ctx_params.device); + + mtmd_model_params model_params; + model_params.image_min_tokens = ctx_params.image_min_tokens; + model_params.image_max_tokens = ctx_params.image_max_tokens; + model_params.no_alloc = no_alloc; + model_params.progress_callback = ctx_params.progress_callback; + model_params.progress_callback_user_data = ctx_params.progress_callback_user_data; + + ggml_backend_buffer_type_t buft = device + ? ggml_backend_dev_buffer_type(device) + : ggml_backend_dev_buffer_type(ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU)); + models = mtmd_model_load(mmproj_fname, model_params, buft); + + // soprano and pocket-tts without voice cloning have no encoder + if (!models.v && !models.a && !models.gen_a) { + throw std::runtime_error(string_format("Failed to load mmproj model from %s\n", mmproj_fname)); + } + + mtmd_subctx_params sctx_params { + /* device */ device, + // no_alloc only measures memory, nothing is computed + /* threadpool */ mtmd_threadpool_init(no_alloc ? 1 : ctx_params.n_threads), + /* flash_attn_type */ mtmd_to_flash_attn_type(ctx_params.flash_attn_type), /* warmup */ ctx_params.warmup, /* cb_eval */ ctx_params.cb_eval, /* cb_eval_user_data */ ctx_params.cb_eval_user_data, - /* no_alloc */ no_alloc, - /* progress_callback */ ctx_params.progress_callback, - /* progress_callback_user_data */ ctx_params.progress_callback_user_data, - /* n_threads */ ctx_params.n_threads, }; - - auto res = clip_init(mmproj_fname, ctx_clip_params); - ctx_v = res.ctx_v; - ctx_a = res.ctx_a; - ctx_gen_a = res.ctx_gen_a; - // soprano and pocket-tts without voice cloning have no encoder - if (!ctx_v && !ctx_a && !ctx_gen_a) { - throw std::runtime_error(string_format("Failed to load CLIP model from %s\n", mmproj_fname)); + if (models.v) { + ctx_v = std::make_unique(*models.v, sctx_params); + } + if (models.a) { + ctx_a = std::make_unique(*models.a, sctx_params); + } + if (models.gen_a) { + ctx_gen_a = std::make_unique(*models.gen_a, sctx_params); } // if both vision and audio mmproj are present, we need to validate their n_embd if (ctx_v && ctx_a) { - int n_embd_v = clip_n_mmproj_embd(ctx_v); - int n_embd_a = clip_n_mmproj_embd(ctx_a); + int n_embd_v = ctx_v->model.n_mmproj_embd(); + int n_embd_a = ctx_a->model.n_mmproj_embd(); if (n_embd_v != n_embd_a) { throw std::runtime_error(string_format( "mismatch between vision and audio mmproj (n_embd_v = %d, n_embd_a = %d)\n", @@ -721,15 +738,15 @@ struct mtmd_context { // since we already validate n_embd of vision and audio mmproj, // we can safely assume that they are the same // a gen-only mmproj is validated below - int n_embd_clip = ctx_v || ctx_a ? clip_n_mmproj_embd(ctx_v ? ctx_v : ctx_a) : n_embd_text; - if (n_embd_text > 0 && n_embd_text != n_embd_clip) { + int n_embd_mmproj = ctx_v || ctx_a ? (ctx_v ? ctx_v : ctx_a)->model.n_mmproj_embd() : n_embd_text; + if (n_embd_text > 0 && n_embd_text != n_embd_mmproj) { throw std::runtime_error(string_format( "mismatch between text model (n_embd = %d) and mmproj (n_embd = %d)\n" "hint: you may be using wrong mmproj\n", - n_embd_text, n_embd_clip)); + n_embd_text, n_embd_mmproj)); } if (ctx_gen_a) { - int n_embd_gen = clip_n_mmproj_embd(ctx_gen_a); + int n_embd_gen = ctx_gen_a->model.n_mmproj_embd(); if (n_embd_text > 0 && n_embd_text != n_embd_gen) { throw std::runtime_error(string_format( "mismatch between text model (n_embd = %d) and gen-audio mmproj (n_embd = %d)\n" @@ -749,7 +766,7 @@ struct mtmd_context { GGML_ASSERT(ctx_v != nullptr); image_preproc.reset(); - projector_type proj = clip_get_projector_type(ctx_v); + projector_type proj = ctx_v->model.proj_type; switch (proj) { case PROJECTOR_TYPE_MLP: @@ -760,16 +777,16 @@ struct mtmd_context { case PROJECTOR_TYPE_JANUS_PRO: case PROJECTOR_TYPE_GLM_EDGE: { - bool has_pinpoints = !clip_get_hparams(ctx_v)->image_res_candidates.empty(); + bool has_pinpoints = !ctx_v->model.hparams.image_res_candidates.empty(); if (has_pinpoints) { - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } else { - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } } break; case PROJECTOR_TYPE_MINICPMV: { - int minicpmv_version = clip_get_hparams(ctx_v)->minicpmv_version; + int minicpmv_version = ctx_v->model.hparams.minicpmv_version; if (minicpmv_version == 2) { // minicpmv 2.5 format: // (overview) (slice) (slice) \n ... @@ -798,7 +815,7 @@ struct mtmd_context { } else if (minicpmv_version != 0) { throw std::runtime_error(string_format("unsupported minicpmv version: %d\n", minicpmv_version)); } - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: @@ -813,10 +830,10 @@ struct mtmd_context { tok_row_end_trail = false; // no trailing end-of-row token ov_img_first = true; use_image_id = true; - if (clip_get_projector_type(ctx_v) == PROJECTOR_TYPE_MINICPM_OCR_1_0) { - image_preproc = std::make_unique(ctx_v); + if (ctx_v->model.proj_type == PROJECTOR_TYPE_MINICPM_OCR_1_0) { + image_preproc = std::make_unique(models.v.get()); } else { - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } } break; case PROJECTOR_TYPE_QWEN3VL: @@ -824,7 +841,7 @@ struct mtmd_context { // <|vision_start|> ... (image embeddings) ... <|vision_end|> img_beg = "<|vision_start|>"; img_end = "<|vision_end|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: @@ -835,26 +852,26 @@ struct mtmd_context { // <|vision_start|> ... (image embeddings) ... <|vision_end|> img_beg = "<|vision_start|>"; img_end = "<|vision_end|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_MINIMAX_M3: { // ]<]start of image[>[ ... (image embeddings) ... ]<]end of image[>[ img_beg = "]<]start of image[>["; img_end = "]<]end of image[>["; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_SENSENOVA_U1: { img_beg = ""; img_end = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_MUSE_GLIMMER: { img_beg = "<|image_start|>"; img_end = "<|image_end|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_LFM3VL: { @@ -863,14 +880,14 @@ struct mtmd_context { img_end = "<|vision_end|>"; vid_beg = img_beg; vid_end = img_end; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_YOUTUVL: { // <|vision_start|> ... (image embeddings) ... <|vision_end|> img_beg = "<|vision_start|>"; img_end = "<|vision_end|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_YASA2: { @@ -883,7 +900,7 @@ struct mtmd_context { // images. This will be implemented in a future PR (dispatch on has_pinpoints // - see LDP/COGVLM branch above) and emit image_grid_pinpoints in the conversion // script. - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_GEMMA3: case PROJECTOR_TYPE_GEMMA3NV: @@ -891,7 +908,7 @@ struct mtmd_context { // ... (image embeddings) ... img_beg = ""; img_end = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_IDEFICS3: { @@ -901,22 +918,22 @@ struct mtmd_context { tok_ov_img_end = {lookup_token("")}; tok_row_end = {lookup_token("\n")}; sli_img_start_tmpl = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_PIXTRAL: { // https://github.com/huggingface/transformers/blob/1cd110c6cb6a6237614130c470e9a902dbc1a4bd/docs/source/en/model_doc/pixtral.md img_end = "[IMG_END]"; - if (clip_get_hparams(ctx_v)->image_longest_edge > 0) { - image_preproc = std::make_unique(ctx_v); + if (ctx_v->model.hparams.image_longest_edge > 0) { + image_preproc = std::make_unique(models.v.get()); } else { - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } } break; case PROJECTOR_TYPE_PHI4: { // Phi-4 uses media marker insertion only. Keep image boundary text empty. - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_LLAMA4: { @@ -925,7 +942,7 @@ struct mtmd_context { img_end = "<|image_end|>"; LOG_WRN("%s: llama 4 vision is known to have degraded quality:\n" " https://github.com/ggml-org/llama.cpp/pull/13282\n", __func__); - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); ov_img_first = false; } break; case PROJECTOR_TYPE_STEP3VL: @@ -942,14 +959,14 @@ struct mtmd_context { tok_row_end = {lookup_token("")}; tok_row_end_trail = false; ov_img_first = false; // patches first, overview last - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_INTERNVL: { // ... (image embeddings) ... img_beg = ""; img_end = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); ov_img_first = false; } break; case PROJECTOR_TYPE_COHERE2V: @@ -960,14 +977,14 @@ struct mtmd_context { tok_sli_img_end = {lookup_token("<|IMG_LINE_BREAK|>")}; tok_ov_img_end = tok_sli_img_end; ov_img_first = false; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_KIMIVL: { // <|media_start|> ... (image embeddings) ... <|media_end|> img_beg = "<|media_start|>"; img_end = "<|media_end|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_KIMIK25: { @@ -982,24 +999,24 @@ struct mtmd_context { img_beg = "<|media_begin|>"; img_end = "<|media_end|>"; } - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_LIGHTONOCR: { // <|im_start|> ... (image embeddings) ... <|im_end|> img_beg = "<|im_start|>"; img_end = "<|im_end|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_DEEPSEEK4V: { // no vocab tokens are added; the start/end/newline markers are learned embeddings emitted by the encoder - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_DEEPSEEK41V: { // as deepseek4v: markers come from the encoder, the block is decoded causally - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_DOTS_OCR: case PROJECTOR_TYPE_DOTS3NOTE_V: @@ -1007,17 +1024,17 @@ struct mtmd_context { // <|img|> ... (image embeddings) ... <|endofimg|> img_beg = "<|img|>"; img_end = "<|endofimg|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_NEMOTRON_V2_VL: { - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_NEMOTRON_OMNI: { img_beg = ""; img_end = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_LFM2: { @@ -1034,33 +1051,33 @@ struct mtmd_context { sli_img_start_tmpl = "<|img_row_%d_col_%d|>"; tok_ov_img_start = {lookup_token("<|img_thumbnail|>")}; ov_img_first = false; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_D1OMNI_V: { // same tiles and thumbnail as lfm2, without separator tokens - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_GLM4V: { // <|begin_of_image|> ... (image embeddings) ... <|end_of_image|> img_beg = "<|begin_of_image|>"; img_end = "<|end_of_image|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_GLM5V: { // <|begin_of_image|> ... (image embeddings) ... <|end_of_image|> img_beg = "<|begin_of_image|>"; img_end = "<|end_of_image|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_PADDLEOCR: { // <|IMAGE_START|> ... (image embeddings) ... <|IMAGE_END|> img_beg = "<|IMAGE_START|>"; img_end = "<|IMAGE_END|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_GEMMA4V: case PROJECTOR_TYPE_GEMMA4UV: @@ -1068,13 +1085,13 @@ struct mtmd_context { // <|image> ... (image embeddings) ... img_beg = "<|image>"; img_end = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_DEEPSEEKOCR: case PROJECTOR_TYPE_DEEPSEEKOCR2: { img_end = "\n"; // prevent empty batch on llama-server - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); ov_img_first = false; } break; case PROJECTOR_TYPE_HUNYUANVL: @@ -1082,21 +1099,21 @@ struct mtmd_context { // note: these use fullwidth | (U+FF5C) and ▁ (U+2581) to match the tokenizer vocabulary img_beg = "<|hy_place▁holder▁no▁100|>"; img_end = "<|hy_place▁holder▁no▁101|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_EXAONE4_5: { // ... (image embeddings) ... img_beg = ""; img_end = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_GRANITE4_VISION: { // ... (image embeddings) \n ... img_beg = ""; img_end = "\n"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; default: throw std::runtime_error(string_format("%s: unexpected vision projector type %d\n", __func__, proj)); @@ -1109,7 +1126,7 @@ struct mtmd_context { GGML_ASSERT(ctx_a != nullptr); audio_preproc.reset(); - projector_type proj = clip_get_projector_type(ctx_a); + projector_type proj = ctx_a->model.proj_type; LOG_WRN("%s: audio input is in experimental stage and may have reduced quality:\n" " https://github.com/ggml-org/llama.cpp/discussions/13759\n", __func__); @@ -1122,31 +1139,31 @@ struct mtmd_context { // <|audio_bos|> ... (embeddings) ... <|audio_eos|> aud_beg = "<|audio_bos|>"; aud_end = "<|audio_eos|>"; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: { aud_beg = "<|soft_audio_start|>"; aud_end = "<|soft_audio_end|>"; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_QWEN3A: { aud_beg = "<|audio_start|>"; aud_end = "<|audio_end|>"; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_VOXTRAL: { // [BEGIN_AUDIO] ... (embeddings) ... aud_beg = "[BEGIN_AUDIO]"; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_MUSIC_FLAMINGO: { // ... (embeddings) ... aud_beg = ""; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_VOXTRAL_RT: { @@ -1156,15 +1173,15 @@ struct mtmd_context { case PROJECTOR_TYPE_GLMA: case PROJECTOR_TYPE_MERALION: { - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_LFM2A: { - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_D1OMNI_A: { - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_LFM3A: { @@ -1173,56 +1190,56 @@ struct mtmd_context { aud_beg = img_beg; aud_end = img_end; } - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_GRANITE_SPEECH: { - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_GEMMA4A: { aud_beg = "<|audio>"; aud_end = ""; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_PARAKEET: { - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: { - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_GEMMA4UA: { aud_beg = "<|audio>"; aud_end = ""; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_DOTS3NOTE_A: { // <|audio_comp_start|> ... (embeddings) ... <|audio_comp_end|> aud_beg = "<|audio_comp_start|>"; aud_end = "<|audio_comp_end|>"; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_MIMO_AUDIO: { aud_beg = "<|mimo_audio_start|>"; aud_end = "<|mimo_audio_end|>"; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: { - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_POCKETTTS_SPKENC: { - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_CHATTERBOX_SPKENC: { - audio_preproc = std::make_unique(ctx_a, cbx_is_mtl(ctx_a)); + audio_preproc = std::make_unique(models.a.get(), cbx_is_mtl(*models.a)); } break; default: throw std::runtime_error(string_format("%s: unexpected audio projector type %d\n", __func__, proj)); @@ -1231,26 +1248,27 @@ struct mtmd_context { // initialize audio preprocessor if (audio_preproc) { audio_preproc->mel_config.n_threads = n_threads; + audio_preproc->mel_config.sched = ctx_a->sched.get(); audio_preproc->initialize(); } } - // get clip ctx based on chunk type - clip_ctx * get_clip_ctx(const mtmd_input_chunk * chunk) const { + // get the subcontext based on chunk type + mtmd_subcontext * get_subctx(const mtmd_input_chunk * chunk) const { if (chunk->type == MTMD_INPUT_CHUNK_TYPE_IMAGE) { - return ctx_v; + return ctx_v.get(); } else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_AUDIO) { - return ctx_a; + return ctx_a.get(); } GGML_ABORT("unknown chunk type"); } projector_type proj_type_v() const { - return ctx_v ? clip_get_projector_type(ctx_v) : PROJECTOR_TYPE_UNKNOWN; + return ctx_v ? ctx_v->model.proj_type : PROJECTOR_TYPE_UNKNOWN; } projector_type proj_type_a() const { - return ctx_a ? clip_get_projector_type(ctx_a) : PROJECTOR_TYPE_UNKNOWN; + return ctx_a ? ctx_a->model.proj_type : PROJECTOR_TYPE_UNKNOWN; } // [LFM3_OMNI] the audio of a video is its soundtrack, see MTMD_VIDEO_TEMPLATE_LFM3_OMNI @@ -1260,20 +1278,14 @@ struct mtmd_context { int64_t n_embd_out() const { if (ctx_v) { - return clip_n_mmproj_embd(ctx_v); + return ctx_v->model.n_mmproj_embd(); } else if (ctx_a) { - return clip_n_mmproj_embd(ctx_a); + return ctx_a->model.n_mmproj_embd(); } else { - throw std::runtime_error("no CLIP model loaded"); + throw std::runtime_error("no mmproj model loaded"); } } - ~mtmd_context() { - clip_free(ctx_a); - clip_free(ctx_v); - clip_free(ctx_gen_a); - } - private: llama_token lookup_token(const std::string & token_text) { if (vocab == nullptr) { @@ -1498,7 +1510,7 @@ struct mtmd_tokenizer { // [QWEN_VIDEO] handle frame merging for models that support it (i.e. qwen-vl) int n_merge_frames = 1; if (ctx->ctx_v) { - n_merge_frames = clip_model_n_temporal_merge(ctx->ctx_v); + n_merge_frames = ctx->ctx_v->model.n_temporal_merge(); GGML_ASSERT(n_merge_frames <= 2 && "we only support merging maximum 2 images for now; open an issue if this model supports merging more"); } @@ -1624,8 +1636,8 @@ struct mtmd_tokenizer { return 2; } - // convert mtmd_bitmap to clip_image_u8 - clip_image_u8 img_u8; + // convert mtmd_bitmap to mtmd_image_u8 + mtmd_image_u8 img_u8; img_u8.set_size( {(int)bmp->nx, (int)bmp->ny}, bmp->is_placeholder()); @@ -1667,7 +1679,7 @@ struct mtmd_tokenizer { add_text("" + std::to_string(n_images_added) + "", true); } add_text(ctx->tok_ov_img_start); - // the separators after are appended by clip, see add_canvas_chunk() + // the separators after are appended by the graph, see add_canvas_chunk() add_canvas_chunk(std::move(preproc_out), bitmaps[0]->id); } else if (has_tiling_grid) { // [QWEN_VIDEO] we do not support "frame merging" for llama-uhd style, so no batching for now @@ -1762,8 +1774,8 @@ struct mtmd_tokenizer { size_t n_tokens = 0; for (auto & e : preproc_out.entries) { - n_tokens += clip_n_output_tokens(ctx->ctx_v, &e); - if (clip_model_n_temporal_merge(ctx->ctx_v) == 2) { + n_tokens += ctx->ctx_v->model.n_output_tokens(&e); + if (ctx->ctx_v->model.n_temporal_merge() == 2) { // [QWEN_VIDEO] pair input is merged to the same embd, so only count as one image break; } @@ -1772,12 +1784,12 @@ struct mtmd_tokenizer { mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens); // [QWEN_VIDEO] improve this in the future - image_tokens->n_temporal_merge = clip_model_n_temporal_merge(ctx->ctx_v); + image_tokens->n_temporal_merge = ctx->ctx_v->model.n_temporal_merge(); if (mtmd_decode_use_mrope(ctx)) { // for Qwen2VL, we need this information for M-RoPE decoding positions - image_tokens->nx = clip_n_output_tokens_x(ctx->ctx_v, &preproc_out.entries[0]); - image_tokens->ny = clip_n_output_tokens_y(ctx->ctx_v, &preproc_out.entries[0]); + image_tokens->nx = ctx->ctx_v->model.n_output_tokens_x(&preproc_out.entries[0]); + image_tokens->ny = ctx->ctx_v->model.n_output_tokens_y(&preproc_out.entries[0]); } else { // other models, we only need the total number of tokens image_tokens->nx = n_tokens; @@ -1798,7 +1810,7 @@ struct mtmd_tokenizer { GGML_ASSERT(n_tokens == (size_t)image_tokens->n_tokens()); } - clip_image_f32_batch batch_f32; + mtmd_image_f32_batch batch_f32; batch_f32.is_audio = false; batch_f32.is_video = bitmaps[0]->is_video_frame(); batch_f32.entries = std::move(preproc_out.entries); @@ -1887,13 +1899,13 @@ struct mtmd_tokenizer { } // one chunk for the five entries of a chatterbox clip, one token per talker conditioning row - if (clip_get_projector_type(ctx->ctx_a) == PROJECTOR_TYPE_CHATTERBOX_SPKENC) { + if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_CHATTERBOX_SPKENC) { GGML_ASSERT(source->entries.size() == 5); // mtl: spkr, perceiver query rows, emotion; turbo: spkr, t3 speech tokens - const size_t n_query = clip_cbx_read_tensor(ctx->ctx_a, "a.cenc.perceiver.pre_attention_query", nullptr, 0) / ctx->n_embd_out(); - const size_t n_tokens = cbx_is_mtl(ctx->ctx_a) ? 2 + n_query : 1 + (size_t) source->entries[2].nx() / 4; + const size_t n_query = ctx->ctx_a->model.cbx_read_tensor("a.cenc.perceiver.pre_attention_query", nullptr, 0) / ctx->n_embd_out(); + const size_t n_tokens = cbx_is_mtl(ctx->ctx_a->model) ? 2 + n_query : 1 + (size_t) source->entries[2].nx() / 4; - clip_image_f32_batch batch_f32; + mtmd_image_f32_batch batch_f32; batch_f32.is_audio = true; batch_f32.entries = std::move(source->entries); @@ -1913,11 +1925,11 @@ struct mtmd_tokenizer { // consider each mel_spec as a separate audio chunk // TODO: maybe support batching, but this may come with memory cost for (size_t i = 0; i < source->entries.size(); i++) { - clip_image_f32 mel_f32 = source->entries[i]; + mtmd_image_f32 mel_f32 = source->entries[i]; - size_t n_tokens = clip_n_output_tokens(ctx->ctx_a, &mel_f32); + size_t n_tokens = ctx->ctx_a->model.n_output_tokens(&mel_f32); - clip_image_f32_batch batch_f32; + mtmd_image_f32_batch batch_f32; batch_f32.is_audio = true; batch_f32.entries.push_back(std::move(mel_f32)); @@ -1995,9 +2007,9 @@ struct mtmd_tokenizer { LOG_ERR("%s: unable to preprocess the soundtrack\n", __func__); return 2; } - clip_image_f32 mel_f32 = source->entries[0]; + mtmd_image_f32 mel_f32 = source->entries[0]; st.track = std::make_shared(); - st.track->n_tokens = clip_n_output_tokens(ctx->ctx_a, &mel_f32); + st.track->n_tokens = ctx->ctx_a->model.n_output_tokens(&mel_f32); if (source->needs_compute()) { st.track->source = source; } @@ -2006,7 +2018,7 @@ struct mtmd_tokenizer { } // rows of [begin, end): round(sample / samples per row), the last piece takes the rest of the track - const double samples_per_row = 8.0 * clip_get_hparams(ctx->ctx_a)->audio_hop_len; // one row per 8 mel frames + const double samples_per_row = 8.0 * ctx->ctx_a->model.hparams.audio_hop_len; // one row per 8 mel frames const uint32_t n_tokens = st.track->n_tokens; st.n_samples += bitmap.nx; const uint32_t begin = st.n_assigned; @@ -2031,7 +2043,7 @@ struct mtmd_tokenizer { return 0; } - // MiniCPM-V 4.7: the overview and all slices go in one chunk, clip appends the separators after each tile: + // MiniCPM-V 4.7: the overview and all slices go in one chunk, the graph appends the separators after each tile: // [ov] [S00] [S01] \n [S10] [S11] void add_canvas_chunk(mtmd_image_preproc_out && preproc_out, const std::string & id) { const int n_col = preproc_out.grid_x; @@ -2041,9 +2053,9 @@ struct mtmd_tokenizer { GGML_ASSERT((int) slices.size() == n_col * n_row); auto & ov = preproc_out.overview; - ov.suffix_type = CLIP_SUFFIX_MINICPMV_OV; + ov.suffix_type = MTMD_SUFFIX_MINICPMV_OV; if (!slices.empty()) { - ov.suffix_type = CLIP_SUFFIX_MINICPMV_OV_SLICE; + ov.suffix_type = MTMD_SUFFIX_MINICPMV_OV_SLICE; ov.anyres.grid_x = n_col; ov.anyres.grid_y = n_row; } @@ -2051,11 +2063,11 @@ struct mtmd_tokenizer { for (int x = 0; x < n_col; x++) { auto & suffix = slices[y * n_col + x].suffix_type; if (y == n_row - 1 && x == n_col - 1) { - suffix = CLIP_SUFFIX_MINICPMV_LAST; + suffix = MTMD_SUFFIX_MINICPMV_LAST; } else if (x == n_col - 1) { - suffix = CLIP_SUFFIX_MINICPMV_ROW_END; + suffix = MTMD_SUFFIX_MINICPMV_ROW_END; } else { - suffix = CLIP_SUFFIX_MINICPMV_SLICE; + suffix = MTMD_SUFFIX_MINICPMV_SLICE; } } } @@ -2069,12 +2081,12 @@ struct mtmd_tokenizer { entries.push_back(std::move(slice)); } // token grid of the last entry, the grids of the other entries are scaled from it - image_tokens->nx = clip_n_output_tokens_x(ctx->ctx_v, &entries.back()); - image_tokens->ny = clip_n_output_tokens_y(ctx->ctx_v, &entries.back()); + image_tokens->nx = ctx->ctx_v->model.n_output_tokens_x(&entries.back()); + image_tokens->ny = ctx->ctx_v->model.n_output_tokens_y(&entries.back()); size_t n_tokens = 0; for (const auto & entry : entries) { - n_tokens += clip_n_output_tokens(ctx->ctx_v, &entry); + n_tokens += ctx->ctx_v->model.n_output_tokens(&entry); } GGML_ASSERT(n_tokens == image_tokens->n_tokens()); @@ -2090,9 +2102,9 @@ struct mtmd_tokenizer { std::vector split_batch_to_chunk(mtmd_image_preproc_out && preproc_out, const std::string & id) { std::vector chunks; - auto process_chunk = [&](clip_image_f32 && img) { + auto process_chunk = [&](mtmd_image_f32 && img) { mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens); - image_tokens->nx = clip_n_output_tokens(ctx->ctx_v, &img); + image_tokens->nx = ctx->ctx_v->model.n_output_tokens(&img); image_tokens->ny = 1; image_tokens->batch_f32.entries.push_back(std::move(img)); image_tokens->id = id; @@ -2213,8 +2225,8 @@ int32_t mtmd_tokenize_from_parts(const mtmd_context * ctx, } static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * image_tokens, std::vector & out_embd) { - clip_ctx * ctx_clip = ctx->ctx_v; - if (!ctx_clip) { + mtmd_subcontext * sctx = ctx->ctx_v.get(); + if (!sctx) { LOG_ERR("%s: this API does not support non-vision input, please use mtmd_encode_chunk instead\n", __func__); return 1; } @@ -2232,10 +2244,10 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im // the tiles differ in size, encode them one by one size_t offset = 0; for (const auto & entry : image_tokens->batch_f32.entries) { - clip_image_f32_batch one; + mtmd_image_f32_batch one; one.entries.push_back(entry); - std::vector embd((size_t) n_embd_out * clip_n_output_tokens(ctx_clip, &entry)); - if (!clip_image_batch_encode(ctx_clip, ctx->n_threads, &one, embd)) { + std::vector embd((size_t) n_embd_out * sctx->model.n_output_tokens(&entry)); + if (!sctx->encode_batch(ctx->n_threads, &one, embd)) { return 1; } GGML_ASSERT(offset + embd.size() <= out_embd.size()); @@ -2246,9 +2258,7 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im return 0; } - bool ok = clip_image_batch_encode( - ctx_clip, - ctx->n_threads, + bool ok = sctx->encode_batch(ctx->n_threads, &image_tokens->batch_f32, out_embd); @@ -2257,8 +2267,8 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im static int32_t cbx_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector & out_embd); -static bool mtmd_audio_batch_encode(mtmd_context * ctx, const clip_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector & out_embd) { - clip_image_f32_batch batch = batch_f32.clone(); +static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector & out_embd) { + mtmd_image_f32_batch batch = batch_f32.clone(); if (source) { if (!ctx->audio_preproc->compute(*source)) { LOG_ERR("%s: unable to compute the mel\n", __func__); @@ -2267,7 +2277,7 @@ static bool mtmd_audio_batch_encode(mtmd_context * ctx, const clip_image_f32_bat GGML_ASSERT(idx < source->entries.size()); batch.entries[0] = source->entries[idx]; } - return clip_image_batch_encode(ctx->ctx_a, ctx->n_threads, &batch, out_embd); + return ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd); } static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk * chunk, std::vector & out_embd) { @@ -2321,7 +2331,7 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk } return 0; } - if (clip_get_projector_type(ctx->ctx_a) == PROJECTOR_TYPE_CHATTERBOX_SPKENC) { + if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_CHATTERBOX_SPKENC) { return cbx_ref_encode(ctx, chunk->tokens_audio.get(), out_embd); } out_embd.resize((size_t)chunk->tokens_audio->n_tokens * ctx->n_embd_out()); @@ -2367,9 +2377,9 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { info.type = MTMD_GEN_AUDIO_TYPE_NONE; return info; } - info.model_variant = clip_get_hparams(ctx->ctx_gen_a)->gen_model_variant.c_str(); - info.n_voices = (int32_t) clip_gen_get_voices(ctx->ctx_gen_a).size(); - switch (clip_get_projector_type(ctx->ctx_gen_a)) { + info.model_variant = ctx->ctx_gen_a->model.hparams.gen_model_variant.c_str(); + info.n_voices = (int32_t) ctx->ctx_gen_a->model.gen_voices.size(); + switch (ctx->ctx_gen_a->model.proj_type) { case PROJECTOR_TYPE_QWEN3TTS_GEN: info.type = MTMD_GEN_AUDIO_TYPE_QWEN3TTS; info.sample_rate = 24000; @@ -2405,19 +2415,19 @@ size_t mtmd_gen_audio_read_tensor(mtmd_context * ctx, const char * name, float * if (!ctx->ctx_gen_a) { return 0; } - return clip_cbx_read_tensor(ctx->ctx_gen_a, name, out, n_max); + return ctx->ctx_gen_a->model.cbx_read_tensor(name, out, n_max); } // raw 192-dim CAMPPlus x-vector of the fbank entry -static bool cbx_ref_xvec(mtmd_context * ctx, const clip_image_f32 & fbank, std::vector & xvec) { - clip_image_f32 mel_img; +static bool cbx_ref_xvec(mtmd_context * ctx, const mtmd_image_f32 & fbank, std::vector & xvec) { + mtmd_image_f32 mel_img; mel_img.set_size(fbank.get_size(), false, true); mel_img.cpy_buf(fbank.get_ro_buf()); - clip_image_f32_batch batch; + mtmd_image_f32_batch batch; batch.is_audio = true; batch.entries.push_back(std::move(mel_img)); xvec.resize(192); - if (!clip_image_batch_encode(ctx->ctx_a, ctx->n_threads, &batch, xvec)) { + if (!ctx->ctx_a->encode_batch(ctx->n_threads, &batch, xvec)) { LOG_ERR("%s: speaker encoder failed\n", __func__); return false; } @@ -2425,28 +2435,28 @@ static bool cbx_ref_xvec(mtmd_context * ctx, const clip_image_f32 & fbank, std:: } // s3 speech tokens of a log-mel entry, rows optionally receives their speech embedding rows -static int32_t cbx_ref_tokenize(mtmd_context * ctx, const clip_image_f32 & mel, +static int32_t cbx_ref_tokenize(mtmd_context * ctx, const mtmd_image_f32 & mel, std::vector & codes, std::vector * rows) { - clip_image_f32 mel_img; + mtmd_image_f32 mel_img; mel_img.set_size(mel.get_size(), false, true); mel_img.cpy_buf(mel.get_ro_buf()); - clip_image_f32_batch batch; + mtmd_image_f32_batch batch; batch.is_audio = true; batch.entries.push_back(std::move(mel_img)); std::vector out_codes; std::vector out_embd; - clip_encode_params params; + mtmd_subctx_encode_params params; params.imgs = &batch; params.n_threads = ctx->n_threads; - params.gen_process = CLIP_GEN_PROCESS_TOKENIZE; + params.gen_process = MTMD_SUBCTX_GEN_PROCESS_TOKENIZE; params.out_codes = &out_codes; params.out_code_embd = rows ? &out_embd : nullptr; - if (!clip_encode(ctx->ctx_gen_a, ¶ms)) { - LOG_ERR("%s: clip_encode failed (tokenize)\n", __func__); + if (!ctx->ctx_gen_a->encode(¶ms)) { + LOG_ERR("%s: encode failed (tokenize)\n", __func__); return 1; } @@ -2458,17 +2468,17 @@ static int32_t cbx_ref_tokenize(mtmd_context * ctx, const clip_image_f32 & mel, } // talker conditioning rows: the speaker row, then on mtl the perceiver block and the emotion row -static int32_t cbx_ref_cond(mtmd_context * ctx, const clip_image_f32 & ve_mel, +static int32_t cbx_ref_cond(mtmd_context * ctx, const mtmd_image_f32 & ve_mel, const std::vector & pse, std::vector & out_rows) { - clip_ctx * ctx_clip = ctx->ctx_a; + mtmd_subcontext * sctx = ctx->ctx_a.get(); // n_expect 0 accepts any non-empty size auto read_t = [&](const char * name, std::vector & v, size_t n_expect) -> bool { - const size_t n = clip_cbx_read_tensor(ctx_clip, name, nullptr, 0); + const size_t n = sctx->model.cbx_read_tensor(name, nullptr, 0); if (n == 0 || (n_expect > 0 && n != n_expect)) { return false; } v.resize(n); - return clip_cbx_read_tensor(ctx_clip, name, v.data(), n) == n; + return sctx->model.cbx_read_tensor(name, v.data(), n) == n; }; // voice encoder (embeds_from_wavs): 3-layer lstm per 160-frame partial, the normalized partial embeddings are averaged @@ -2725,7 +2735,7 @@ static int32_t cbx_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audi } // [spkr, perceiver block] on mtl, [spkr, reference speech rows] on turbo - const bool mtl = cbx_is_mtl(ctx->ctx_a); + const bool mtl = cbx_is_mtl(ctx->ctx_a->model); std::vector pse; if (mtl) { pse = std::move(t3_rows); @@ -2754,7 +2764,7 @@ const char * mtmd_gen_audio_voice_name(const mtmd_context * ctx, int32_t idx) { if (!ctx->ctx_gen_a || idx < 0) { return nullptr; } - const auto & voices = clip_gen_get_voices(ctx->ctx_gen_a); + const auto & voices = ctx->ctx_gen_a->model.gen_voices; return idx < (int32_t) voices.size() ? std::next(voices.begin(), idx)->first.c_str() : nullptr; } @@ -2763,7 +2773,7 @@ mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * na return nullptr; } // the name is matched case-insensitively, as the lfm3a "tts" chat template lowers it - const auto & voices = clip_gen_get_voices(ctx->ctx_gen_a); + const auto & voices = ctx->ctx_gen_a->model.gen_voices; const auto it = std::find_if(voices.begin(), voices.end(), [name](const auto & voice) { return std::equal(voice.first.begin(), voice.first.end(), name, name + strlen(name), [](unsigned char a, unsigned char b) { return std::tolower(a) == std::tolower(b); }); @@ -2771,7 +2781,7 @@ mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * na if (it == voices.end()) { return nullptr; } - return mtmd_bitmap::init_voice(it->first, it->second, clip_n_mmproj_embd(ctx->ctx_gen_a)); + return mtmd_bitmap::init_voice(it->first, it->second, ctx->ctx_gen_a->model.n_mmproj_embd()); } mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { @@ -2781,7 +2791,7 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { return inp; } - switch (clip_get_projector_type(ctx->ctx_gen_a)) { + switch (ctx->ctx_gen_a->model.proj_type) { case PROJECTOR_TYPE_QWEN3TTS_GEN: // https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base/blob/main/generation_config.json inp.top_k = 50; @@ -2806,7 +2816,7 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { case PROJECTOR_TYPE_LFM3A_GEN: { // MaskGIT sampling runs in the helper, with the settings of the checkpoint - const auto * hparams = clip_get_hparams(ctx->ctx_gen_a); + const auto * hparams = &ctx->ctx_gen_a->model.hparams; inp.top_k = hparams->gen_top_k; inp.top_p = 1.0f; inp.temp = hparams->gen_temp; @@ -2820,8 +2830,8 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { } struct mtmd_gen_state { - clip_ctx * ctx_clip = nullptr; - clip_gen_state * clip_state = nullptr; + mtmd_subcontext * sctx = nullptr; + mtmd_subctx_gen_state * gen_state = nullptr; std::vector out_codes; // this frame's 16 sampled codes (GEN_CODE) std::vector out_feats; // this frame's continuous features, if any (GEN_CODE) @@ -2829,13 +2839,13 @@ struct mtmd_gen_state { std::vector out_audio; // decoded PCM samples for the current frame (GEN_WAV) // voice reference for GEN_WAV, from a speaker clip or a predefined voice (chatterbox) - clip_gen_ref ref; + mtmd_gen_ref ref; // lfm3a: the codec graph yields STFT frames, the ISTFT runs on the host std::unique_ptr istft; ~mtmd_gen_state() { - clip_gen_state_free(clip_state); + delete gen_state; } }; @@ -2851,13 +2861,13 @@ mtmd_gen_state * mtmd_gen_state_init(mtmd_context * ctx, mtmd_gen_state_params p return nullptr; } auto st = std::make_unique(); - st->ctx_clip = ctx->ctx_gen_a; - st->clip_state = clip_gen_state_init(ctx->ctx_gen_a, params.seed); - if (!st->clip_state) { + st->sctx = ctx->ctx_gen_a.get(); + st->gen_state = ctx->ctx_gen_a->gen_state_init(params.seed); + if (!st->gen_state) { return nullptr; } - if (clip_get_projector_type(ctx->ctx_gen_a) == PROJECTOR_TYPE_LFM3A_GEN) { - const auto * hparams = clip_get_hparams(ctx->ctx_gen_a); + if (ctx->ctx_gen_a->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN) { + const auto * hparams = &ctx->ctx_gen_a->model.hparams; st->istft = std::make_unique(hparams->audio_n_fft, hparams->audio_hop_len); } return st.release(); @@ -2868,17 +2878,17 @@ void mtmd_gen_state_free(mtmd_gen_state * state) { } void mtmd_gen_state_reset(mtmd_gen_state * state) { - clip_gen_state_reset(state->clip_state); + state->gen_state->reset(); if (state->istft) { - const auto * hparams = clip_get_hparams(state->ctx_clip); + const auto * hparams = &state->sctx->model.hparams; state->istft = std::make_unique(hparams->audio_n_fft, hparams->audio_hop_len); } } // lfm3a GEN_WAV: ISTFT of the codec output, n_codes == 0 flushes and resets the state static int32_t mtmd_gen_wav_lfm3a(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) { - clip_ctx * ctx_clip = ctx->ctx_gen_a; - const auto & hparams = *clip_get_hparams(ctx_clip); + mtmd_subcontext * sctx = ctx->ctx_gen_a.get(); + const auto & hparams = sctx->model.hparams; const int n_bins = hparams.audio_n_fft / 2 + 1; const int n_trim = (hparams.audio_n_fft - hparams.audio_hop_len) / 2; @@ -2896,23 +2906,23 @@ static int32_t mtmd_gen_wav_lfm3a(mtmd_context * ctx, mtmd_gen_state * state, co std::vector feats; // no hidden-state input, the batch entry is an unused placeholder - clip_image_f32 dummy; + mtmd_image_f32 dummy; dummy.set_size({1, 1}, false, true); dummy.cpy_buf(std::vector(1, 0.0f)); - clip_image_f32_batch batch; + mtmd_image_f32_batch batch; batch.is_audio = true; batch.entries.push_back(std::move(dummy)); - clip_encode_params params; + mtmd_subctx_encode_params params; params.imgs = &batch; params.n_threads = ctx->n_threads; - params.gen_process = CLIP_GEN_PROCESS_GEN_WAV; + params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV; params.codes = &in_codes; params.out_feats = &feats; - params.gen_state = state->clip_state; - if (!clip_encode(ctx_clip, ¶ms)) { - LOG_ERR("%s: clip_encode failed (codec)\n", __func__); + params.gen_state = state->gen_state; + if (!sctx->encode(¶ms)) { + LOG_ERR("%s: encode failed (codec)\n", __func__); return 1; } @@ -2938,7 +2948,7 @@ static int32_t mtmd_gen_wav_lfm3a(mtmd_context * ctx, mtmd_gen_state * state, co } int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) { - if (!state || state->ctx_clip != ctx->ctx_gen_a) { + if (!state || state->sctx != ctx->ctx_gen_a.get()) { LOG_ERR("%s: invalid gen state\n", __func__); return 1; } @@ -2951,35 +2961,35 @@ int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) { } static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) { - clip_ctx * ctx_clip = ctx->ctx_gen_a; - if (!ctx_clip) { + mtmd_subcontext * sctx = ctx->ctx_gen_a.get(); + if (!sctx) { LOG_ERR("%s: model does not support audio generation\n", __func__); return 1; } - if (!state || state->ctx_clip != ctx_clip) { + if (!state || state->sctx != sctx) { LOG_ERR("%s: invalid gen state\n", __func__); return 1; } *out = {}; - if (clip_get_projector_type(ctx_clip) == PROJECTOR_TYPE_SOPRANO) { - const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip); + if (sctx->model.proj_type == PROJECTOR_TYPE_SOPRANO) { + const size_t n_embd = (size_t) sctx->model.n_mmproj_embd(); if (inp->type != MTMD_GEN_PROCESS_TYPE_GEN_WAV || !inp->feats || inp->n_feats < 2 * n_embd || inp->n_feats % n_embd != 0 || inp->codes) { LOG_ERR("%s: soprano requires at least 2 frames of features, without codes\n", __func__); return 1; } } - if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_WAV && clip_get_projector_type(ctx_clip) == PROJECTOR_TYPE_LFM3A_GEN) { + if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_WAV && sctx->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN) { return mtmd_gen_wav_lfm3a(ctx, state, inp, out); } if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE) { - const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip); + const size_t n_embd = (size_t) sctx->model.n_mmproj_embd(); // the hidden state may be omitted when the graph does not use it - clip_image_f32 hidden_state; + mtmd_image_f32 hidden_state; hidden_state.set_size({(int) n_embd, 1}, false, true); hidden_state.cpy_buf(inp->embd ? std::vector(inp->embd, inp->embd + n_embd) : std::vector(n_embd, 0.0f)); @@ -2988,7 +2998,7 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * in_codes.assign(inp->codes, inp->codes + inp->n_codes); } - clip_image_f32_batch batch; + mtmd_image_f32_batch batch; batch.is_audio = true; batch.entries.push_back(std::move(hidden_state)); @@ -2997,10 +3007,10 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * std::vector out_feats; bool is_eos = false; - clip_encode_params params; + mtmd_subctx_encode_params params; params.imgs = &batch; params.n_threads = ctx->n_threads; - params.gen_process = CLIP_GEN_PROCESS_GEN_CODE; + params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; params.out_embd = &out_embd; params.out_codes = &out_codes; params.out_feats = &out_feats; @@ -3010,10 +3020,10 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * params.temp = inp->temp; params.codes = inp->codes ? &in_codes : nullptr; params.out_is_eos = &is_eos; - params.gen_state = state->clip_state; + params.gen_state = state->gen_state; - if (!clip_encode(ctx_clip, ¶ms)) { - LOG_ERR("%s: clip_encode failed (gen_code)\n", __func__); + if (!sctx->encode(¶ms)) { + LOG_ERR("%s: encode failed (gen_code)\n", __func__); return 1; } @@ -3031,7 +3041,7 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * } // MTMD_GEN_PROCESS_TYPE_GEN_WAV - if (clip_get_projector_type(ctx_clip) == PROJECTOR_TYPE_CHATTERBOX) { + if (sctx->model.proj_type == PROJECTOR_TYPE_CHATTERBOX) { if (!inp->codes || inp->n_codes == 0) { LOG_ERR("%s: codes required for gen_wav\n", __func__); return 1; @@ -3043,24 +3053,24 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * std::vector in_codes(inp->codes, inp->codes + inp->n_codes); // the batch entry is unused, present to satisfy the encode interface - clip_image_f32 dummy; + mtmd_image_f32 dummy; dummy.set_size({1, 1}, false, true); dummy.cpy_buf(std::vector(1, 0.0f)); - clip_image_f32_batch batch; + mtmd_image_f32_batch batch; batch.is_audio = true; batch.entries.push_back(std::move(dummy)); - clip_encode_params params; + mtmd_subctx_encode_params params; params.imgs = &batch; params.n_threads = ctx->n_threads; - params.gen_process = CLIP_GEN_PROCESS_TTS; + params.gen_process = MTMD_SUBCTX_GEN_PROCESS_TTS; params.codes = &in_codes; params.out_audio = &state->out_audio; params.ref = &state->ref; - params.gen_state = state->clip_state; + params.gen_state = state->gen_state; - if (!clip_encode(ctx_clip, ¶ms)) { - LOG_ERR("%s: clip_encode failed (gen_wav)\n", __func__); + if (!sctx->encode(¶ms)) { + LOG_ERR("%s: encode failed (gen_wav)\n", __func__); return 1; } @@ -3085,25 +3095,25 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * // gen_wav has no hidden-state input, the batch entry is an unused placeholder // TODO @ngxson : some models in the future may require hidden-state input, need to update this code later - clip_image_f32 dummy; + mtmd_image_f32 dummy; dummy.set_size({1, 1}, false, true); dummy.cpy_buf(std::vector(1, 0.0f)); - clip_image_f32_batch batch; + mtmd_image_f32_batch batch; batch.is_audio = true; batch.entries.push_back(std::move(dummy)); - clip_encode_params params; + mtmd_subctx_encode_params params; params.imgs = &batch; params.n_threads = ctx->n_threads; - params.gen_process = CLIP_GEN_PROCESS_GEN_WAV; + params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV; params.codes = has_codes ? &in_codes : nullptr; params.feats = has_feats ? &in_feats : nullptr; params.out_audio = &state->out_audio; - params.gen_state = state->clip_state; + params.gen_state = state->gen_state; - if (!clip_encode(ctx_clip, ¶ms)) { - LOG_ERR("%s: clip_encode failed (code2wav)\n", __func__); + if (!sctx->encode(¶ms)) { + LOG_ERR("%s: encode failed (code2wav)\n", __func__); return 1; } @@ -3138,7 +3148,7 @@ int32_t mtmd_batch_add_chunk(mtmd_batch * batch, const mtmd_input_chunk * chunk) return 1; } - auto * ctx = batch->ctx->get_clip_ctx(chunk); + auto * ctx = batch->ctx->get_subctx(chunk); if (!ctx) { LOG_ERR("%s: model does not support input chunk type %d\n", __func__, (int)chunk->type); return 1; @@ -3150,7 +3160,7 @@ int32_t mtmd_batch_add_chunk(mtmd_batch * batch, const mtmd_input_chunk * chunk) return 0; } - if (!clip_support_batch(ctx)) { + if (!ctx->support_batch) { // if no batching support, batch can only have one single chunk return 2; // "batch too large" error code } @@ -3260,7 +3270,7 @@ bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk { // E2B (n_embd = 1536) and E4B (n_embd = 2560) always use causal // note: use mmproj n_embd, because text model may not be provided (e.g. mtmd_get_memory_usage) - const int n_embd = clip_n_mmproj_embd(ctx->ctx_v); + const int n_embd = ctx->ctx_v->model.n_mmproj_embd(); return n_embd != 1536 && n_embd != 2560; } case PROJECTOR_TYPE_GEMMA4UV: @@ -3289,7 +3299,7 @@ int mtmd_get_audio_sample_rate(const mtmd_context * ctx) { if (!ctx->ctx_a) { return -1; } - return clip_get_hparams(ctx->ctx_a)->audio_sample_rate; + return ctx->ctx_a->model.hparams.audio_sample_rate; } mtmd_video_config mtmd_video_config_init(mtmd_video_template tmpl, const mtmd_image_preprocessor * preproc, @@ -3312,7 +3322,7 @@ mtmd_video_config mtmd_video_config_init(mtmd_video_template tmpl, const mtmd_im n = std::min(n, src_n_frames); cfg.n_frames = n; - clip_image_size size; + mtmd_image_size size; if (preproc && preproc->get_video_frame_size(n, src_width, src_height, size)) { cfg.frame_width = size.width; cfg.frame_height = size.height; @@ -3324,7 +3334,7 @@ mtmd_video_config mtmd_get_video_config(const mtmd_context * ctx, int32_t src_width, int32_t src_height, int32_t src_n_frames, float src_fps) { mtmd_video_template tmpl = MTMD_VIDEO_TEMPLATE_DEFAULT; - switch (ctx->ctx_v ? clip_get_projector_type(ctx->ctx_v) : PROJECTOR_TYPE_UNKNOWN) { + switch (ctx->ctx_v ? ctx->ctx_v->model.proj_type : PROJECTOR_TYPE_UNKNOWN) { case PROJECTOR_TYPE_LFM3VL: tmpl = ctx->video_has_soundtrack() ? MTMD_VIDEO_TEMPLATE_LFM3_OMNI : MTMD_VIDEO_TEMPLATE_LFM3; break; @@ -3650,7 +3660,7 @@ static mtmd_decoder_pos mtmd_canvas_decoder_pos(const mtmd_image_tokens * image_ } i -= gw * gh; - const size_t n_sep = clip_suffix_rows(entries[k].suffix_type).size(); + const size_t n_sep = mtmd_suffix_rows(entries[k].suffix_type).size(); if (i < n_sep) { const uint32_t p = t + 1 + i; pos.t = p; @@ -3752,7 +3762,7 @@ llama_pos mtmd_image_tokens_get_n_pos(const mtmd_image_tokens * image_tokens) { // one position per tile, plus one per separator llama_pos n_pos = 0; for (const auto & entry : image_tokens->batch_f32.entries) { - n_pos += 1 + (llama_pos) clip_suffix_rows(entry.suffix_type).size(); + n_pos += 1 + (llama_pos) mtmd_suffix_rows(entry.suffix_type).size(); } return n_pos; } @@ -3796,14 +3806,16 @@ mtmd_input_chunks * mtmd_test_create_input_chunks() { return chunks; } +struct mtmd_logger_state g_logger_state = {mtmd_log_callback_default, NULL}; + void mtmd_log_set(ggml_log_callback log_callback, void * user_data) { - g_logger_state.log_callback = log_callback ? log_callback : clip_log_callback_default; + g_logger_state.log_callback = log_callback ? log_callback : mtmd_log_callback_default; g_logger_state.log_callback_user_data = user_data; } struct mtmd_caps mtmd_get_cap_from_file(const char * fname) { try { - auto tmp = clip_get_cap(fname); + auto tmp = mtmd_model_get_caps(fname); mtmd_caps cap; cap.inp_audio = tmp.has_audio; cap.inp_vision = tmp.has_vision; @@ -3819,14 +3831,12 @@ struct mtmd_caps mtmd_get_cap_from_file(const char * fname) { // Debugging API (NOT intended for public use) // -static void mtmd_debug_encode_impl(mtmd_context * ctx, clip_ctx * ctx_clip, clip_image_f32 & image) { - clip_set_debug_output_embeddings(ctx_clip, true); - int n_mmproj_embd = clip_n_mmproj_embd(ctx_clip); - int n_tokens = clip_n_output_tokens(ctx_clip, &image); +static void mtmd_debug_encode_impl(mtmd_context * ctx, mtmd_subcontext * sctx, mtmd_image_f32 & image) { + sctx->debug_output_embeddings = true; + int n_mmproj_embd = sctx->model.n_mmproj_embd(); + int n_tokens = sctx->model.n_output_tokens(&image); std::vector embd_output(n_tokens * n_mmproj_embd, 0.0f); - bool ok = clip_image_encode( - ctx_clip, - ctx->n_threads, + bool ok = sctx->encode_image(ctx->n_threads, &image, embd_output); if (!ok) { @@ -3845,11 +3855,11 @@ void mtmd_debug_encode_image(mtmd_context * ctx, const std::vectorctx_v, inp_image); + mtmd_debug_encode_impl(ctx, ctx->ctx_v.get(), inp_image); } void mtmd_debug_encode_audio(mtmd_context * ctx, const std::vector & input) { @@ -3857,7 +3867,7 @@ void mtmd_debug_encode_audio(mtmd_context * ctx, const std::vector & inpu LOG_ERR("%s: model does not support audio input\n", __func__); return; } - int n_mel = clip_get_hparams(ctx->ctx_a)->n_mel_bins; + int n_mel = ctx->ctx_a->model.hparams.n_mel_bins; const int audio_nx = (int)input.size(); std::vector audio_buf(audio_nx * n_mel); for (int i = 0; i < audio_nx; i++) { @@ -3865,11 +3875,11 @@ void mtmd_debug_encode_audio(mtmd_context * ctx, const std::vector & inpu audio_buf[j * audio_nx + i] = input[i]; } } - clip_image_f32 inp_audio; + mtmd_image_f32 inp_audio; inp_audio.set_size({audio_nx, n_mel}, false, true); inp_audio.cpy_buf(audio_buf); LOG_INF("%s: created input audio with nx=%d, ny=%d\n", __func__, audio_nx, n_mel); - mtmd_debug_encode_impl(ctx, ctx->ctx_a, inp_audio); + mtmd_debug_encode_impl(ctx, ctx->ctx_a.get(), inp_audio); } void mtmd_debug_preprocess_image(mtmd_context * ctx, const std::vector & rgb_values, int nx, int ny) { @@ -3877,13 +3887,13 @@ void mtmd_debug_preprocess_image(mtmd_context * ctx, const std::vector LOG_ERR("%s: model does not support vision input\n", __func__); return; } - clip_image_u8 img_u8; + mtmd_image_u8 img_u8; img_u8.set_size({nx, ny}, false); img_u8.cpy_buf(rgb_values); GGML_ASSERT(ctx->image_preproc != nullptr); mtmd_image_preproc_out preproc_out = ctx->image_preproc->preprocess(img_u8); - clip_image_f32_batch batch_f32; + mtmd_image_f32_batch batch_f32; batch_f32.is_audio = false; for (auto & entry : preproc_out.entries) { batch_f32.entries.push_back(std::move(entry)); @@ -3938,23 +3948,23 @@ mtmd_memory_usage mtmd_get_memory_usage(const char * mmproj_fname, ctx.reset(new mtmd_context(mmproj_fname, nullptr, ctx_params, true)); mtmd_log_set(saved_log_callback, saved_log_user_data); // restore log callback std::map total_mem; - auto merge = [&](const struct clip_ctx * c) { - for (auto & [dev, size] : clip_get_mem_usage(c)) { + auto merge = [&](const struct mtmd_subcontext * c) { + for (auto & [dev, size] : c->get_mem_usage()) { total_mem[dev] += size; } }; if (ctx->ctx_v) { - merge(ctx->ctx_v); + merge(ctx->ctx_v.get()); } if (ctx->ctx_a) { - merge(ctx->ctx_a); + merge(ctx->ctx_a.get()); } if (ctx->ctx_gen_a) { - merge(ctx->ctx_gen_a); + merge(ctx->ctx_gen_a.get()); } mtmd_memory_usage res; res.backend_mem_usage = std::move(total_mem); - res.image_max_tokens = ctx->ctx_v ? clip_get_image_max_tokens(ctx->ctx_v) : -1; + res.image_max_tokens = ctx->ctx_v ? ctx->ctx_v->model.image_max_tokens() : -1; res.use_non_causal = ctx->ctx_v ? mtmd_decode_use_non_causal(ctx.get(), nullptr) : false; return res; } catch (const std::exception & e) { @@ -3971,7 +3981,7 @@ mtmd_memory_usage mtmd_get_memory_usage(const char * mmproj_fname, // the model-independent part of a stream, the model-specific part is in the subclasses below struct mtmd_audio_stream { mtmd_context * ctx; - clip_ctx * ctx_clip; + mtmd_subcontext * sctx; // model constants int n_mel; @@ -4015,10 +4025,10 @@ struct mtmd_audio_stream { // Voxtral Realtime: a row is an audio embedding plus the embedding of the previous token struct mtmd_audio_stream_voxtral_rt : mtmd_audio_stream { - clip_enc_state * enc_state = nullptr; + mtmd_subctx_enc_state * enc_state = nullptr; int n_fft; - int n_frames_max; // encoder frames per clip_encode() call + int n_frames_max; // encoder frames per encode() call int n_right_pad; // silence tokens added by flush() float log_mel_max; // global_log_mel_max in params.json @@ -4030,7 +4040,7 @@ struct mtmd_audio_stream_voxtral_rt : mtmd_audio_stream { std::vector tok_embd; ~mtmd_audio_stream_voxtral_rt() override { - clip_enc_state_free(enc_state); + delete enc_state; } bool init(const mtmd_audio_stream_params & params) override; @@ -4043,7 +4053,7 @@ struct mtmd_audio_stream_voxtral_rt : mtmd_audio_stream { // training samples are max 30 s: restart the sequence at a silent chunk after 15 s, always after 30 s; the encoder keeps its state struct mtmd_audio_stream_lfm3a : mtmd_audio_stream { int win; // samples per mel frame - clip_enc_state * enc_state = nullptr; + mtmd_subctx_enc_state * enc_state = nullptr; // live ASR static constexpr int64_t chunk_rows = 12; @@ -4073,7 +4083,7 @@ struct mtmd_audio_stream_lfm3a : mtmd_audio_stream { bool end_done = false; ~mtmd_audio_stream_lfm3a() override { - clip_enc_state_free(enc_state); + delete enc_state; } bool init(const mtmd_audio_stream_params & params) override; @@ -4088,7 +4098,7 @@ bool mtmd_support_audio_stream(const mtmd_context * ctx) { if (!ctx->ctx_a || !ctx->text_model) { return false; } - const projector_type proj = clip_get_projector_type(ctx->ctx_a); + const projector_type proj = ctx->ctx_a->model.proj_type; return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A; } @@ -4105,19 +4115,19 @@ mtmd_audio_stream * mtmd_audio_stream_init(mtmd_context * ctx, mtmd_audio_stream LOG_ERR("%s: the model does not support streaming audio input\n", __func__); return nullptr; } - const clip_hparams * hparams = clip_get_hparams(ctx->ctx_a); + const mtmd_hparams * hparams = &ctx->ctx_a->model.hparams; std::unique_ptr s; - switch (clip_get_projector_type(ctx->ctx_a)) { + switch (ctx->ctx_a->model.proj_type) { case PROJECTOR_TYPE_VOXTRAL_RT: s = std::make_unique(); break; case PROJECTOR_TYPE_LFM3A: s = std::make_unique(); break; default: GGML_ABORT("unsupported projector"); } s->ctx = ctx; - s->ctx_clip = ctx->ctx_a; + s->sctx = ctx->ctx_a.get(); s->n_mel = hparams->n_mel_bins; s->hop = hparams->audio_hop_len; - s->n_embd = clip_n_mmproj_embd(ctx->ctx_a); + s->n_embd = ctx->ctx_a->model.n_mmproj_embd(); if (s->n_embd != ctx->n_embd_text) { LOG_ERR("%s: mmproj embedding size %d does not match the text model (%d)\n", __func__, s->n_embd, ctx->n_embd_text); return nullptr; @@ -4166,7 +4176,7 @@ bool mtmd_audio_stream_is_seq_start(const mtmd_audio_stream * s) { // bool mtmd_audio_stream_voxtral_rt::init(const mtmd_audio_stream_params & params) { - const clip_hparams * hparams = clip_get_hparams(ctx_clip); + const mtmd_hparams * hparams = &sctx->model.hparams; const llama_vocab * vocab = ctx->vocab; n_fft = hparams->audio_n_fft; @@ -4180,7 +4190,7 @@ bool mtmd_audio_stream_voxtral_rt::init(const mtmd_audio_stream_params & params) // the encoder works in whole decoder rows int n_step = (int) ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 / hop); n_step = std::max(n_step, frames_per_tok); - n_step = CLIP_ALIGN(n_step, frames_per_tok); + n_step = MTMD_ALIGN(n_step, frames_per_tok); n_step_mel = std::min(n_step, 2 * n_frames_max); if (params.n_prompt > 0) { @@ -4209,7 +4219,7 @@ bool mtmd_audio_stream_voxtral_rt::init(const mtmd_audio_stream_params & params) cache.fill_hann_window(hparams->audio_window_len, true); cache.fill_mel_filterbank_matrix(n_mel, n_fft, hparams->audio_sample_rate); - enc_state = clip_enc_state_init(ctx_clip, n_frames_max); + enc_state = sctx->enc_state_init(n_frames_max); if (!enc_state) { return false; } @@ -4293,20 +4303,20 @@ static bool mtmd_audio_stream_encode(mtmd_audio_stream_voxtral_rt * s, bool forc } } - clip_image_f32 img; + mtmd_image_f32 img; img.set_size({n_in, s->n_mel}, false, /* is_audio */ true); img.cpy_buf(data); - clip_image_f32_batch batch; + mtmd_image_f32_batch batch; batch.is_audio = true; batch.entries.push_back(std::move(img)); std::vector res((size_t) s->n_embd * (n / s->frames_per_tok)); - clip_encode_params params; + mtmd_subctx_encode_params params; params.n_threads = s->ctx->n_threads; params.imgs = &batch; params.out_embd = &res; params.enc_state = s->enc_state; - if (!clip_encode(s->ctx_clip, ¶ms)) { + if (!s->sctx->encode(¶ms)) { LOG_ERR("%s: failed to encode audio\n", __func__); return false; } @@ -4385,7 +4395,7 @@ int32_t mtmd_audio_stream_voxtral_rt::step(llama_token last_token) { // bool mtmd_audio_stream_lfm3a::init(const mtmd_audio_stream_params & params) { - const clip_hparams * hparams = clip_get_hparams(ctx_clip); + const mtmd_hparams * hparams = &sctx->model.hparams; // one row per 8 mel frames (three stride-2 convs), mel frames are not centered win = hparams->audio_window_len; @@ -4393,7 +4403,7 @@ bool mtmd_audio_stream_lfm3a::init(const mtmd_audio_stream_params & params) { samples_per_tok = frames_per_tok * hop; const int n_step = (int) ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 / hop); - n_step_mel = CLIP_ALIGN(std::max(n_step, frames_per_tok), frames_per_tok); + n_step_mel = MTMD_ALIGN(std::max(n_step, frames_per_tok), frames_per_tok); if (params.n_prompt > 0) { // live ASR, a chunk per encoder call @@ -4416,7 +4426,7 @@ bool mtmd_audio_stream_lfm3a::init(const mtmd_audio_stream_params & params) { max_seq_rows = std::lround(30.0 * hparams->audio_sample_rate / samples_per_tok); } - enc_state = clip_enc_state_init(ctx_clip, n_step_mel); + enc_state = sctx->enc_state_init(n_step_mel); if (!enc_state) { return false; } @@ -4508,22 +4518,22 @@ static bool mtmd_audio_stream_encode(mtmd_audio_stream_lfm3a * s, bool flush) { data[(size_t) m * n + t] = s->mel[(size_t) t * s->n_mel + m]; } } - clip_image_f32 img; + mtmd_image_f32 img; img.set_size({n, s->n_mel}, false, /* is_audio */ true); img.cpy_buf(data); - const int n_rows = clip_n_output_tokens(s->ctx_clip, &img); + const int n_rows = s->sctx->model.n_output_tokens(&img); - clip_image_f32_batch batch; + mtmd_image_f32_batch batch; batch.is_audio = true; batch.entries.push_back(std::move(img)); std::vector res((size_t) s->n_embd * n_rows); - clip_encode_params params; + mtmd_subctx_encode_params params; params.n_threads = s->ctx->n_threads; params.imgs = &batch; params.out_embd = &res; params.enc_state = s->enc_state; - if (!clip_encode(s->ctx_clip, ¶ms)) { + if (!s->sctx->encode(¶ms)) { LOG_ERR("%s: failed to encode audio\n", __func__); return false; }