diff --git a/conversion/chatterbox.py b/conversion/chatterbox.py
index bca0a2633..d438228e5 100644
--- a/conversion/chatterbox.py
+++ b/conversion/chatterbox.py
@@ -355,7 +355,7 @@ class ChatterboxMmprojModel(MmprojModel):
def set_gguf_parameters(self):
self.gguf_writer.add_file_type(self.ftype)
- # speaker encoder, the front-end hparams are hard-coded in clip.cpp
+ # speaker encoder, the front-end hparams are hard-coded in tools/mtmd
self.gguf_writer.add_clip_has_audio_encoder(True)
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.CHATTERBOX_SPKENC)
self.gguf_writer.add_audio_projection_dim(80)
diff --git a/conversion/deepseek.py b/conversion/deepseek.py
index 581111da5..d559b604e 100644
--- a/conversion/deepseek.py
+++ b/conversion/deepseek.py
@@ -1078,7 +1078,7 @@ class DeepseekV4FlashVisionModel(MmprojModel):
self.gguf_writer.add_vision_use_silu(True) # SwiGLU MLP
self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"])
self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"])
- # hardcoded on the C++ side (see PROJECTOR_TYPE_DEEPSEEK4V in clip.cpp)
+ # hardcoded on the C++ side (see PROJECTOR_TYPE_DEEPSEEK4V in tools/mtmd)
# if future models use different values, add GGUF keys for those
assert self.global_config["vision_max_n_token"] == 384
assert self.global_config["vision_max_wh_ratio"] == 8
diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py
index 53980c1d0..6b098a037 100644
--- a/conversion/deepseek41.py
+++ b/conversion/deepseek41.py
@@ -458,7 +458,7 @@ class DeepseekV41VisionModel(DeepseekV4FlashVisionModel):
self.gguf_writer.add_vision_use_silu(True)
self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"])
self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"])
- # hardcoded in clip.cpp (PROJECTOR_TYPE_DEEPSEEK41V); add GGUF keys if a later model differs
+ # hardcoded in tools/mtmd (PROJECTOR_TYPE_DEEPSEEK41V); add GGUF keys if a later model differs
assert self.hparams_vision["max_image_tokens"] == 1024, self.hparams_vision["max_image_tokens"]
assert self.hparams_vision["max_wh_ratio"] is None, self.hparams_vision["max_wh_ratio"]
assert self.hparams_vision["rope_theta"] == 10000, self.hparams_vision["rope_theta"]
diff --git a/conversion/drama3.py b/conversion/drama3.py
index 7c590a014..ac01f481c 100644
--- a/conversion/drama3.py
+++ b/conversion/drama3.py
@@ -77,7 +77,7 @@ class Drama3MmprojModel(MmprojModel):
self.gguf_writer.add_gen_audio_head_count(cfg["n_head"])
self.gguf_writer.add_gen_audio_head_count_kv(cfg["n_local_heads"])
self.gguf_writer.add_gen_audio_attention_layernorm_eps(cfg["norm_eps"])
- # other tower and codec hparams are hardcoded in clip.cpp
+ # other tower and codec hparams are hardcoded in tools/mtmd
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
diff --git a/conversion/lfm3.py b/conversion/lfm3.py
index a0287d4bb..38fbb5a31 100644
--- a/conversion/lfm3.py
+++ b/conversion/lfm3.py
@@ -181,11 +181,11 @@ class LFM3VLVisionMixin(MmprojModel):
def _set_lfm3vl_gguf_parameters(self):
assert self.hparams_vision is not None
- # clip.cpp hardcodes the GELU projector and its input norm eps
+ # mtmd hardcodes the GELU projector and its input norm eps
if (self.global_config["projector_hidden_act"] != "gelu" or not self.global_config["projector_bias"]
or self.global_config["projector_layer_norm_eps"] != 1e-6):
raise ValueError("unsupported projector_hidden_act / projector_bias / projector_layer_norm_eps")
- # clip.cpp hardcodes the vision rope base, the RGB patchifier and the 3x3 spatial conv
+ # mtmd hardcodes the vision rope base, the RGB patchifier and the 3x3 spatial conv
if (self.hparams_vision["rope_parameters"]["rope_theta"] != 1e6 or self.hparams_vision["in_channels"] != 3
or self.hparams_vision["conv_kernel_size"] != 3):
raise ValueError("unsupported vision rope_theta / in_channels / conv_kernel_size")
@@ -347,7 +347,7 @@ class LFM3AudioMmprojModel(MmprojModel):
"feed_forward.w3": gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP,
"feed_forward.w2": gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN,
}
- # the codec graph in clip.cpp hardcodes the liquid-codec FSQ front end and decoder shape, conversion fails on a mismatch
+ # the codec graph in mtmd hardcodes the liquid-codec FSQ front end and decoder shape, conversion fails on a mismatch
_codec = {
"model_type": "liquid_codec", "fsq_levels": [8, 7, 6, 6], "n_q": 12, "quantizer_dim": 256,
"encoder_dim": 512, "stft_n_fft": 1280, "stft_hop_length": 320, "stft_win_length": 1280, "stft_frame_stack": 6,
@@ -405,7 +405,7 @@ class LFM3AudioMmprojModel(MmprojModel):
if fe[k] != v:
raise ValueError(f"unsupported feature_extractor.{k}={fe[k]!r}, expected {v!r}")
- # clip.cpp hardcodes the encoder rope base
+ # mtmd hardcodes the encoder rope base
if self.hparams_audio["rope_parameters"]["rope_theta"] != 1e6:
raise ValueError("unsupported audio encoder rope_theta")
@@ -420,7 +420,7 @@ class LFM3AudioMmprojModel(MmprojModel):
if not self._audio_output:
return
- # audio generation: depthformer hparams, the codec ones are constant and live in clip.cpp
+ # audio generation: depthformer hparams, the codec ones are constant and live in mtmd
gen = self.get_generation_config()
for k, v in self._generation.items():
if gen[k] != v:
diff --git a/conversion/minicpm.py b/conversion/minicpm.py
index d8fceb0c0..8c7be8a9a 100644
--- a/conversion/minicpm.py
+++ b/conversion/minicpm.py
@@ -161,7 +161,7 @@ class MiniCPMV4_6VisionModel(MmprojModel):
# In MiniCPM-V 4.6 `vision_config.image_size` (980) describes the SigLIP
# positional embedding bucket grid (70 x 70), while the per-slice processing
# resolution is the preprocessor's `scale_resolution` (typically 448).
- # The CLIP loader in tools/mtmd/clip.cpp consumes `clip.vision.image_size`
+ # The mmproj loader in tools/mtmd/mtmd-model.cpp consumes `clip.vision.image_size`
# as the slice size and warmup resolution, so report `scale_resolution` there
# to match the upstream MiniCPMV4_6ImageProcessorPil slicing rules.
scale_resolution = self.preprocessor_config.get(
@@ -173,7 +173,7 @@ class MiniCPMV4_6VisionModel(MmprojModel):
super().set_gguf_parameters()
assert self.hparams_vision is not None
- # projector type string is consumed by clip_projector_type_from_string() in clip.cpp
+ # projector type string is consumed by projector_type_from_string() in tools/mtmd/mtmd-impl.h
self.gguf_writer.add_clip_projector_type(self.projector_type)
self.gguf_writer.add_vision_projector_scale_factor(
@@ -243,7 +243,7 @@ class MiniCPMV4_7VisionModel(MiniCPMV4_6VisionModel):
projector_type = gguf.VisionProjectorType.MINICPMV4_7
# MiniCPMV4_7ImageProcessorPil default
default_scale_resolution = 448
- # rows of v.tok_embd_sep, the order must match clip_suffix_rows() in clip-impl.h
+ # rows of v.tok_embd_sep, the order must match mtmd_suffix_rows() in tools/mtmd/mtmd-impl.h
tok_embd_sep = ["", "", "", "\n"]
def get_downsample_mode(self) -> str:
diff --git a/conversion/nemotron.py b/conversion/nemotron.py
index 066149744..c818a73e1 100644
--- a/conversion/nemotron.py
+++ b/conversion/nemotron.py
@@ -208,7 +208,7 @@ class NemotronNanoV2VLModel(MmprojModel):
return name, gen
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- # RADIO's pos_embed doesn't have .weight suffix, but clip.cpp expects it
+ # RADIO's pos_embed doesn't have .weight suffix, but mtmd expects it
if "patch_generator.pos_embed" in name:
# Downsample position embeddings for fixed 512x512 image size
import torch.nn.functional as F
diff --git a/conversion/pockettts.py b/conversion/pockettts.py
index 1b5c84111..c9198a221 100644
--- a/conversion/pockettts.py
+++ b/conversion/pockettts.py
@@ -302,7 +302,7 @@ class PocketTTSMmprojModel(MmprojModel):
# voice-prompt encoder: mimi encoder + speaker_proj
if self.has_audio_encoder:
self.gguf_writer.add_clip_has_audio_encoder(True)
- # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models
+ # note: the 24kHz sample rate is hardcoded on the mtmd side, like the other audio models
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC)
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"])
@@ -314,7 +314,7 @@ class PocketTTSMmprojModel(MmprojModel):
self.gguf_writer.add_audio_num_mel_bins(1)
# generation: flow-matching decoder + mimi decoder
- # the SEANet and flow net hparams are constant across the family, clip.cpp holds them
+ # the SEANet and flow net hparams are constant across the family, mtmd holds them
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_GEN)
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py
index cdb1e0c9b..28bb6e095 100644
--- a/conversion/qwen3tts.py
+++ b/conversion/qwen3tts.py
@@ -235,11 +235,11 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
# handle speaker encoder config
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
- # mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the clip.cpp default)
+ # mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the mtmd default)
self.gguf_writer.add_audio_num_mel_bins(128)
# 3 SE-Res2Net stages; the stem conv, mfa, asp and fc are not counted here
self.gguf_writer.add_audio_block_count(3)
- # ECAPA-TDNN has no attention/FFN, these are dummy to allow clip.cpp to load it
+ # ECAPA-TDNN has no attention/FFN, these are dummy to allow mtmd to load it
self.gguf_writer.add_audio_embedding_length(1536)
self.gguf_writer.add_audio_head_count(1)
self.gguf_writer.add_audio_feed_forward_length(1536)
@@ -256,7 +256,7 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
self.gguf_writer.add_gen_audio_head_count(code_predictor_config["num_attention_heads"])
self.gguf_writer.add_gen_audio_head_count_kv(code_predictor_config["num_key_value_heads"])
self.gguf_writer.add_gen_audio_attention_layernorm_eps(code_predictor_config["rms_norm_eps"])
- # note: code2wav hparams are hardcoded on the mtmd/clip.cpp side for now, not written here
+ # note: code2wav hparams are hardcoded on the mtmd side for now, not written here
def _wav_decoder_config(self) -> dict[str, Any] | None:
# code2wav has its own config.json, inside the speech_tokenizer dir
diff --git a/docs/development/HOWTO-add-model.md b/docs/development/HOWTO-add-model.md
index abf6503f2..558363f4e 100644
--- a/docs/development/HOWTO-add-model.md
+++ b/docs/development/HOWTO-add-model.md
@@ -128,7 +128,7 @@ Note: to debug the inference graph: you can use [llama-eval-callback](/examples/
If the new model supports multimodal inputs, you will need to add a new encoder definition in `libmtmd`. You can find more information about llama.cpp's multimodal support in [the docs](../multimodal.md) and in the `tools/mtmd` source directory.
1. In the conversion script, make sure you add a subclass that extends `MmprojModel` or another class that inherits from the same base class.
-2. Add the encoder definition in `clip.cpp`.
+2. Add the encoder definition: hparams and tensor loading in `mtmd-model.cpp`, graph inputs in `mtmd-subcontext.cpp`.
3. Implement the preprocessor in `mtmd.cpp`. In most cases, you can reuse an existing preprocessor.
4. Implement the encoder GGML graph, either in a dedicated file if the model is truly different from existing ones, or by reusing an existing implementation (for example: siglip, pixtral, or qwen) and adding a model-specific projector.
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 61c51c8fd..5f9cbf1e3 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -589,7 +589,7 @@ class GGUFType:
class MODEL_ARCH(IntEnum):
- MMPROJ = auto() # dummy arch for clip.cpp
+ MMPROJ = auto() # dummy arch for mmproj
LLAMA = auto()
LLAMA4 = auto()
DECI = auto()
@@ -1508,7 +1508,7 @@ class MODEL_TENSOR(IntEnum):
MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
- MODEL_ARCH.MMPROJ: "clip", # dummy arch for clip.cpp
+ MODEL_ARCH.MMPROJ: "clip", # dummy arch for mmproj files (tools/mtmd)
MODEL_ARCH.LLAMA: "llama",
MODEL_ARCH.LLAMA4: "llama4",
MODEL_ARCH.DECI: "deci",
@@ -2200,14 +2200,14 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_GEN_CODE_EMBD: "a.gen.code.embd",
MODEL_TENSOR.A_GEN_CODE_HEAD: "a.gen.code.head",
MODEL_TENSOR.A_GEN_CODE_OUT_EMBD: "a.gen.code.out_embd",
- MODEL_TENSOR.A_GEN_CODE_ATTN_NORM: "a.gen.code.blk.{bid}.ln1", # reuses the generic clip.cpp block loader (TN_LN_1)
+ MODEL_TENSOR.A_GEN_CODE_ATTN_NORM: "a.gen.code.blk.{bid}.ln1", # reuses the generic mtmd block loader (TN_LN_1)
MODEL_TENSOR.A_GEN_CODE_ATTN_Q: "a.gen.code.blk.{bid}.attn_q",
MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM: "a.gen.code.blk.{bid}.attn_q_norm",
MODEL_TENSOR.A_GEN_CODE_ATTN_K: "a.gen.code.blk.{bid}.attn_k",
MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM: "a.gen.code.blk.{bid}.attn_k_norm",
MODEL_TENSOR.A_GEN_CODE_ATTN_V: "a.gen.code.blk.{bid}.attn_v",
MODEL_TENSOR.A_GEN_CODE_ATTN_OUT: "a.gen.code.blk.{bid}.attn_out",
- MODEL_TENSOR.A_GEN_CODE_FFN_NORM: "a.gen.code.blk.{bid}.ln2", # reuses the generic clip.cpp block loader (TN_LN_2)
+ MODEL_TENSOR.A_GEN_CODE_FFN_NORM: "a.gen.code.blk.{bid}.ln2", # reuses the generic mtmd block loader (TN_LN_2)
MODEL_TENSOR.A_GEN_CODE_FFN_GATE: "a.gen.code.blk.{bid}.ffn_gate",
MODEL_TENSOR.A_GEN_CODE_FFN_UP: "a.gen.code.blk.{bid}.ffn_up",
MODEL_TENSOR.A_GEN_CODE_FFN_DOWN: "a.gen.code.blk.{bid}.ffn_down",
diff --git a/skills/add-new-model/SKILL.md b/skills/add-new-model/SKILL.md
index 710a1ebb4..3e4d178eb 100644
--- a/skills/add-new-model/SKILL.md
+++ b/skills/add-new-model/SKILL.md
@@ -50,7 +50,7 @@ Skill-specific addition: before writing `src/models/.cpp`, read at least 1
## Step 4 - Optional: multimodal encoder
-Only do this if the contributor flagged a vision/audio encoder in Step 0. Follow HOWTO-add-model.md section 4 and `docs/multimodal.md` for the actual touch points (`MmprojModel` subclass, `clip.cpp`, `mtmd.cpp`, encoder graph in `tools/mtmd/models`, etc.).
+Only do this if the contributor flagged a vision/audio encoder in Step 0. Follow HOWTO-add-model.md section 4 and `docs/multimodal.md` for the actual touch points (`MmprojModel` subclass, `mtmd-model.cpp`, `mtmd-subcontext.cpp`, `mtmd.cpp`, encoder graph in `tools/mtmd/models`, etc.).
Skill-specific addition, and read this carefully: **whether the multimodal encoder can be bundled into the same PR as the base text-model support depends on how conventional the change is.** It's OK to bundle it if the encoder support is conventional - i.e. no new infra or logic is needed, it's just a new cgraph reusing existing preprocessing/projector machinery (e.g. siglip/pixtral/qwen with just a new projector). If it requires anything beyond that - a new preprocessor, non-standard projector logic, or changes to shared `libmtmd` infra/logic - STOP, tell the contributor this is non-conventional, and have them land the text model first with the encoder as a dedicated follow-up PR. Do not let this decision pass silently - call it out explicitly to the contributor before writing any `clip.cpp`/`mtmd.cpp` code.
diff --git a/src/models/clip.cpp b/src/models/clip.cpp
index 537766aeb..3bf7ac3b8 100644
--- a/src/models/clip.cpp
+++ b/src/models/clip.cpp
@@ -14,5 +14,5 @@ void llama_model_clip::load_arch_tensors(llama_model_loader &) {
[[noreturn]]
std::unique_ptr llama_model_clip::build_arch_graph(const llm_graph_params &) const {
- GGML_ABORT("CLIP has no inference graph via llama_model dispatch; runtime lives in tools/mtmd/clip.cpp");
+ GGML_ABORT("CLIP has no inference graph via llama_model dispatch; runtime lives in tools/mtmd");
}
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 016fca3c3..a17f6b43e 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -6561,7 +6561,7 @@ struct test_rope : public test_case {
a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0);
ggml_set_name(a, "view_of_a");
} else if (v == 2) {
- // second-half slice along dim 0 (mimics build_rope_2d in clip.cpp).
+ // second-half slice along dim 0 (mimics build_rope_2d in tools/mtmd/mtmd-graph.cpp).
// The non-zero view offset (ne_a[0] * elem_size) often produces a
// non-aligned buffer offset, which exercises backends' alignment paths.
auto ne = ne_a; ne[0] *= 2;
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp
index e9c6ac856..a0201bdde 100644
--- a/tests/test-mtmd-impl.cpp
+++ b/tests/test-mtmd-impl.cpp
@@ -3,6 +3,7 @@
#include "ggml-backend.h"
#include "ggml-cpp.h"
#include "mtmd-audio.h"
+#include "mtmd-subcontext.h"
#include "mtmd-helper.h"
#include "mtmd-image.h"
#include "mtmd-internal.h"
@@ -52,12 +53,12 @@ struct test_registry {
//
MAKE_TEST(test_image_preprocessor_longest_edge) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.image_longest_edge = 1540;
hparams.patch_size = 14;
hparams.n_merge = 2;
- const std::vector> cases = {
+ const std::vector> cases = {
{ { 463, 463 }, { 476, 476 } },
{ { 463, 300 }, { 476, 308 } },
{ { 448, 280 }, { 448, 280 } },
@@ -65,7 +66,7 @@ MAKE_TEST(test_image_preprocessor_longest_edge) {
};
for (const auto & [input, expected] : cases) {
- const clip_image_size actual = mtmd_image_preprocessor_longest_edge::get_target_size(hparams, input);
+ const mtmd_image_size actual = mtmd_image_preprocessor_longest_edge::get_target_size(hparams, input);
const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height);
t.assert_equal(name + " width", expected.width, actual.width);
t.assert_equal(name + " height", expected.height, actual.height);
@@ -75,7 +76,7 @@ MAKE_TEST(test_image_preprocessor_longest_edge) {
MAKE_TEST(test_image_preprocessor_dyn_size) {
// { patch_size, image size, expected size }
// a side that is an even number plus .5 merged patches rounds down, like Python round() in smart_resize
- const std::vector> cases = {
+ const std::vector> cases = {
{ 16, { 768, 400 }, { 768, 384 } },
{ 16, { 800, 336 }, { 800, 320 } },
{ 16, { 400, 400 }, { 384, 384 } },
@@ -84,13 +85,13 @@ MAKE_TEST(test_image_preprocessor_dyn_size) {
};
for (const auto & [patch_size, input, expected] : cases) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.patch_size = patch_size;
hparams.n_merge = 2;
hparams.image_round_half_even = true;
hparams.set_limit_image_tokens(8, 4096);
- const clip_image_size actual = mtmd_image_preprocessor_dyn_size::get_target_size(hparams, input);
+ const mtmd_image_size actual = mtmd_image_preprocessor_dyn_size::get_target_size(hparams, input);
const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height);
t.assert_equal(name + " width", expected.width, actual.width);
t.assert_equal(name + " height", expected.height, actual.height);
@@ -98,13 +99,13 @@ MAKE_TEST(test_image_preprocessor_dyn_size) {
}
MAKE_TEST(test_image_preprocessor_qwen3vl_video) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.patch_size = 16;
hparams.n_merge = 2;
const mtmd_image_preprocessor_qwen3vl preproc(hparams);
// { n_frames, frame size, expected size from Qwen3VLVideoProcessor smart_resize }
- const std::vector> cases = {
+ const std::vector> cases = {
{ 20, { 720, 358 }, { 704, 352 } }, // 720 / 32 = 22.5 rounds half to even
{ 768, { 1920, 1080 }, { 224, 128 } }, // over the longest_edge budget
{ 4, { 16, 40 }, { 32, 64 } }, // edge below the factor is upscaled first
@@ -114,7 +115,7 @@ MAKE_TEST(test_image_preprocessor_qwen3vl_video) {
};
for (const auto & [n_frames, input, expected] : cases) {
- clip_image_size actual;
+ mtmd_image_size actual;
const std::string name = std::to_string(n_frames) + " frames of " + std::to_string(input.width) + "x" + std::to_string(input.height);
if (!t.assert_true(name + " has a frame size", preproc.get_video_frame_size(n_frames, input.width, input.height, actual))) {
continue;
@@ -124,7 +125,7 @@ MAKE_TEST(test_image_preprocessor_qwen3vl_video) {
}
}
-static std::string image_size_to_str(const clip_image_size & size) {
+static std::string image_size_to_str(const mtmd_image_size & size) {
return std::to_string(size.width) + "x" + std::to_string(size.height);
}
@@ -143,8 +144,8 @@ static std::string resize_algo_to_str(resize_algo algo) {
}
// pseudo-random pixels: s = s * 1103515245 + 12345 from s = 1, one byte (s >> 16) & 255 per channel
-static clip_image_u8 make_test_image(const clip_image_size & size) {
- clip_image_u8 img;
+static mtmd_image_u8 make_test_image(const mtmd_image_size & size) {
+ mtmd_image_u8 img;
img.set_size(size, false);
uint32_t s = 1;
auto next = [&s]() {
@@ -160,7 +161,7 @@ static clip_image_u8 make_test_image(const clip_image_size & size) {
}
// FNV-1a of the pixels
-static uint32_t hash_pixels(const clip_image_u8 & img, uint32_t hash = 2166136261u) {
+static uint32_t hash_pixels(const mtmd_image_u8 & img, uint32_t hash = 2166136261u) {
for (uint8_t v : img.get_ro_buf()) {
hash = (hash ^ v) * 16777619u;
}
@@ -168,7 +169,7 @@ static uint32_t hash_pixels(const clip_image_u8 & img, uint32_t hash = 216613626
}
MAKE_TEST(test_image_preprocessor_lfm2) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.patch_size = 16;
hparams.n_merge = 2;
hparams.set_limit_image_tokens(64, 256);
@@ -178,7 +179,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) {
hparams.preproc_max_pixels_tolerance = 2.0f;
// { image size, expected tiling }
- const std::vector> cases = {
+ const std::vector> cases = {
{ { 704, 704 }, false },
// 720 / (patch_size * n_merge) is exactly 22.5, so this only matches HF
// if round_by_factor rounds half to even (22) instead of away from zero (23)
@@ -198,7 +199,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) {
}
// { image size, expected size from HF smart_resize }
- const std::vector> resize_cases = {
+ const std::vector> resize_cases = {
{ { 400, 600 }, { 384, 608 } }, // 400 / 32 = 12.5 rounds half to even
{ { 480, 336 }, { 480, 320 } },
{ { 64, 169 }, { 160, 416 } }, // 169 * beta / 32 is exactly 13 in double
@@ -211,7 +212,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) {
}
// { image size, expected grid from HF }
- const std::vector> grid_cases = {
+ const std::vector> grid_cases = {
{ { 720, 1600 }, { 2, 4 } }, // 0.45 is an exact tie between 2/4 and 2/5 in double, not in float
{ { 420, 1440 }, { 1, 3 } },
{ { 255, 2160 }, { 1, 8 } },
@@ -231,7 +232,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) {
}
MAKE_TEST(test_image_preprocessor_lfm2_torch_resize) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.patch_size = 16;
hparams.n_merge = 2;
hparams.set_limit_image_tokens(64, 256);
@@ -243,7 +244,7 @@ MAKE_TEST(test_image_preprocessor_lfm2_torch_resize) {
hparams.image_pad_rf = PAD_NONE;
// { resize algo, image size, expected hash of the tiles and the thumbnail from Lfm2VlImageProcessor (torchvision 0.29, native uint8 resize: arm64, or x86 with AVX2/AVX512) }
- const std::vector> cases = {
+ const std::vector> cases = {
{ RESIZE_ALGO_BILINEAR, { 620, 410 }, 0xf5e9c941 }, // 608x416
{ RESIZE_ALGO_BILINEAR, { 2000, 1500 }, 0x3331f56c }, // 3x2 tiles, 576x416 thumbnail
{ RESIZE_ALGO_BICUBIC, { 620, 410 }, 0x2b34eac9 },
@@ -338,14 +339,14 @@ MAKE_TEST(test_temporal_merge_grouping) {
//
MAKE_TEST(test_video_config) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.patch_size = 16;
hparams.n_merge = 2;
const mtmd_image_preprocessor_qwen3vl preproc(hparams);
// { template, n_frames and fps of the source, expected fps, n_frames and frame size }
// the 1920x1080 sizes are from Qwen3VLVideoProcessor smart_resize
- const std::vector> cases = {
+ const std::vector> cases = {
{ MTMD_VIDEO_TEMPLATE_QWEN3VL, 36000, 30.0f, 2.0f, 768, { 224, 128 } },
{ MTMD_VIDEO_TEMPLATE_LFM3, 36000, 30.0f, 2.0f, 768, { 224, 128 } },
{ MTMD_VIDEO_TEMPLATE_LFM3_OMNI, 36000, 30.0f, 2.0f, 384, { 320, 192 } },
@@ -405,13 +406,13 @@ static std::string video_sequence_to_str(testing & t, mtmd_helper_video_seq & vi
}
MAKE_TEST(test_video_sequence_qwen3vl) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.patch_size = 16;
hparams.n_merge = 2;
const mtmd_image_preprocessor_qwen3vl preproc(hparams);
// { source video, expected frame size and sequence from Qwen3VLProcessor (transformers 4.57.1) }
- const std::vector> cases = {
+ const std::vector> cases = {
{ { 720, 358, 24.0f, 240 }, { 704, 352 },
"<0.3 seconds>[0][13]<1.3 seconds>[25][38]<2.4 seconds>[50][63]<3.4 seconds>[75][88]<4.5 seconds>[101][113]"
"<5.5 seconds>[126][138]<6.6 seconds>[151][164]<7.6 seconds>[176][189]<8.6 seconds>[201][214]<9.7 seconds>[226][239]" },
@@ -439,7 +440,7 @@ MAKE_TEST(test_video_sequence_qwen3vl) {
// the soundtrack of each group comes after its frames
MAKE_TEST(test_video_sequence_lfm3_omni) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.patch_size = 16;
hparams.n_merge = 2;
const mtmd_image_preprocessor_lfm3vl preproc(hparams);
@@ -462,7 +463,7 @@ MAKE_TEST(test_video_sequence_lfm3_omni) {
//
MAKE_TEST(test_audio_preprocessor_conformer) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.n_mel_bins = 128;
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 512;
@@ -504,7 +505,7 @@ MAKE_TEST(test_audio_preprocessor_conformer) {
if (!t.assert_true("preprocess", !out.entries.empty() && preproc.compute(out))) {
return;
}
- const clip_image_f32 & mel = out.entries[0];
+ const mtmd_image_f32 & mel = out.entries[0];
t.assert_equal("n_len", 101, mel.nx());
for (const auto & [bin, frame, expected] : cases) {
@@ -515,7 +516,7 @@ MAKE_TEST(test_audio_preprocessor_conformer) {
}
MAKE_TEST(test_audio_preprocessor_threads) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.n_mel_bins = 128;
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 512;
@@ -539,7 +540,7 @@ MAKE_TEST(test_audio_preprocessor_threads) {
if (!t.assert_true("preprocess", ref.entries.size() == 1 && ref.needs_compute())) {
return;
}
- const clip_image_size shape = ref.entries[0].get_size();
+ const mtmd_image_size shape = ref.entries[0].get_size();
if (!t.assert_true("compute", preproc.compute(ref))) {
return;
}
@@ -586,7 +587,7 @@ MAKE_TEST(test_audio_log_mel_precision) {
}
std::vector out((size_t) n_frames * n_mel);
- if (!t.assert_true("log mel", clip_audio_log_mel(config, samples.data(), cache.hann_window.data(), n_frames, out.data()))) {
+ if (!t.assert_true("log mel", mtmd_audio_log_mel(config, samples.data(), cache.hann_window.data(), n_frames, out.data()))) {
return;
}
@@ -650,8 +651,8 @@ MAKE_TEST(test_audio_log_mel_blocks) {
std::vector all((size_t) n_frames * n_mel);
std::vector sub((size_t) n_sub * n_mel);
const float * window = cache.hann_window.data();
- if (!t.assert_true("all", clip_audio_log_mel(config, samples.data(), window, n_frames, all.data())) ||
- !t.assert_true("sub", clip_audio_log_mel(config, samples.data() + (size_t) off * hop, window, n_sub, sub.data()))) {
+ if (!t.assert_true("all", mtmd_audio_log_mel(config, samples.data(), window, n_frames, all.data())) ||
+ !t.assert_true("sub", mtmd_audio_log_mel(config, samples.data() + (size_t) off * hop, window, n_sub, sub.data()))) {
return;
}
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index fd8b24efd..c19815d67 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -22,11 +22,13 @@ set(MTMD_CORE_SOURCES
mtmd-helper-gen.cpp
mtmd-helper-common.h
mtmd-helper.h
- clip.cpp
- clip.h
- clip-impl.h
- clip-model.h
- clip-graph.h
+ mtmd-model.cpp
+ mtmd-subcontext.cpp
+ mtmd-graph.cpp
+ mtmd-subcontext.h
+ mtmd-impl.h
+ mtmd-model.h
+ mtmd-graph.h
)
add_library(mtmd
diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md
index 5dad86100..4ca410c67 100644
--- a/tools/mtmd/README-dev.md
+++ b/tools/mtmd/README-dev.md
@@ -7,6 +7,7 @@ Please refer to [multimodal.md](../../docs/multimodal.md) for a broader context.
In short:
- `libmtmd` started as a wrapper around `libllava` / `clip.cpp`
- Various components that used to be in `clip.cpp` are moved progressively to mtmd. For example, preprocessor is now part of mtmd
+- `clip.cpp` has since been split into `mtmd-model` (hparams and weights, one `mtmd_model` per modality), `mtmd-subcontext` (backends and compute, one `mtmd_subcontext` per modality) and `mtmd-graph` (graph builders, see `models/`). Only the GGUF format keeps the `clip.` name (metadata keys and the `clip` architecture), for compatibility with existing mmproj files
## Terminologies
@@ -15,6 +16,26 @@ In short:
- tiles / slices: for llava-uhd-style models, the preprocessor breaks a large input into smaller square images called tiles or slices
- chunk: a mtmd_input_chunk represents a preprocessed input that can then be passed through `mtmd_encode()`
+## Structure
+
+Overview of the `libmtmd` structure:
+
+- `mtmd_context` (`mtmd.cpp`): the public handle
+ - `mtmd_models`: one mmproj file, loaded by `mtmd_model_load()`
+ - `mtmd_model` `v` / `a` / `gen_a` (`mtmd-model.h`): one per modality, holds hparams, weights (`ctx_data` + `buf`) and requirements
+ - `mtmd_subcontext` `ctx_v` / `ctx_a` / `ctx_gen_a` (`mtmd-subcontext.h`): one per model, holds a const ref to it
+ - backends + `sched`: GPU/CPU backends, compute buffers, resolved flash attn
+ - `mtmd_graph` (`mtmd-graph.h`, `models/`): built for each `encode()` call, temporary
+ - `mtmd_image_preprocessor` / `mtmd_audio_preprocessor` (`mtmd-image.h`, `mtmd-audio.h`): read the hparams of their model
+- `mtmd_gen_state` / `mtmd_audio_stream`: public, one per sequence / stream
+ - `mtmd_subctx_gen_state` / `mtmd_subctx_enc_state`: allocated on the backend of a subcontext, see `gen_state_init()` / `enc_state_init()`
+
+Rules of thumb:
+- `mtmd_model` is read-only once loaded. Anything that depends on the backend (scheduler, compute buffers, flash attn resolution, warmup) belongs to `mtmd_subcontext`
+- A model declares its runtime needs (e.g. `max_nodes`, `flash_attn_auto`) in `load_hparams`, the subcontext applies them
+- Models must outlive their subcontexts, `mtmd_context` declares `models` first for that reason
+- Backends are owned by each subcontext for now; they will move to `mtmd_context` so that subcontexts can share them
+
## Pipeline
A typical pipeline of the core libmtmd is as follows:
@@ -67,21 +88,21 @@ Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system i
- Backbone model should be converted to a normal text model (loadable via `libllama`)
- If model used hard-coded embedding row ID, append them to token embeddings and assign token name for them (see `qwen3tts.py`)
- If model have a specific output logits head for audio codes (usually semantic code), keep the head as-is and pad the logits at inference time (see `src/models/qwen3vl.cpp`)
- - Sidecar models (code2wav, bigvgan, etc) must live inside the mmproj GGUF (but can be in different `clip_context` if necessary)
+ - Sidecar models (code2wav, bigvgan, etc) must live inside the mmproj GGUF (but can be in different `mtmd_subcontext` if necessary)
- Note: it should use `ggml_build_forward_select` to select graphs if multiple graphs living in the same context
- - Reuse existing GGUF metadata key name and tensor name whenever possible; think twice before adding extensive changes to GGUF writer. For example, Qwen3-TTS hard-code part of the hparams to `clip.cpp` as they won't likely to change.
+ - Reuse existing GGUF metadata key name and tensor name whenever possible; think twice before adding extensive changes to GGUF writer. For example, Qwen3-TTS hard-code part of the hparams in `mtmd-model.cpp` as they won't likely to change.
- For tensor naming:
- Prefixed with `a.*` for tensors used by speaker encoder pipeline
- Prefixed with `a.gen.*` for generation stages (code / mel-spectrogram / PCM generation)
- For GGUF metadata:
- Reuse as many existing keys as possible
- - In most cases, you can hard-code model configs in the model graph class, or in `clip_hparams`
+ - In most cases, you can hard-code model configs in the model graph class, or in `mtmd_hparams`
- If some values need to be exposed to the `mtmd_helper` layer, hard-code them in `mtmd_helper` and distinguish by pipeline and `mtmd_gen_audio_info::model_variant` if necessary
- Do NOT add new GGUF metadata or new fields to `mtmd_gen_audio_info` unless you can prove that you absolutely need them
4. Make sure most of the changes happen inside `mtmd-helper-gen.cpp`. A good PR looks like this:
- 10-20% changes is to add new backbone (text) model and conversion
- 60% changes inside `mtmd-helper-gen.cpp`
- - 10% changes inside `libmtmd` and `clip.cpp` systems
+ - 10% changes inside `libmtmd` core (`mtmd-model`, `mtmd-subcontext`, `models/`)
- The rest downstream code (CLI, server) should have no changes at all
5. Update usage documentation in `tools/tts/README.md`
diff --git a/tools/mtmd/README.md b/tools/mtmd/README.md
index 701941947..f0290aae5 100644
--- a/tools/mtmd/README.md
+++ b/tools/mtmd/README.md
@@ -32,7 +32,7 @@ Consequently, running a multimodal model typically requires two GGUF files:
As outlined in the history, `libmtmd` is the modern library designed to replace the original `llava.cpp` implementation for handling multimodal inputs.
-Built upon `clip.cpp` (similar to `llava.cpp`), `libmtmd` offers several advantages:
+Originally built upon `clip.cpp` (similar to `llava.cpp`), `libmtmd` offers several advantages:
- **Unified Interface:** Aims to consolidate interaction for various multimodal models.
- **Improved UX/DX:** Features a more intuitive API, inspired by the `Processor` class in the Hugging Face `transformers` library.
- **Flexibility:** Designed to support multiple input types (text, audio, images) while respecting the wide variety of chat templates used by different models.
diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h
deleted file mode 100644
index f8fd41ef5..000000000
--- a/tools/mtmd/clip.h
+++ /dev/null
@@ -1,200 +0,0 @@
-#pragma once
-
-#include "ggml.h"
-#include "mtmd.h"
-
-#include
-#include
-
-#include