Files
llama.cpp/tools/mtmd/mtmd-model.cpp
T
LumpiastyandClaude Opus 5.5 f58520aa4f llama: drop the model file's page cache after a copying load
With -lm none the GGUF is read() into our own buffers, but the kernel keeps
the file in page cache afterwards, charged to the pod's cgroup: about 20 GB
after a 35B load. On 2026-10-24 the Talos OOM controller killed the pod right
after a load under exactly that pressure. Those pages are pure waste once the
data is copied.

llama_file::drop_cache() wraps posix_fadvise(POSIX_FADV_DONTNEED) (Linux only,
errors logged at debug). load_all_data() drops each copied tensor's range right
after reading it, so the cache never grows past about one tensor, and drops the
whole file at the final cleanup to catch the metadata and partial pages at
tensor edges. Both only when the loader holds no mapping of the files
(mappings.empty(): not mmap, not mmap+mlock, no lazy tensors), so nothing that
is mapped is ever advised away. Split GGUFs are covered (every file in files).
clip.cpp always copies the mmproj via ifstream, so it drops the whole file
after loading it.

Measured on the pod, gemma-4-E2B UD-Q4_K_XL (2.5 GiB) + mmproj BF16 (0.9 GiB),
CPU only, -lm none, cold cache, same binary with the drops disabled as the
control (4 rounds, rotated order):
- file pages resident after load (mincore): 2431-2444 / 940 MiB -> 0 / 0
- peak model-file residency during load: 2.5 GiB -> 1.2-1.5 GiB (largest tensor)
- pod cgroup file after load: +3.37 GiB -> +0; host MemFree after load
  0.6 GB -> 2.5-2.8 GB (MemAvailable counts clean cache, so it cannot show this)
- load time: 5.87 s -> 6.82 s median (+0.95 s); a post-load-only drop is
  6.56 s (+0.68 s). The cost is synchronous page freeing that reclaim would
  otherwise do later; -lm dio (direct IO) costs the same (+0.9 s) and does not
  cover metadata or the mmproj.
- greedy output byte-identical; -lm mmap keeps the model file fully resident
  and mapped (2498 MiB, no refaults); -lzm on skips the drop (file mapped).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-21 19:39:51 +01:00

5395 lines
331 KiB
C++

#include "mtmd-impl.h"
#include "mtmd-model.h"
#include "models/models.h"
#include "ggml.h"
#include "ggml-cpp.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include "gguf.h"
#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <map>
#include <random>
#include <stdexcept>
#include <unordered_set>
#include <vector>
#include <cinttypes>
#include <limits>
#include <array>
#include <functional>
#include <float.h>
#if defined(__linux__)
#include <fcntl.h>
#include <unistd.h>
#endif
//
// mtmd_model_loader
//
struct mtmd_model_loader {
ggml_context_ptr ctx_meta;
gguf_context_ptr ctx_gguf;
std::string fname;
size_t model_size = 0; // in bytes
bool has_vision = false;
bool has_audio = false;
bool has_gen_audio = false;
mtmd_progress_callback progress_callback = nullptr;
void * progress_callback_user_data = nullptr;
mtmd_model_loader(const char * fname,
bool skip_tensors = false,
mtmd_progress_callback progress_cb = nullptr,
void * progress_user_data = nullptr)
: fname(fname),
progress_callback(progress_cb),
progress_callback_user_data(progress_user_data) {
struct ggml_context * meta = nullptr;
struct gguf_init_params params = {
/*.no_alloc = */ true,
/*.ctx = */ &meta,
};
ctx_gguf = gguf_context_ptr(gguf_init_from_file(fname, params));
if (!ctx_gguf.get()) {
throw std::runtime_error(mtmd_format("%s: failed to load mmproj model from %s. Does this file exist?\n", __func__, fname));
}
ctx_meta.reset(meta);
const int n_tensors = gguf_get_n_tensors(ctx_gguf.get());
// print gguf info
{
std::string name;
get_string(KEY_NAME, name, false);
std::string description;
get_string(KEY_DESCRIPTION, description, false);
LOG_INF("%s: model name: %s\n", __func__, name.c_str());
LOG_INF("%s: description: %s\n", __func__, description.c_str());
LOG_INF("%s: GGUF version: %d\n", __func__, gguf_get_version(ctx_gguf.get()));
LOG_INF("%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx_gguf.get()));
LOG_INF("%s: n_tensors: %d\n", __func__, n_tensors);
LOG_INF("%s: n_kv: %d\n", __func__, (int)gguf_get_n_kv(ctx_gguf.get()));
LOG_INF("\n");
}
// modalities
{
get_bool(KEY_HAS_VISION_ENC, has_vision, false);
get_bool(KEY_HAS_AUDIO_ENC, has_audio, false);
get_bool(KEY_HAS_GEN_AUDIO_ENC, has_gen_audio, false);
if (has_vision) {
LOG_INF("%s: has vision encoder\n", __func__);
}
if (has_audio) {
LOG_INF("%s: has audio encoder\n", __func__);
}
if (has_gen_audio) {
LOG_INF("%s: has audio generation (gen) encoder\n", __func__);
}
}
// tensors
if (!skip_tensors) {
for (int i = 0; i < n_tensors; ++i) {
const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);
const size_t offset = gguf_get_tensor_offset(ctx_gguf.get(), i);
enum ggml_type type = gguf_get_tensor_type(ctx_gguf.get(), i);
ggml_tensor * cur = ggml_get_tensor(meta, name);
size_t tensor_size = ggml_nbytes(cur);
model_size += tensor_size;
LOG_DBG("%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, offset=%zu, shape:[%" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "], type = %s\n",
__func__, i, ggml_n_dims(cur), cur->name, tensor_size, offset, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3], ggml_type_name(type));
}
}
}
void load_hparams(mtmd_model & model, mtmd_modality modality) {
auto & hparams = model.hparams;
std::string log_ffn_op; // for logging
// sanity check
if (modality == MTMD_MODALITY_VISION) {
GGML_ASSERT(has_vision);
} else if (modality == MTMD_MODALITY_AUDIO) {
GGML_ASSERT(has_audio);
} else if (modality == MTMD_MODALITY_GEN_AUDIO) {
GGML_ASSERT(has_gen_audio);
}
model.modality = modality;
// projector type
std::string proj_type;
{
// default key
get_string(KEY_PROJ_TYPE, proj_type, false);
// for models with mixed modalities
if (proj_type.empty()) {
if (modality == MTMD_MODALITY_VISION) {
get_string(KEY_VISION_PROJ_TYPE, proj_type, false);
} else if (modality == MTMD_MODALITY_AUDIO) {
get_string(KEY_AUDIO_PROJ_TYPE, proj_type, false);
} else if (modality == MTMD_MODALITY_GEN_AUDIO) {
get_string(KEY_GEN_AUDIO_PROJ_TYPE, proj_type, false);
} else {
GGML_ABORT("unknown modality");
}
}
model.proj_type = projector_type_from_string(proj_type);
if (model.proj_type == PROJECTOR_TYPE_UNKNOWN) {
throw std::runtime_error(mtmd_format("%s: unknown projector type: %s\n", __func__, proj_type.c_str()));
}
// correct arch for multimodal models (legacy method)
if (model.proj_type == PROJECTOR_TYPE_QWEN25O) {
model.proj_type = modality == MTMD_MODALITY_VISION
? PROJECTOR_TYPE_QWEN25VL
: PROJECTOR_TYPE_QWEN2A;
}
}
const bool is_vision = model.modality == MTMD_MODALITY_VISION;
const bool is_audio = model.modality == MTMD_MODALITY_AUDIO;
const bool is_gen_audio = model.modality == MTMD_MODALITY_GEN_AUDIO;
// other hparams
{
const char * prefix = is_vision ? "vision" : (is_audio ? "audio" : "gen.audio");
get_u32(mtmd_format(KEY_N_EMBD, prefix), hparams.n_embd);
get_u32(mtmd_format(KEY_N_HEAD, prefix), hparams.n_head);
get_u32(mtmd_format(KEY_N_EMBD_HEAD, prefix), hparams.n_embd_head, false);
get_u32(mtmd_format(KEY_N_FF, prefix), hparams.n_ff);
get_u32(mtmd_format(KEY_N_BLOCK, prefix), hparams.n_layer);
get_u32(mtmd_format(KEY_PROJ_DIM, prefix), hparams.projection_dim);
get_f32(mtmd_format(KEY_LAYER_NORM_EPS, prefix), hparams.eps);
// n_head_kv is optional (for GQA), default to n_head
hparams.n_head_kv = hparams.n_head;
get_u32(mtmd_format(KEY_N_HEAD_KV, prefix), hparams.n_head_kv, false);
if (is_vision) {
get_u32(KEY_IMAGE_SIZE, hparams.image_size);
get_u32(KEY_PATCH_SIZE, hparams.patch_size);
get_u32(KEY_MAX_SLICE_NUMS, hparams.max_slice_nums, false);
get_i32(KEY_MINICPMV_VERSION, hparams.minicpmv_version, false); // legacy
get_u32(KEY_MINICPMV_QUERY_NUM, hparams.minicpmv_query_num, false);
if (hparams.minicpmv_query_num == 0) {
// Fallback to hardcoded values for legacy models
if (hparams.minicpmv_version == 3) {
hparams.minicpmv_query_num = 64;
} else if (hparams.minicpmv_version == 4) {
hparams.minicpmv_query_num = 64;
} else if (hparams.minicpmv_version == 5) {
hparams.minicpmv_query_num = 64;
} else if (hparams.minicpmv_version == 6) {
hparams.minicpmv_query_num = 64;
} else if (hparams.minicpmv_version == 100045) {
hparams.minicpmv_query_num = 64;
} else {
hparams.minicpmv_query_num = 96;
}
}
} else if (is_audio) {
get_u32(KEY_A_NUM_MEL_BINS, hparams.n_mel_bins);
// some hparams are unused, but still need to set to avoid issues
hparams.image_size = 0;
hparams.patch_size = 1;
} else if (is_gen_audio) {
// these are unused, but still need to be set to avoid issues
hparams.image_size = 0;
hparams.patch_size = 1;
get_string(KEY_GEN_AUDIO_VARIANT, hparams.gen_model_variant, false);
get_string(KEY_GEN_AUDIO_DEFAULT_VOICE, hparams.gen_default_voice, false);
} else {
GGML_ASSERT(false && "unknown modality");
}
// for pinpoints, we need to convert it into a list of resolution candidates
{
std::vector<int> pinpoints;
get_arr_int(KEY_IMAGE_GRID_PINPOINTS, pinpoints, false);
if (pinpoints.size() % 2 != 0) {
throw std::runtime_error(mtmd_format("%s: image_grid_pinpoints must have an even number of elements, got %zu\n", __func__, pinpoints.size()));
}
if (!pinpoints.empty()) {
for (size_t i = 0; i < pinpoints.size(); i += 2) {
hparams.image_res_candidates.push_back({
pinpoints[i],
pinpoints[i+1],
});
}
}
}
// default warmup value
hparams.warmup_image_size = hparams.image_size;
{
bool use_gelu = false;
bool use_silu = false;
get_bool(KEY_USE_GELU, use_gelu, false);
get_bool(KEY_USE_SILU, use_silu, false);
if (use_gelu && use_silu) {
throw std::runtime_error(mtmd_format("%s: both use_gelu and use_silu are set to true\n", __func__));
}
if (use_gelu) {
hparams.ffn_op = FFN_GELU;
log_ffn_op = "gelu";
} else if (use_silu) {
hparams.ffn_op = FFN_SILU;
log_ffn_op = "silu";
} else {
hparams.ffn_op = FFN_GELU_QUICK;
log_ffn_op = "gelu_quick";
}
}
{
std::string mm_patch_merge_type;
get_string(KEY_MM_PATCH_MERGE_TYPE, mm_patch_merge_type, false);
if (mm_patch_merge_type == "spatial_unpad") {
hparams.mm_patch_merge_type = PATCH_MERGE_SPATIAL_UNPAD;
}
}
if (is_vision) {
std::vector<float> image_mean;
std::vector<float> image_std;
get_arr_f32(KEY_IMAGE_MEAN, image_mean, false);
get_arr_f32(KEY_IMAGE_STD , image_std, false);
if (image_mean.size() < 3 || image_std.size() < 3) {
throw std::runtime_error(mtmd_format("%s: image_mean/image_std arrays must have at least 3 elements, got %zu and %zu\n", __func__, image_mean.size(), image_std.size()));
}
for (int i = 0; i < 3; ++i) {
hparams.image_mean[i] = image_mean[i];
hparams.image_std[i] = image_std[i];
}
}
// Load the vision/audio feature layer indices if they are explicitly provided
// NOTE: gguf conversions should standardize the values of the vision feature layer to be non-negative, since we use -1 to mark values as unset here.
get_arr_int(mtmd_format(KEY_FEATURE_LAYERS, prefix), hparams.feature_layers, false);
for (const auto & v : hparams.feature_layers) {
if (v > (int) hparams.n_layer) {
throw std::runtime_error(mtmd_format("%s: feature layer index %d is out of range (n_layer: %d)",
__func__, v, hparams.n_layer));
}
}
// model-specific params
switch (model.proj_type) {
case PROJECTOR_TYPE_MLP:
case PROJECTOR_TYPE_MLP_NORM:
case PROJECTOR_TYPE_LDP:
case PROJECTOR_TYPE_LDPV2:
case PROJECTOR_TYPE_COGVLM:
{
hparams.has_llava_projector = model.proj_type != PROJECTOR_TYPE_COGVLM;
hparams.image_pad_color = {122, 116, 104};
if (!hparams.image_res_candidates.empty()) {
hparams.image_resize_pad = PAD_CEIL;
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
} else {
// llava-1.6 default params
hparams.image_pad_ov = PAD_NONE;
hparams.image_pad_rf = PAD_CEIL;
hparams.image_pad_color_rf = {122, 116, 104};
}
} break;
case PROJECTOR_TYPE_GLM_EDGE:
{
hparams.image_resize_pad = PAD_CEIL;
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
} break;
case PROJECTOR_TYPE_MINICPMV:
{
// use default llava-uhd preprocessing params
if (hparams.minicpmv_version == 0) {
hparams.minicpmv_version = 2; // default to 2 if not set
}
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
// MiniCPM-V 4.6/4.7 unified merger projector
// ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension
hparams.n_merge = 4;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
GGML_ASSERT(hparams.n_merge == 2 || hparams.n_merge == 4);
// no padding: the reference stretches the refined image to the target size
hparams.image_pad_ov = PAD_NONE;
hparams.image_pad_rf = PAD_NONE;
// borrow wa_layer_indexes for vit_merger insertion point
std::vector<int> wa_layer_indexes_vec;
get_arr_int(KEY_WIN_ATTN_LAYER_INDEXES, wa_layer_indexes_vec, false);
if (!wa_layer_indexes_vec.empty()) {
const int insert_lid = wa_layer_indexes_vec[0];
if (insert_lid < 0 || insert_lid >= (int) hparams.n_layer) {
throw std::runtime_error(mtmd_format("%s: layer index %d is out of range (n_layer: %d)",
__func__, insert_lid, hparams.n_layer));
}
hparams.insert_layer_id = insert_lid;
}
} break;
case PROJECTOR_TYPE_INTERNVL:
{
// use default llava-uhd preprocessing params
// older version of internvl doesn't have min/max tiles, we need to provide default values for them to avoid issues
hparams.preproc_min_tiles = 1;
hparams.preproc_max_tiles = 12;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false);
get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false);
GGML_ASSERT(hparams.preproc_min_tiles <= hparams.preproc_max_tiles && hparams.preproc_max_tiles < INT32_MAX);
set_internvl_dhr_res_candidates(model);
} break;
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
{
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
} break;
case PROJECTOR_TYPE_NEMOTRON_OMNI:
{
hparams.ffn_op = FFN_GELU_ERF;
log_ffn_op = "gelu_erf";
hparams.image_resize_pad = PAD_NONE;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
hparams.warmup_image_size = hparams.image_size;
} break;
case PROJECTOR_TYPE_PARAKEET:
{
get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor);
GGML_ASSERT(hparams.subsampling_factor == 8 &&
"subsampling_factor must match the conv strides in mtmd_graph_parakeet::build()");
get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size);
GGML_ASSERT(hparams.audio_conv_kernel_size > 0 && hparams.audio_conv_kernel_size % 2 == 1 &&
"audio_conv_kernel_size must be a positive odd integer");
hparams.audio_chunk_len = 0;
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 512;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
} break;
case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
{
get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor);
get_u32(KEY_A_ROPE_DIMS, hparams.audio_rope_dims);
get_u32(KEY_A_SPK_N_BLOCK, hparams.spk_n_layer);
get_u32(KEY_A_SPK_N_HEAD, hparams.spk_n_head);
if (hparams.subsampling_factor != 8) {
throw std::runtime_error(mtmd_format("%s: unsupported subsampling factor %d\n", __func__, hparams.subsampling_factor));
}
if (hparams.spk_n_layer <= 0 || hparams.spk_n_layer > (int) gguf_get_n_tensors(ctx_gguf.get())) {
throw std::runtime_error(mtmd_format("%s: invalid speaker block count %d\n", __func__, hparams.spk_n_layer));
}
hparams.audio_chunk_len = 0;
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 512;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
// one full window
using nt = mtmd_graph_nemotron_transcribe;
hparams.warmup_audio_size = (nt::window_left + nt::window_core + nt::window_right) * hparams.subsampling_factor;
} break;
case PROJECTOR_TYPE_IDEFICS3:
{
// use default llava-uhd preprocessing params
hparams.image_resize_algo = RESIZE_ALGO_LANCZOS;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false);
hparams.set_limit_image_tokens();
} break;
case PROJECTOR_TYPE_COHERE2V:
{
hparams.image_pad_rf = PAD_NONE;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles);
if (hparams.preproc_max_tiles <= 0 || hparams.preproc_max_tiles > 256) {
throw std::runtime_error(mtmd_format("%s: preproc_max_tiles (%d) must be in range [1, 256]\n", __func__, hparams.preproc_max_tiles));
}
} break;
case PROJECTOR_TYPE_LFM2:
case PROJECTOR_TYPE_D1OMNI_V:
{
// default for older GGUFs
std::string resize_algo = "bilinear";
get_string(KEY_IMAGE_RESIZE_ALGO, resize_algo, false);
// Lfm2VlImageProcessor and d1-omni vision.py resize bilinear and bicubic with torchvision
if (resize_algo == "bilinear") {
hparams.set_image_resize_algo(RESIZE_ALGO_BILINEAR, true);
} else if (resize_algo == "bicubic") {
hparams.set_image_resize_algo(RESIZE_ALGO_BICUBIC, true);
} else if (resize_algo == "lanczos") {
hparams.set_image_resize_algo(RESIZE_ALGO_LANCZOS);
} else {
throw std::runtime_error(mtmd_format("%s: unsupported image resize algo: %s\n", __func__, resize_algo.c_str()));
}
hparams.image_resize_algo_rf = hparams.image_resize_algo;
hparams.image_resize_algo_ov = hparams.image_resize_algo;
// the tiles stretch the image to the grid (Lfm2VlImageProcessor, d1-omni vision.py)
hparams.image_pad_rf = PAD_NONE;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
// defaults for GGUFs converted before tiling params were written
// ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json
hparams.preproc_min_tiles = 2;
hparams.preproc_max_tiles = 10;
hparams.preproc_tile_size = 512;
hparams.preproc_max_pixels_tolerance = 2.0f;
get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false);
get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false);
get_u32(KEY_PREPROC_TILE_SIZE, hparams.preproc_tile_size, false);
get_f32(KEY_PREPROC_MAX_PIXELS_TOLERANCE, hparams.preproc_max_pixels_tolerance, false);
GGML_ASSERT(hparams.preproc_min_tiles >= 1 && hparams.preproc_min_tiles <= hparams.preproc_max_tiles);
hparams.set_limit_image_tokens(64, 256);
} break;
case PROJECTOR_TYPE_PHI4:
{
hparams.n_merge = 1;
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
hparams.set_warmup_n_tokens(16*16);
} break;
case PROJECTOR_TYPE_PIXTRAL:
{
// ref: https://huggingface.co/mistral-community/pixtral-12b/blob/main/preprocessor_config.json
// TODO: verify the image_min_tokens
hparams.n_merge = 1; // the original pixtral does not use patch merging
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.rope_theta = 10000.0f;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false);
if (hparams.image_longest_edge > 0) {
const int64_t patch_size = int64_t(hparams.patch_size) * hparams.n_merge;
if (patch_size <= 0) {
throw std::runtime_error("Pixtral merged patch size must be positive");
}
const int64_t n_tokens_per_side =
(hparams.image_longest_edge + patch_size - 1) / patch_size;
if (n_tokens_per_side * patch_size > std::sqrt(std::numeric_limits<int>::max())) {
throw std::runtime_error("Pixtral preprocessing image size is too large");
}
hparams.image_resize_pad = PAD_NONE;
hparams.image_floor_before_align = true;
hparams.set_limit_image_tokens(0, n_tokens_per_side * n_tokens_per_side);
} else {
hparams.set_limit_image_tokens(8, 1024);
}
hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_LIGHTONOCR:
{
hparams.n_merge = 1;
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.rope_theta = 10000.0f;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
hparams.image_longest_edge = hparams.image_size;
get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false);
hparams.set_limit_image_tokens();
hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_DOTS_OCR:
{
hparams.rope_theta = 10000.0f;
hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding
hparams.image_round_half_even = true; // smart_resize uses Python round()
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_DOTS3NOTE_V:
{
hparams.rope_theta = 10000.0f;
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
get_u32(KEY_VISION_N_EXPERT_USED, hparams.n_expert_used);
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_DOTS3NOTE_A:
{
hparams.rope_theta = 10000.0f;
hparams.audio_chunk_len = 60; // in seconds
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 400;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
} break;
case PROJECTOR_TYPE_KIMIVL:
{
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.rope_theta = 10000.0f;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
// TODO: check kimivl preprocessor for exact values
hparams.set_limit_image_tokens(8, 1024);
hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_KIMIK25:
case PROJECTOR_TYPE_KIMIK31:
{
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.rope_theta = 10000.0f;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
int min_pixels = 0, max_pixels = 0;
get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels, false);
get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels, false);
if (min_pixels > 0 && max_pixels > 0) {
hparams.image_min_pixels = min_pixels;
hparams.image_max_pixels = max_pixels;
hparams.warmup_image_size = static_cast<int>(std::sqrt(max_pixels));
} else {
hparams.set_limit_image_tokens(2, 4096);
}
if (model.proj_type == PROJECTOR_TYPE_KIMIK31) {
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
}
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.image_pad_color = {127, 127, 127};
hparams.rope_theta = 10000.0f;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
hparams.dsv4_max_n_token = 384;
hparams.dsv4_max_wh_ratio = 8;
const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
// handle min/max token counts from CLI
if (hparams.custom_image_min_tokens > 0) {
hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area;
}
if (hparams.custom_image_max_tokens > 0) {
// the cap is on the whole token block, keep some room for the resize solver
hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16);
}
hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area;
// a small custom max token count also lowers the min-pixel upscale threshold
hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels);
// avoid OOM on warmup
const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token));
hparams.set_warmup_n_tokens(warmup_side * warmup_side);
} break;
case PROJECTOR_TYPE_DEEPSEEK41V:
{
// ref: load_image in inference/image_processor.py (V4.1), PIL ImageOps.pad on gray 127
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.image_pad_color = {127, 127, 127};
hparams.rope_theta = 10000.0f;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
// vision_max_n_token = 1024, vision_max_wh_ratio = null (converter asserts both)
hparams.dsv4_max_n_token = 1024;
hparams.dsv4_max_wh_ratio = 0;
const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
if (hparams.custom_image_min_tokens > 0) {
hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area;
}
if (hparams.custom_image_max_tokens > 0) {
// the cap is on the whole token block (newlines + start/end included)
hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16);
}
hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area;
hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels);
const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token));
hparams.set_warmup_n_tokens(warmup_side * warmup_side);
} break;
case PROJECTOR_TYPE_GEMMA3:
{
// default value (used by all model sizes in gemma 3 family)
// number of patches for each **side** is reduced by a factor of 4
hparams.n_merge = 4;
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
// test model (tinygemma3) has a different value, we optionally read it
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
} break;
case PROJECTOR_TYPE_GEMMA4V:
case PROJECTOR_TYPE_GEMMA4UV:
{
hparams.rope_theta = 100.0f;
hparams.ffn_op = FFN_GELU;
log_ffn_op = "gelu";
hparams.n_merge = 3; // pooling_kernel_size
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.image_resize_pad = PAD_NONE;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
if (model.proj_type == PROJECTOR_TYPE_GEMMA4UV) {
// for "unified" variant, we directly use a bigger patch size, because the "token merging" is done directly on conv layer
hparams.patch_size = hparams.patch_size * hparams.n_merge;
hparams.n_merge = 1;
}
hparams.set_limit_image_tokens(70, 1120);
hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_GEMMA3NV:
{
// Gemma3n uses MobileNetV5 which produces 256 tokens (16x16)
// Similar configuration to Gemma3
hparams.n_merge = 1; // MobileNetV5 handles resizing internally
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
} break;
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_LING3VL:
{
hparams.n_merge = 2; // default value for Qwen 2 and 2.5
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding
hparams.image_round_half_even = true; // smart_resize uses Python round()
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
if (hparams.n_merge != 2) {
// the temporal merge input and the merger use a 2x2 patch grid
throw std::runtime_error(mtmd_format("%s: spatial_merge_size (%d) must be 2\n", __func__, hparams.n_merge));
}
get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, model.proj_type == PROJECTOR_TYPE_QWEN25VL); // only 2.5 requires it
// ref: https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct/blob/main/preprocessor_config.json
hparams.set_limit_image_tokens(8, 4096);
// optional limits of the model, the custom values take precedence
if (hparams.custom_image_min_tokens <= 0) {
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels, false);
}
if (hparams.custom_image_max_tokens <= 0) {
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels, false);
}
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
const int warn_min_pixels = 1024 * hparams.n_merge * hparams.n_merge * hparams.patch_size * hparams.patch_size;
if (hparams.image_min_pixels < warn_min_pixels) {
LOG_WRN("%s: Qwen-VL models require at minimum 1024 image tokens to function correctly on grounding tasks\n", __func__);
LOG_WRN("%s: if you encounter problems with accuracy, try adding --image-min-tokens 1024\n", __func__);
LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__);
}
} break;
case PROJECTOR_TYPE_MAGEVL:
{
hparams.ffn_op = FFN_GELU_ERF;
log_ffn_op = "gelu_erf";
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding
hparams.image_round_half_even = true; // smart_resize uses Python round()
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_SENSENOVA_U1:
{
hparams.n_merge = 2;
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
hparams.rope_theta = 10000.0f;
hparams.set_limit_image_tokens(16, 4096);
if (hparams.n_merge != 2) {
throw std::runtime_error(mtmd_format("%s: SenseNova U1 requires spatial_merge_size=2, got %u", __func__, hparams.n_merge));
}
hparams.set_warmup_n_tokens(16*16);
} break;
case PROJECTOR_TYPE_LFM3VL:
{
std::string resize_algo;
get_string(KEY_IMAGE_RESIZE_ALGO, resize_algo);
// Lfm3VlImageProcessor resizes with torchvision
if (resize_algo == "bilinear") {
hparams.set_image_resize_algo(RESIZE_ALGO_BILINEAR, true);
} else if (resize_algo == "bicubic") {
hparams.set_image_resize_algo(RESIZE_ALGO_BICUBIC, true);
} else {
throw std::runtime_error(mtmd_format("%s: unsupported image resize algo: %s\n", __func__, resize_algo.c_str()));
}
// smart_resize stretches to the aligned size, no letterbox
hparams.image_resize_pad = PAD_NONE;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge);
// token counts and the projector graph assume a 2x2 merge
if (hparams.n_merge != 2) {
throw std::runtime_error(mtmd_format("%s: unsupported spatial_merge_size %d for lfm3vl, only 2 is supported\n", __func__, hparams.n_merge));
}
get_u32(mtmd_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
hparams.rope_theta = 1000000.0f; // vision_rope_theta
// HF processor size; the max is also capped at 4096 tokens as an OOM guard
int min_pixels = 0;
int max_pixels = 0;
get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels);
if (min_pixels <= 0 || max_pixels < min_pixels) {
throw std::runtime_error(mtmd_format("%s: invalid image min/max pixels %d/%d\n", __func__, min_pixels, max_pixels));
}
const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
hparams.set_limit_image_tokens(min_pixels / patch_area, std::min(4096, max_pixels / patch_area));
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_MINIMAX_M3:
{
hparams.n_merge = 2; // spatial_merge_size
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.image_resize_pad = PAD_NONE;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
// n_merge is used as a divisor in mtmd_subcontext::encode()
// (gh / n_merge); reject 0 to avoid int div-by-zero (DoS).
GGML_ASSERT(hparams.n_merge > 0);
hparams.rope_theta = 10000.0f; // vision_config.rope_theta
// MiniMax-M3: max_pixels 451584 (=672^2) -> 576 merged tokens (image_seq_length)
hparams.set_limit_image_tokens(8, 576);
hparams.set_warmup_n_tokens(16*16);
} break;
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
{
hparams.audio_sample_rate = 24000; // mono waveform, no frontend
get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size, true);
get_arr_int(KEY_A_HIDDEN_SIZES, hparams.audio_hidden_sizes, true);
get_arr_int(KEY_A_PATCH_SIZES, hparams.audio_patch_sizes, true);
get_arr_int(KEY_A_DEPTHS, hparams.audio_depths, true);
// the graph indexes the three arrays by stage and model.layers by the running depth sum
const size_t n_stage = hparams.audio_patch_sizes.size();
GGML_ASSERT(n_stage > 0);
GGML_ASSERT(hparams.audio_hidden_sizes.size() == n_stage);
GGML_ASSERT(hparams.audio_depths.size() == n_stage);
GGML_ASSERT(hparams.n_mel_bins == 1);
GGML_ASSERT(hparams.audio_conv_kernel_size >= 1);
int64_t n_blocks = 0;
for (size_t i = 0; i < n_stage; ++i) {
GGML_ASSERT(hparams.audio_hidden_sizes[i] >= 1);
GGML_ASSERT(hparams.audio_depths[i] >= 0);
n_blocks += hparams.audio_depths[i];
}
if (n_blocks != hparams.n_layer) {
throw std::runtime_error(mtmd_format("%s: sum of audio depths (%" PRId64 ") does not match n_layer (%d)\n", __func__, n_blocks, hparams.n_layer));
}
hparams.warmup_audio_size = hparams.audio_samples_per_token();
} break;
case PROJECTOR_TYPE_VIBEVOICE_ASR:
{
hparams.audio_sample_rate = 24000; // mono waveform, no frontend
hparams.audio_chunk_len = 30; // in seconds, any length works as the convs are causal
get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size, true);
get_arr_int(KEY_A_PATCH_SIZES, hparams.audio_patch_sizes, true);
get_arr_int(KEY_A_DEPTHS, hparams.audio_depths, true);
// bounds keep the receptive field below from overflowing
const size_t n_stage = hparams.audio_patch_sizes.size();
if (n_stage == 0 || n_stage > 16 || hparams.audio_depths.size() != n_stage || hparams.n_mel_bins != 1 ||
hparams.audio_conv_kernel_size < 1 || hparams.audio_conv_kernel_size > 64) {
throw std::runtime_error(mtmd_format("%s: invalid vibevoice audio hparams\n", __func__));
}
// receptive field: stem conv (kernel), downsample convs (2 * stride), blocks (kernel), head conv (kernel)
const int64_t k = hparams.audio_conv_kernel_size;
int64_t hop = 1;
int64_t rf = 0;
int64_t n_blocks = 0;
for (size_t i = 0; i < n_stage; ++i) {
const int64_t stride = hparams.audio_patch_sizes[i];
if (stride < 1 || stride > 64 || hop * stride > (1 << 20) || hparams.audio_depths[i] < 0 || hparams.audio_depths[i] > 64) {
throw std::runtime_error(mtmd_format("%s: invalid vibevoice audio hparams\n", __func__));
}
rf += ((i == 0 ? k : 2 * stride) - 1) * hop;
hop *= stride;
rf += hparams.audio_depths[i] * (k - 1) * hop;
n_blocks += hparams.audio_depths[i];
}
rf += (k - 1) * hop;
if (rf > 60 * hparams.audio_sample_rate) {
throw std::runtime_error(mtmd_format("%s: vibevoice receptive field of %" PRId64 " samples is too long\n", __func__, rf));
}
if (n_blocks != hparams.n_layer) {
throw std::runtime_error(mtmd_format("%s: sum of audio depths (%" PRId64 ") does not match n_layer (%d)\n", __func__, n_blocks, hparams.n_layer));
}
hparams.audio_n_ctx = (int32_t) ((rf + hop - 1) / hop * hop);
hparams.warmup_audio_size = (int32_t) ((int64_t) hparams.audio_chunk_len * hparams.audio_sample_rate / hop * hop) + hparams.audio_n_ctx;
} break;
case PROJECTOR_TYPE_INKLING2V:
{
hparams.ffn_op = FFN_GELU_ERF;
log_ffn_op = "gelu_erf";
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge);
hparams.rope_theta = 10000.0f;
hparams.set_limit_image_tokens(1, 4096);
hparams.set_warmup_n_tokens(32*32); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_MUSE_GLIMMER:
{
hparams.n_merge = 2; // pixel-shuffle downsample after the ViT
hparams.image_resize_algo = RESIZE_ALGO_LANCZOS;
hparams.rope_theta = 10000.0f;
hparams.muse_glimmer_patch_temporal = 2;
hparams.muse_glimmer_sparse_factor = 4; // 3 sparse layers + 1 global, repeating
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
hparams.set_limit_image_tokens(1, 4096);
hparams.set_warmup_n_tokens(32*32);
} break;
case PROJECTOR_TYPE_MIMOVL:
{
hparams.n_merge = 2; // spatial_merge_size
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding
hparams.image_round_half_even = true; // smart_resize uses Python round()
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
get_u32(mtmd_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
// 1D banded sliding-window radius (visual_token_window_size); required
get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size);
std::vector<int> pat;
get_arr_int(KEY_WA_PATTERN_MODE, pat, true);
GGML_ASSERT((int) pat.size() == hparams.n_layer && "mimovl wa_pattern_mode length must equal n_layer");
hparams.wa_pattern_mode.assign(pat.begin(), pat.end());
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_STEP3VL:
{
hparams.n_merge = 4; // two stride-2 downsamplers after patching
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
hparams.rope_theta = 10000.0f;
get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false);
if (hparams.image_longest_edge == 0) {
hparams.image_longest_edge = 3024;
}
// note: the step3vl preprocessor slices based on a fixed window grid, so it does not support custom min/max image tokens
hparams.warmup_image_size = hparams.image_size;
} break;
case PROJECTOR_TYPE_YOUTUVL:
{
hparams.n_merge = 2;
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
hparams.image_resize_pad = PAD_NONE;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true);
std::vector<int> wa_layer_indexes_vec;
get_arr_int(KEY_WIN_ATTN_LAYER_INDEXES, wa_layer_indexes_vec, true);
for (auto & layer : wa_layer_indexes_vec) {
hparams.wa_layer_indexes.insert(layer);
}
// support max_height * max_width = 8000 * 8000. 8000/16/2 = 250 image tokens
hparams.set_limit_image_tokens(1, 62500);
hparams.set_warmup_n_tokens(16*16); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_YASA2:
{
hparams.ffn_op = FFN_GELU_ERF;
log_ffn_op = "gelu_erf";
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
// reka model performs better when the image is stretched to fit
// fixed square size (no padding)
hparams.image_resize_pad = PAD_NONE;
} break;
case PROJECTOR_TYPE_GLM4V:
{
hparams.rope_theta = 10000.0f;
hparams.n_merge = 2; // default value for GLM4-V
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
hparams.set_limit_image_tokens(8, 4096);
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_GLM5V:
{
// glm4v tower with clamped SwiGLU, ceil-aligned resize and its own token budget
hparams.rope_theta = 10000.0f;
hparams.n_merge = 2;
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
get_f32(KEY_SWIGLU_CLAMP, hparams.swiglu_clamp, true);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
hparams.set_limit_image_tokens();
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_LLAMA4:
{
hparams.rope_theta = 10000.0f;
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
set_llava_uhd_res_candidates(model, 3);
} break;
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_QWEN2A:
case PROJECTOR_TYPE_HIGGS_AUDIO:
case PROJECTOR_TYPE_QWEN3A:
case PROJECTOR_TYPE_GLMA:
case PROJECTOR_TYPE_VOXTRAL:
case PROJECTOR_TYPE_MERALION:
case PROJECTOR_TYPE_MUSIC_FLAMINGO:
{
bool require_stack = model.proj_type == PROJECTOR_TYPE_ULTRAVOX ||
model.proj_type == PROJECTOR_TYPE_VOXTRAL ||
model.proj_type == PROJECTOR_TYPE_MERALION ||
model.proj_type == PROJECTOR_TYPE_GLMA;
get_u32(KEY_A_PROJ_STACK_FACTOR, hparams.proj_stack_factor, require_stack);
hparams.ffn_op = FFN_GELU_ERF;
log_ffn_op = "gelu_erf"; // temporary solution for logging
// audio preprocessing params
hparams.audio_chunk_len = 30; // in seconds
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 400;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
if (model.proj_type == PROJECTOR_TYPE_HIGGS_AUDIO) {
// each 4s chunk is encoded on its own
hparams.audio_chunk_len = 4;
hparams.warmup_audio_size = 400;
}
} break;
case PROJECTOR_TYPE_VOXTRAL_RT:
{
get_u32(KEY_A_PROJ_STACK_FACTOR, hparams.proj_stack_factor);
get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size);
hparams.rope_theta = 1000000.0f; // params.json
hparams.ffn_op = FFN_SILU;
log_ffn_op = "silu";
// audio preprocessing params
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 400;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
// 2 context frames + 8 s of new frames, see mtmd_subcontext::enc_state_init()
hparams.warmup_audio_size = 2 + 2*400;
} break;
case PROJECTOR_TYPE_MIMO_AUDIO:
{
get_u32(KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers, false);
get_arr_int(KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size, false);
if (hparams.rvq_num_quantizers <= 0) {
throw std::runtime_error(mtmd_format("%s: mimo_audio: missing %s\n", __func__, KEY_A_RVQ_NUM_QUANTIZERS));
}
if ((int) hparams.rvq_codebook_size.size() != hparams.rvq_num_quantizers) {
throw std::runtime_error(mtmd_format(
"%s: mimo_audio: %s length (%zu) must equal %s (%d)\n", __func__,
KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size.size(),
KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers));
}
hparams.ffn_op = FFN_GELU_ERF; // PyTorch F.gelu default (approximate="none")
hparams.rope_theta = 10000.0f;
// audio preprocessing params (mel spectrogram)
hparams.audio_sample_rate = 24000;
hparams.audio_n_fft = 960;
hparams.audio_window_len = 960;
hparams.audio_hop_len = 240;
get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size);
std::vector<int> wa_pattern;
get_arr_int(KEY_A_WA_PATTERN_MODE, wa_pattern, true);
if ((int) wa_pattern.size() != hparams.n_layer) {
throw std::runtime_error(mtmd_format(
"%s: mimo_audio: %s length (%zu) must equal n_layer (%d)\n", __func__,
KEY_A_WA_PATTERN_MODE, wa_pattern.size(), hparams.n_layer));
}
hparams.wa_pattern_mode.assign(wa_pattern.begin(), wa_pattern.end());
get_u32(KEY_A_LOCAL_BLOCK_COUNT, hparams.audio_local_n_layer);
get_u32(KEY_A_LOCAL_GROUP_SIZE, hparams.audio_local_group_size);
if (hparams.audio_local_group_size <= 0) {
throw std::runtime_error(mtmd_format(
"%s: mimo_audio: %s must be > 0\n", __func__, KEY_A_LOCAL_GROUP_SIZE));
}
} break;
case PROJECTOR_TYPE_ZONOS3_SPKENC:
{
// kaldi fbank front-end
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 512;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
hparams.warmup_audio_size = 1000;
} break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
// ECAPA-TDNN speaker encoder, mel front-end uses the Slaney default (fmin=0, fmax=sr/2)
hparams.audio_sample_rate = 24000;
hparams.audio_n_fft = 1024;
hparams.audio_window_len = 1024;
hparams.audio_hop_len = 256;
} break;
case PROJECTOR_TYPE_CHATTERBOX_SPKENC:
{
// kaldi fbank front-end of CAMPPlus: 25 ms / 10 ms framing, 512-point spectrum
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 512;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
} break;
case PROJECTOR_TYPE_LFM3A_GEN:
{
// TODO: CUDA has no flash attention kernel for the depthformer head dim 32, enable it once it does
model.flash_attn_auto = false;
// fixed by LiquidCodecConfig, head counts come from the tensors
hparams.rope_theta = 1000000.0f;
hparams.wav_tfm_n_layer = 8;
hparams.wav_tfm_n_embd = 512;
hparams.wav_tfm_eps = 1e-5f;
hparams.wav_tfm_rope_theta = 1000000.0f;
hparams.wav_tfm_swa = 30; // decoder_window
hparams.wav_frame_stack = 6; // stft_frame_stack, STFT frames per codec frame
// the ISTFT runs on the host, see mtmd_gen_audio_process()
hparams.audio_sample_rate = 24000;
hparams.audio_n_fft = 1280;
hparams.audio_window_len = 1280;
hparams.audio_hop_len = 320;
// speech-code sampling, the converter writes the model card values; else the LFM3-Omni card
hparams.gen_temp = 0.4f;
hparams.gen_top_k = 0;
hparams.gen_penalty_repeat = 1.2f;
hparams.gen_penalty_last_n = 64;
get_f32(KEY_GEN_AUDIO_TEMP, hparams.gen_temp, false);
get_i32(KEY_GEN_AUDIO_TOP_K, hparams.gen_top_k, false);
get_f32(KEY_GEN_AUDIO_PENALTY_REPEAT, hparams.gen_penalty_repeat, false);
get_i32(KEY_GEN_AUDIO_PENALTY_LAST_N, hparams.gen_penalty_last_n, false);
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
// TODO: hardcoded for now, read from code_predictor_config instead
hparams.rope_theta = 1000000.0f;
// reference encoder: mimi with the stride-2 downsample of the 12.5 Hz tokenizer
// seanet ratios are [8,6,5,4] in the config, the encoder reverses them
hparams.audio_sample_rate = 24000;
hparams.seanet_ratios = { 4, 5, 6, 8 };
hparams.seanet_n_stage = (int32_t) hparams.seanet_ratios.size();
hparams.mimi_downsample = 2;
hparams.mimi_tfm_context = 250;
// code2wav params
hparams.wav_tfm_n_layer = 8;
hparams.wav_tfm_n_embd = 512;
hparams.wav_tfm_n_ff = 1024;
hparams.wav_tfm_n_head = 16;
hparams.wav_tfm_n_head_kv = 16;
hparams.wav_tfm_eps = 1e-5f;
hparams.wav_tfm_rope_theta = 10000.0f;
hparams.wav_upsample_n_block = 2;
hparams.wav_dac_n_block = 4;
hparams.wav_dac_n_res = 3;
// matches the reference decoder's sliding_window (speech_tokenizer/config.json)
hparams.wav_tfm_swa = 72;
} break;
case PROJECTOR_TYPE_ZONOS3_GEN:
{
hparams.audio_sample_rate = 44100;
} break;
case PROJECTOR_TYPE_YUE2_GEN:
{
hparams.audio_sample_rate = 48000;
} break;
case PROJECTOR_TYPE_VOXCPM2_GEN:
{
hparams.audio_sample_rate = 48000;
// the flow graph unrolls up to n_steps_max LocDiT passes
model.max_nodes = 65536;
} break;
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
{
hparams.audio_sample_rate = 44100;
} break;
case PROJECTOR_TYPE_DRAMA3_GEN:
case PROJECTOR_TYPE_FISH_S2_GEN:
{
// fish_s2 fast AR, drama3 hardcodes its tower RoPE in the graph
hparams.rope_theta = 1000000.0f;
// Fish DAC decoder, codec.pth has no config
hparams.audio_sample_rate = 44100;
hparams.wav_tfm_n_layer = 8;
hparams.wav_tfm_n_embd = 1024;
hparams.wav_tfm_n_ff = 3072;
hparams.wav_tfm_n_head = 16;
hparams.wav_tfm_n_head_kv = 16;
hparams.wav_tfm_eps = 1e-5f;
hparams.wav_tfm_rope_theta = 10000.0f;
hparams.wav_tfm_swa = 128;
hparams.wav_upsample_n_block = 2;
hparams.wav_dac_n_block = 4;
hparams.wav_dac_n_res = 3;
} break;
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
case PROJECTOR_TYPE_POCKETTTS_GEN:
{
// mimi front-end takes the raw waveform, no mel
hparams.audio_sample_rate = 24000;
// seanet ratios are [6,5,4] in the config, the encoder reverses them
hparams.seanet_ratios = { 4, 5, 6 };
hparams.seanet_n_stage = (int32_t) hparams.seanet_ratios.size();
hparams.mimi_downsample = 16;
// matches the reference transformer's "context"
hparams.mimi_tfm_context = 250;
hparams.rope_theta = 10000.0f;
// flow_lm defaults, see pocket_tts/default_parameters.py
hparams.flow_n_step = 1;
hparams.gen_eos_threshold = -4.0f;
} break;
case PROJECTOR_TYPE_MOSHI:
case PROJECTOR_TYPE_MOSHI_GEN:
{
// mimi of the moshi family, see _seanet_kwargs and _transformer_kwargs of the reference
hparams.audio_sample_rate = 24000;
// seanet ratios are [8,6,5,4] in the config, the encoder reverses them
hparams.seanet_ratios = { 4, 5, 6, 8 };
hparams.seanet_n_stage = (int32_t) hparams.seanet_ratios.size();
hparams.mimi_downsample = 2;
hparams.mimi_tfm_context = 250;
hparams.rope_theta = 10000.0f;
// one step of the audio stream
hparams.warmup_audio_size = 4 * mtmd_graph_moshi::n_samples_per_frame;
} break;
case PROJECTOR_TYPE_VOXCPM2_SPKENC:
{
// AudioVAE encoder, one backbone row per patch of latents (checked against the strides at load)
hparams.audio_sample_rate = 16000;
hparams.audio_patch_sizes = { 2, 5, 8, 8, mtmd_graph_voxcpm2_gen::n_patch };
// whole patches, about 10 s like a typical reference clip
hparams.warmup_audio_size = hparams.audio_samples_per_token() * 63;
} break;
case PROJECTOR_TYPE_FISH_S2_SPKENC:
{
// Fish DAC encoder, codec.pth has no config
// one row per 2048 samples: encoder strides, then the quantizer downsample
hparams.audio_sample_rate = 44100;
hparams.audio_patch_sizes = { 2, 4, 8, 8, 2, 2 };
// whole frames, about 10 s like a typical reference clip
hparams.warmup_audio_size = hparams.audio_samples_per_token() * 215;
hparams.wav_tfm_n_head = 16;
hparams.wav_tfm_n_head_kv = 16;
hparams.wav_tfm_eps = 1e-5f;
hparams.wav_tfm_rope_theta = 10000.0f;
// drama3 shares the codec, only the backbone rows of a frame differ
std::string gen_proj;
get_string(KEY_GEN_AUDIO_PROJ_TYPE, gen_proj, false);
hparams.spkenc_drama3_rows = projector_type_from_string(gen_proj) == PROJECTOR_TYPE_DRAMA3_GEN;
} break;
case PROJECTOR_TYPE_CHATTERBOX:
{
// the classic cfm solver unrolls up to 10 steps x 2 cfg estimator evaluations in one graph
model.max_nodes = 65536;
// s3gen output rate
hparams.audio_sample_rate = 24000;
} break;
case PROJECTOR_TYPE_SOPRANO:
{
// SopranoDecoder defaults, n_fft comes from the head tensor
hparams.audio_sample_rate = 32000;
hparams.audio_hop_len = 512;
} break;
case PROJECTOR_TYPE_KOKORO:
case PROJECTOR_TYPE_TINY_VOCA:
{
hparams.audio_sample_rate = 24000;
} break;
case PROJECTOR_TYPE_PADDLEOCR:
{
hparams.n_merge = 2;
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
hparams.set_warmup_n_tokens(28*28); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
{
hparams.patch_size = 16;
hparams.image_size = 1024;
hparams.warmup_image_size = 1024;
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.image_pad_color = {127, 127, 127};
get_u32(KEY_SAM_N_BLOCK, hparams.sam_n_layer, true);
get_u32(KEY_SAM_N_HEAD, hparams.sam_n_head, true);
get_u32(KEY_SAM_N_EMBD, hparams.sam_n_embd, true);
get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true);
hparams.preproc_min_tiles = 2;
if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR) {
hparams.preproc_max_tiles = 9;
hparams.preproc_tile_size = 640;
// the CLIP/ViT body runs its layernorms at 1e-5 (the SAM stage uses 1e-6)
hparams.eps = 1e-5f;
}
if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2) {
hparams.preproc_max_tiles = 6;
hparams.preproc_tile_size = 768;
// qwen2 encoder is GQA, requires KEY_N_HEAD_KV
get_u32(mtmd_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
}
// unlimited-ocr shares the v1 projector but tiles up to 32
get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false);
get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false);
GGML_ASSERT(hparams.preproc_min_tiles >= 0
&& hparams.preproc_min_tiles <= hparams.preproc_max_tiles
&& hparams.preproc_max_tiles <= 256);
} break;
case PROJECTOR_TYPE_HUNYUANVL:
{
hparams.n_merge = 2;
hparams.image_resize_algo = RESIZE_ALGO_LANCZOS;
hparams.image_resize_pad = PAD_NONE;
hparams.ffn_op = FFN_GELU;
hparams.set_limit_image_tokens(256, 16384);
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels, false);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels, false);
hparams.set_warmup_n_tokens(32*32);
} break;
case PROJECTOR_TYPE_LFM2A:
case PROJECTOR_TYPE_D1OMNI_A:
{
// audio preprocessing params
hparams.audio_chunk_len = 1; // in seconds
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 512;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
} break;
case PROJECTOR_TYPE_LFM3A:
{
get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size);
if (hparams.attn_window_size <= 0) {
throw std::runtime_error(mtmd_format("%s: invalid audio attention window %d\n", __func__, hparams.attn_window_size));
}
hparams.rope_theta = 1000000.0f; // checked by the converter
// audio preprocessing params, see mtmd_audio_preprocessor_lfm3a
hparams.audio_chunk_len = 0;
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 400;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
} break;
case PROJECTOR_TYPE_EXAONE4_5:
{
hparams.n_merge = 2;
hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding
hparams.image_round_half_even = true; // smart_resize uses Python round()
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, false);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
hparams.set_warmup_n_tokens(46 * 46);
if (hparams.rope_theta <= 0.0f) {
hparams.rope_theta = 10000.0f;
}
get_u32(mtmd_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
} break;
case PROJECTOR_TYPE_GEMMA4A:
{
// Gemma4 feature_extraction_gemma4.py:
// frame_length_ms=20 -> 320 samples, n_fft=512, hop=10ms -> 160
hparams.audio_chunk_len = 0; // no fixed-length padding
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 512;
hparams.audio_window_len = 320; // 20ms frame (NOT 25ms/400)
hparams.audio_hop_len = 160;
// due to a mistake in the original conversion code, rms_norm_eps is set to a wrong value
// since all gemma4a models use 1e-6, we just hardcode it here to avoid re-conversion
hparams.eps = 1e-6f;
} break;
case PROJECTOR_TYPE_GEMMA4UA:
{
// Encoder-free: raw 16 kHz waveform chunked into 640-sample frames.
hparams.audio_chunk_len = 0;
hparams.audio_sample_rate = 16000;
hparams.eps = 1e-6f;
hparams.n_mel_bins = 640;
} break;
case PROJECTOR_TYPE_GRANITE_SPEECH:
{
hparams.audio_chunk_len = 0;
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 512;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
get_u32(KEY_A_CHUNK_SIZE, hparams.audio_chunk_size);
// context_size is squared for the attn_dists/mask buffers; cap to prevent int32 overflow
// (legitimate values are small, e.g. 12-200; 8192^2 = 67M still fits int32)
GGML_ASSERT(hparams.audio_chunk_size > 0 && hparams.audio_chunk_size <= 8192);
get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size);
get_u32(KEY_A_MAX_POS_EMB, hparams.audio_max_pos_emb);
get_u32(KEY_A_PROJ_WINDOW_SIZE, hparams.audio_proj_window_size);
get_u32(KEY_A_PROJ_DOWNSAMPLE_RATE, hparams.audio_proj_downsample_rate);
get_u32(KEY_A_PROJ_HEAD_COUNT, hparams.audio_proj_head_count);
// NOTE: feature layers loaded above in common path
} break;
case PROJECTOR_TYPE_JANUS_PRO:
{
hparams.image_pad_color = {127, 127, 127};
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
} break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
// SigLIP tower.
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.image_resize_pad = PAD_CEIL;
// NOTE: feature_layers loaded in common path as optional
get_arr_int(KEY_PROJ_SPATIAL_OFFSETS, hparams.proj_spatial_offsets);
if (hparams.feature_layers.size() != hparams.proj_spatial_offsets.size()) {
throw std::runtime_error(mtmd_format("%s: feature_layers.size() %d != proj_spatial_offsets.size() %d",
hparams.feature_layers.size(), hparams.proj_spatial_offsets.size()));
}
get_u32(KEY_PROJ_SAMPLE_QUERY_SIDE, hparams.downsample_query_side);
get_u32(KEY_PROJ_SAMPLE_WINDOW_SIDE, hparams.downsample_window_side);
hparams.warmup_image_size = hparams.image_size;
} break;
default:
throw std::runtime_error(mtmd_format("%s: unknown vision projector type %s\n", __func__, proj_type.c_str()));
}
// sanity check
{
if (hparams.image_size < 0) {
// note: some models having hparams.image_size == 0, which means the image size is dynamic
throw std::runtime_error(mtmd_format("%s: image_size (%d) cannot be negative\n", __func__, hparams.image_size));
}
if (hparams.image_size > 8192) {
// cap prevents int32 overflow in n_patches = (image_size/patch_size)^2
throw std::runtime_error(mtmd_format("%s: image_size (%d) is too large (max 8192)\n", __func__, hparams.image_size));
}
if (hparams.patch_size <= 0 || hparams.patch_size >= 65536) {
throw std::runtime_error(mtmd_format("%s: patch_size (%d) must be positive and less than 65536\n", __func__, hparams.patch_size));
}
if (hparams.n_embd <= 0) {
throw std::runtime_error(mtmd_format("%s: n_embd (%d) must be greater than 0\n", __func__, hparams.n_embd));
}
if (hparams.image_max_pixels < hparams.image_min_pixels) {
throw std::runtime_error(mtmd_format("%s: image_max_pixels (%d) is less than image_min_pixels (%d)\n", __func__, hparams.image_max_pixels, hparams.image_min_pixels));
}
if (hparams.n_merge <= 0 || hparams.n_merge >= 65536) {
throw std::runtime_error(mtmd_format("%s: n_merge (%d) must be greater than 0 and less than 65536\n", __func__, hparams.n_merge));
}
if (hparams.attn_window_size > 4096) {
throw std::runtime_error(mtmd_format("%s: attn_window_size (%d) is too large (max 4096)\n", __func__, hparams.attn_window_size));
}
}
LOG_INF("%s: projector: %s\n", __func__, proj_type.c_str());
LOG_INF("%s: n_embd: %d\n", __func__, hparams.n_embd);
LOG_INF("%s: n_head: %d\n", __func__, hparams.n_head);
LOG_INF("%s: n_ff: %d\n", __func__, hparams.n_ff);
LOG_INF("%s: n_layer: %d\n", __func__, hparams.n_layer);
LOG_INF("%s: ffn_op: %s\n", __func__, log_ffn_op.c_str());
LOG_INF("%s: projection_dim: %d\n", __func__, hparams.projection_dim);
if (is_vision) {
LOG_INF("\n--- vision hparams ---\n");
LOG_INF("%s: image_size: %d\n", __func__, hparams.image_size);
LOG_INF("%s: patch_size: %d\n", __func__, hparams.patch_size);
LOG_INF("%s: has_llava_proj: %d\n", __func__, hparams.has_llava_projector);
LOG_INF("%s: minicpmv_version: %d\n", __func__, hparams.minicpmv_version);
LOG_INF("%s: n_merge: %d\n", __func__, hparams.n_merge);
LOG_INF("%s: n_wa_pattern: %d\n", __func__, hparams.n_wa_pattern);
if (!hparams.wa_layer_indexes.empty()) {
LOG_INF("%s: wa_layer_indexes: ", __func__);
for (auto & layer : hparams.wa_layer_indexes) {
LOG_INF("%d ", layer);
}
LOG_INF("\n");
}
if (hparams.image_min_pixels > 0) {
LOG_INF("%s: image_min_pixels: %d%s\n", __func__, hparams.image_min_pixels, hparams.custom_image_min_tokens > 0 ? " (custom value)" : "");
}
if (hparams.image_max_pixels > 0) {
LOG_INF("%s: image_max_pixels: %d%s\n", __func__, hparams.image_max_pixels, hparams.custom_image_max_tokens > 0 ? " (custom value)" : "");
}
if (hparams.preproc_max_tiles > 0) {
LOG_INF("%s: preproc_tiles: %d - %d\n", __func__, hparams.preproc_min_tiles, hparams.preproc_max_tiles);
}
} else if (is_audio) {
LOG_INF("\n--- audio hparams ---\n");
LOG_INF("%s: n_mel_bins: %d\n", __func__, hparams.n_mel_bins);
LOG_INF("%s: proj_stack_factor: %d\n", __func__, hparams.proj_stack_factor);
LOG_INF("%s: audio_chunk_len: %d\n", __func__, hparams.audio_chunk_len);
LOG_INF("%s: audio_sample_rate: %d\n", __func__, hparams.audio_sample_rate);
LOG_INF("%s: audio_n_fft: %d\n", __func__, hparams.audio_n_fft);
LOG_INF("%s: audio_window_len: %d\n", __func__, hparams.audio_window_len);
LOG_INF("%s: audio_hop_len: %d\n", __func__, hparams.audio_hop_len);
// GEMMA4UA is encoder-free: it uses n_mel_bins as a raw-waveform frame size (640) and has no FFT/filterbank, so the mel-range and FFT
// checks below do not apply to it.
// pocket-tts is encoder-free in the same sense: mimi convolves the raw waveform
// Muse Spark is encoder-free in the same sense: its conv pyramid reads the raw waveform, one channel takes the place of the mel bins
const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA &&
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC &&
model.proj_type != PROJECTOR_TYPE_MOSHI &&
model.proj_type != PROJECTOR_TYPE_FISH_S2_SPKENC &&
model.proj_type != PROJECTOR_TYPE_VOXCPM2_SPKENC;
// Validate audio hparams loaded from GGUF metadata
if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) {
throw std::runtime_error(mtmd_format("%s: n_mel_bins (%d) must be in range [1, 256]\n", __func__, hparams.n_mel_bins));
}
if (fft_based && (hparams.audio_sample_rate <= 0 || hparams.audio_n_fft <= 0 || hparams.audio_hop_len <= 0 || hparams.audio_window_len <= 0)) {
throw std::runtime_error(mtmd_format("%s: audio hparams invalid: sample_rate=%d n_fft=%d window_len=%d hop_len=%d\n",
__func__, hparams.audio_sample_rate, hparams.audio_n_fft, hparams.audio_window_len, hparams.audio_hop_len));
}
}
LOG_INF("\n");
LOG_INF("%s: model size: %.2f MiB\n", __func__, model_size / 1024.0 / 1024.0);
LOG_INF("%s: metadata size: %.2f MiB\n", __func__, ggml_get_mem_size(ctx_meta.get()) / 1024.0 / 1024.0);
}
}
void load_tensors(mtmd_model & model, ggml_backend_buffer_type_t buft, bool no_alloc) {
auto & hparams = model.hparams;
std::map<std::string, size_t> tensor_offset;
std::vector<ggml_tensor *> tensors_to_load;
auto fin = open_ifstream_binary(fname);
if (!fin) {
throw std::runtime_error(mtmd_format("%s: failed to open %s\n", __func__, fname.c_str()));
}
// TODO @ngxson : support both audio and video in the future
const char * prefix = model.modality == MTMD_MODALITY_AUDIO ? "a"
: model.modality == MTMD_MODALITY_GEN_AUDIO ? "a.gen.code"
: "v";
// get offsets
for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) {
const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);
tensor_offset[name] = gguf_get_data_offset(ctx_gguf.get()) + gguf_get_tensor_offset(ctx_gguf.get(), i);
}
// create data context
struct ggml_init_params params = {
/*.mem_size =*/ static_cast<size_t>(gguf_get_n_tensors(ctx_gguf.get()) + 1) * ggml_tensor_overhead(),
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ true,
};
model.ctx_data.reset(ggml_init(params));
if (!model.ctx_data) {
throw std::runtime_error(mtmd_format("%s: failed to init ggml context\n", __func__));
}
// helper function
std::unordered_set<std::string> loaded_tensor_names;
auto get_tensor = [&](const std::string & name, bool required = true) {
// Each tensor should only be loaded once; duplicates indicate a bug
if (loaded_tensor_names.count(name)) {
throw std::runtime_error(mtmd_format("%s: tensor already loaded: %s\n", __func__, name.c_str()));
}
ggml_tensor * cur = ggml_get_tensor(ctx_meta.get(), name.c_str());
if (!cur && required) {
throw std::runtime_error(mtmd_format("%s: unable to find tensor %s\n", __func__, name.c_str()));
}
if (cur) {
tensors_to_load.push_back(cur);
ggml_tensor * data_tensor = ggml_dup_tensor(model.ctx_data.get(), cur);
ggml_set_name(data_tensor, cur->name);
loaded_tensor_names.insert(name);
cur = data_tensor;
// add to weight memory counter
model.mem_weights[ggml_backend_buft_get_device(buft)] += ggml_nbytes(cur);
}
return cur;
};
// pocket-tts: the encoder and the decoder share the same layout, only the prefix differs
auto load_seanet = [&](mtmd_seanet & seanet, bool is_decoder) {
const char * conv_in = is_decoder ? TN_A_GEN_WAV_SEANET_CONV_IN : TN_A_SEANET_CONV_IN;
const char * conv_out = is_decoder ? TN_A_GEN_WAV_SEANET_CONV_OUT : TN_A_SEANET_CONV_OUT;
const char * res1 = is_decoder ? TN_A_GEN_WAV_SEANET_RES_CONV1 : TN_A_SEANET_RES_CONV1;
const char * res2 = is_decoder ? TN_A_GEN_WAV_SEANET_RES_CONV2 : TN_A_SEANET_RES_CONV2;
const char * scale = is_decoder ? TN_A_GEN_WAV_SEANET_SCALE_CONV : TN_A_SEANET_SCALE_CONV;
seanet.conv_in_w = get_tensor(mtmd_format(conv_in, "weight"));
seanet.conv_in_b = get_tensor(mtmd_format(conv_in, "bias"));
seanet.conv_out_w = get_tensor(mtmd_format(conv_out, "weight"));
seanet.conv_out_b = get_tensor(mtmd_format(conv_out, "bias"));
seanet.stages.resize(hparams.seanet_n_stage);
for (int i = 0; i < hparams.seanet_n_stage; i++) {
auto & stage = seanet.stages[i];
stage.res_conv1_w = get_tensor(mtmd_format(res1, i, "weight"));
stage.res_conv1_b = get_tensor(mtmd_format(res1, i, "bias"));
stage.res_conv2_w = get_tensor(mtmd_format(res2, i, "weight"));
stage.res_conv2_b = get_tensor(mtmd_format(res2, i, "bias"));
stage.scale_conv_w = get_tensor(mtmd_format(scale, i, "weight"));
stage.scale_conv_b = get_tensor(mtmd_format(scale, i, "bias"));
}
};
// mimi decoder: upsample, transformer, SEANet decoder
auto load_mimi_decoder = [&](int n_tfm_layer) {
model.gen_upsample_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UPSAMPLE, "weight"));
load_seanet(model.seanet, true);
model.gen_tfm_layers.resize(n_tfm_layer);
for (int il = 0; il < n_tfm_layer; il++) {
auto & layer = model.gen_tfm_layers[il];
const char * p = "a.gen.wav.tfm";
layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, p, il, "weight"));
layer.ln_1_b = get_tensor(mtmd_format(TN_LN_1, p, il, "bias"));
layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, p, il, "weight"));
layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, p, il, "weight"));
layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, p, il, "weight"));
layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight"));
layer.ls_1_w = get_tensor(mtmd_format(TN_LS_1, p, il, "weight"));
layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, p, il, "weight"));
layer.ln_2_b = get_tensor(mtmd_format(TN_LN_2, p, il, "bias"));
layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, p, il, "weight"));
layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, p, il, "weight"));
layer.ls_2_w = get_tensor(mtmd_format(TN_LS_2, p, il, "weight"));
}
};
auto get_vector = [&](const std::string & name) {
std::vector<float> result;
auto it = tensor_offset.find(name);
if (it == tensor_offset.end()) {
return result;
}
const int64_t idx = gguf_find_tensor(ctx_gguf.get(), name.c_str());
if (idx < 0) {
throw std::runtime_error(mtmd_format("%s: failed to find tensor %s\n", __func__, name.c_str()));
}
if (const auto type = gguf_get_tensor_type(ctx_gguf.get(), idx); type != GGML_TYPE_F32) {
throw std::runtime_error(mtmd_format("%s: %s must be %s, was %s\n", __func__,
name.c_str(), ggml_type_name(GGML_TYPE_F32), ggml_type_name(type)));
}
const size_t n_bytes = gguf_get_tensor_size(ctx_gguf.get(), idx);
if (n_bytes == 0) {
throw std::runtime_error(mtmd_format("%s: tensor %s is empty\n", __func__, name.c_str()));
}
const size_t n_elems = n_bytes / sizeof(float);
result.resize(n_elems);
fin.seekg(it->second, std::ios::beg);
fin.read(reinterpret_cast<char*>(result.data()), n_bytes);
return result;
};
auto get_scalar = [&](const std::string & name, float default_val) {
auto v = get_vector(name);
if (v.empty()) {
return default_val;
}
if (v.size() != 1) {
throw std::runtime_error(mtmd_format("%s: expected scalar tensor '%s' but got %d elements\n",
__func__, name.c_str(), (int) v.size()));
}
return v[0];
};
model.class_embedding = get_tensor(TN_CLASS_EMBD, false);
model.pre_ln_w = get_tensor(mtmd_format(TN_LN_PRE, prefix, "weight"), false);
model.pre_ln_b = get_tensor(mtmd_format(TN_LN_PRE, prefix, "bias"), false);
model.post_ln_w = get_tensor(mtmd_format(TN_LN_POST, prefix, "weight"), false);
model.post_ln_b = get_tensor(mtmd_format(TN_LN_POST, prefix, "bias"), false);
// every vision encoder starts with a patch embedding, except the ones with their own stem
const bool has_patch_embd = (
model.modality == MTMD_MODALITY_VISION &&
model.proj_type != PROJECTOR_TYPE_GEMMA3NV &&
model.proj_type != PROJECTOR_TYPE_DEEPSEEKOCR &&
model.proj_type != PROJECTOR_TYPE_DEEPSEEKOCR2);
model.patch_bias = get_tensor(TN_PATCH_BIAS, false);
model.patch_embeddings_0 = get_tensor(TN_PATCH_EMBD, has_patch_embd);
model.patch_embeddings_1 = get_tensor(TN_PATCH_EMBD_1, false);
model.norm_embd_w = get_tensor(mtmd_format(TN_NORM_EMBD, "weight"), false);
model.norm_embd_b = get_tensor(mtmd_format(TN_NORM_EMBD, "bias"), false);
model.position_embeddings = get_tensor(mtmd_format(TN_POS_EMBD, prefix), false);
// lfm3vl
model.lfm3vl_embedding_norm_w = get_tensor(mtmd_format(TN_LFM3VL_EMBEDDING_NORM, prefix, "weight"), false);
const bool has_standard_layers = (
model.proj_type != PROJECTOR_TYPE_GEMMA3NV &&
model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC &&
model.proj_type != PROJECTOR_TYPE_ZONOS3_SPKENC &&
model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN &&
model.proj_type != PROJECTOR_TYPE_SOPRANO &&
model.proj_type != PROJECTOR_TYPE_KOKORO &&
model.proj_type != PROJECTOR_TYPE_TINY_VOCA &&
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
model.proj_type != PROJECTOR_TYPE_CHATTERBOX &&
model.proj_type != PROJECTOR_TYPE_CHATTERBOX_SPKENC &&
model.proj_type != PROJECTOR_TYPE_YUE2_GEN);
// layers
const int n_layers_to_load = has_standard_layers ? hparams.n_layer : 0;
model.layers.resize(n_layers_to_load);
for (int il = 0; il < n_layers_to_load; ++il) {
auto & layer = model.layers[il];
// lfm3vl / lfm3a conv layers have no attention tensors
const bool o_w_required = model.proj_type != PROJECTOR_TYPE_LFM3VL
&& model.proj_type != PROJECTOR_TYPE_LFM3A;
layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, prefix, il, "weight"), false);
layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, prefix, il, "weight"), false);
layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, prefix, il, "weight"), false);
layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, prefix, il, "weight"), o_w_required);
layer.qkv_w = get_tensor(mtmd_format(TN_ATTN_QKV, prefix, il, "weight"), false);
layer.k_norm = get_tensor(mtmd_format(TN_ATTN_K_NORM, prefix, il, "weight"), false);
layer.q_norm = get_tensor(mtmd_format(TN_ATTN_Q_NORM, prefix, il, "weight"), false);
layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, prefix, il, "weight"), false);
layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, prefix, il, "weight"), false);
layer.ls_1_w = get_tensor(mtmd_format(TN_LS_1, prefix, il, "weight"), false); // no bias
layer.ls_2_w = get_tensor(mtmd_format(TN_LS_2, prefix, il, "weight"), false); // no bias
layer.ls_out_w = get_tensor(mtmd_format(TN_LS_OUT, prefix, il, "weight"), false); // no bias
layer.attn_post_norm_w = get_tensor(mtmd_format(TN_ATTN_POST_NORM, prefix, il, "weight"), false); // no bias
layer.ff_post_norm_w = get_tensor(mtmd_format(TN_FFN_POST_NORM, prefix, il, "weight"), false); // no bias
layer.k_b = get_tensor(mtmd_format(TN_ATTN_K, prefix, il, "bias"), false);
layer.q_b = get_tensor(mtmd_format(TN_ATTN_Q, prefix, il, "bias"), false);
layer.v_b = get_tensor(mtmd_format(TN_ATTN_V, prefix, il, "bias"), false);
layer.o_b = get_tensor(mtmd_format(TN_ATTN_OUTPUT, prefix, il, "bias"), false);
layer.qkv_b = get_tensor(mtmd_format(TN_ATTN_QKV, prefix, il, "bias"), false);
layer.ln_1_b = get_tensor(mtmd_format(TN_LN_1, prefix, il, "bias"), false);
layer.ln_2_b = get_tensor(mtmd_format(TN_LN_2, prefix, il, "bias"), false);
// MoE ffn (dots3note vision pyramid blocks); replaces the dense ffn when present
layer.ff_gate_inp_w = get_tensor(mtmd_format(TN_FFN_GATE_INP, prefix, il, "weight"), false);
layer.ff_gate_exps_w = get_tensor(mtmd_format(TN_FFN_GATE_EXPS, prefix, il, "weight"), false);
layer.ff_up_exps_w = get_tensor(mtmd_format(TN_FFN_UP_EXPS, prefix, il, "weight"), false);
layer.ff_down_exps_w = get_tensor(mtmd_format(TN_FFN_DOWN_EXPS, prefix, il, "weight"), false);
layer.ff_exp_probs_b = get_tensor(mtmd_format(TN_FFN_EXP_PROBS_B, prefix, il, "weight"), false);
const bool is_moe = layer.ff_gate_exps_w != nullptr;
// ffn
layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, prefix, il, "weight"), !is_moe);
layer.ff_up_b = get_tensor(mtmd_format(TN_FFN_UP, prefix, il, "bias"), false);
layer.ff_gate_w = get_tensor(mtmd_format(TN_FFN_GATE, prefix, il, "weight"), false);
layer.ff_gate_b = get_tensor(mtmd_format(TN_FFN_GATE, prefix, il, "bias"), false);
layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, prefix, il, "weight"), !is_moe);
layer.ff_down_b = get_tensor(mtmd_format(TN_FFN_DOWN, prefix, il, "bias"), false);
// mimovl per-head attention sink bias
layer.attn_sinks = get_tensor(mtmd_format(TN_ATTN_SINKS, prefix, il), false);
// qwen3vl deepstack layer
layer.deepstack_norm_w = get_tensor(mtmd_format(TN_DEEPSTACK_NORM, il, "weight"), false);
layer.deepstack_norm_b = get_tensor(mtmd_format(TN_DEEPSTACK_NORM, il, "bias"), false);
layer.deepstack_fc1_w = get_tensor(mtmd_format(TN_DEEPSTACK_FC1, il, "weight"), false);
layer.deepstack_fc1_b = get_tensor(mtmd_format(TN_DEEPSTACK_FC1, il, "bias"), false);
layer.deepstack_fc2_w = get_tensor(mtmd_format(TN_DEEPSTACK_FC2, il, "weight"), false);
layer.deepstack_fc2_b = get_tensor(mtmd_format(TN_DEEPSTACK_FC2, il, "bias"), false);
if (layer.has_deepstack()) {
model.n_deepstack_layers++;
}
// lfm3vl conv layers
layer.conv2d_in_proj_w = get_tensor(mtmd_format(TN_LFM3VL_CONV2D_IN_PROJ, prefix, il, "weight"), false);
layer.conv2d_spatial_w = get_tensor(mtmd_format(TN_LFM3VL_CONV2D_SPATIAL, prefix, il, "weight"), false);
layer.conv2d_out_proj_w = get_tensor(mtmd_format(TN_LFM3VL_CONV2D_OUT_PROJ, prefix, il, "weight"), false);
// some models already exported with legacy (incorrect) naming which is quite messy, let's fix it here
// note: Qwen model converted from the old surgery script has n_ff = 0, so we cannot use n_ff to check!
bool is_ffn_swapped = (
// only old models need this fix
model.proj_type == PROJECTOR_TYPE_MLP
|| model.proj_type == PROJECTOR_TYPE_MLP_NORM
|| model.proj_type == PROJECTOR_TYPE_LDP
|| model.proj_type == PROJECTOR_TYPE_LDPV2
|| model.proj_type == PROJECTOR_TYPE_QWEN2VL
|| model.proj_type == PROJECTOR_TYPE_QWEN25VL
|| model.proj_type == PROJECTOR_TYPE_EXAONE4_5
|| model.proj_type == PROJECTOR_TYPE_GLM_EDGE
|| model.proj_type == PROJECTOR_TYPE_GEMMA3
|| model.proj_type == PROJECTOR_TYPE_IDEFICS3
|| model.proj_type == PROJECTOR_TYPE_MINICPMV
|| model.proj_type == PROJECTOR_TYPE_MINICPMV4_6
|| model.proj_type == PROJECTOR_TYPE_MINICPMV4_7
|| model.proj_type == PROJECTOR_TYPE_MINICPM_OCR_1_0
) && layer.ff_up_w && layer.ff_down_w && layer.ff_down_w->ne[0] == hparams.n_embd;
if (is_ffn_swapped) {
// swap up and down weights
ggml_tensor * tmp = layer.ff_up_w;
layer.ff_up_w = layer.ff_down_w;
layer.ff_down_w = tmp;
// swap up and down biases
tmp = layer.ff_up_b;
layer.ff_up_b = layer.ff_down_b;
layer.ff_down_b = tmp;
if (il == 0) {
LOG_WRN("%s: ffn up/down are swapped\n", __func__);
}
}
}
// DAC-style residual unit: Snake, dilated conv, Snake, conv k = 1 (voxcpm2 AudioVAE)
auto load_dac_res = [&](const std::string & q) {
mtmd_code2wav::dac_res ru;
ru.act1_alpha = get_tensor(q + ".snake1.alpha");
ru.act1_beta = get_tensor(q + ".snake1.beta");
ru.conv1_w = get_tensor(q + ".conv1.weight");
ru.conv1_b = get_tensor(q + ".conv1.bias");
ru.act2_alpha = get_tensor(q + ".snake2.alpha");
ru.act2_beta = get_tensor(q + ".snake2.beta");
ru.conv2_w = get_tensor(q + ".conv2.weight");
ru.conv2_b = get_tensor(q + ".conv2.bias");
return ru;
};
// code2wav: RVQ codes -> raw PCM, drama3 has no quant/pre_conv/tfm projections
auto load_code2wav = [&](bool with_proj) {
auto & c2w = model.c2w;
c2w.quant_first_in_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_IN, "weight"), with_proj);
c2w.quant_first_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight"), with_proj);
c2w.quant_first_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight"));
c2w.quant_rest_in_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_IN, "weight"), with_proj);
c2w.quant_rest_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_OUT, "weight"), with_proj);
c2w.quant_rest_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_CB, "weight"));
c2w.pre_conv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_PRE_CONV, "weight"), with_proj);
c2w.pre_conv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_PRE_CONV, "bias"), with_proj);
c2w.tfm_in_proj_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_IN_PROJ, "weight"), with_proj);
c2w.tfm_in_proj_b = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_IN_PROJ, "bias"), with_proj);
c2w.tfm_out_proj_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "weight"), with_proj);
c2w.tfm_out_proj_b = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias"), with_proj);
c2w.tfm_output_norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight"));
// loaded manually, the generic model.layers loop is taken by code_predictor
c2w.tfm_layers.resize(hparams.wav_tfm_n_layer);
for (int il = 0; il < hparams.wav_tfm_n_layer; il++) {
auto & layer = c2w.tfm_layers[il];
const char * p = "a.gen.wav.tfm";
layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, p, il, "weight"));
layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, p, il, "weight"));
layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, p, il, "weight"));
layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight"));
layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, p, il, "weight"));
layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, p, il, "weight"));
layer.ls_1_w = get_tensor(mtmd_format(TN_LS_1, p, il, "weight"));
layer.ls_2_w = get_tensor(mtmd_format(TN_LS_2, p, il, "weight"));
layer.ff_gate_w = get_tensor(mtmd_format(TN_FFN_GATE, p, il, "weight"));
layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, p, il, "weight"));
layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, p, il, "weight"));
}
// upsample: 2x (causal ConvTranspose1d + ConvNeXt block)
c2w.upsample.resize(hparams.wav_upsample_n_block);
for (int il = 0; il < hparams.wav_upsample_n_block; il++) {
auto & up = c2w.upsample[il];
up.conv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_CONV, il, "weight"));
up.conv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_CONV, il, "bias"));
up.dwconv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight"));
up.dwconv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias"));
up.norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_NORM, il, "weight"));
up.norm_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_NORM, il, "bias"));
up.pw1_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW1, il, "weight"));
up.pw1_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW1, il, "bias"));
up.pw2_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW2, il, "weight"));
up.pw2_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW2, il, "bias"));
up.gamma = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_GAMMA, il));
}
// DAC decoder: conv_pre + n upsample blocks (each with n_res residual units) + conv_post
c2w.dac_entry_w = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_ENTRY, "weight"));
c2w.dac_entry_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_ENTRY, "bias"));
c2w.dac.resize(hparams.wav_dac_n_block);
for (int il = 0; il < hparams.wav_dac_n_block; il++) {
auto & blk = c2w.dac[il];
blk.snake_alpha = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_SNAKE, il, "alpha"));
blk.snake_beta = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_SNAKE, il, "beta"));
blk.conv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_CONV, il, "weight"));
blk.conv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_CONV, il, "bias"));
blk.res.resize(hparams.wav_dac_n_res);
for (int ir = 0; ir < hparams.wav_dac_n_res; ir++) {
auto & res = blk.res[ir];
res.act1_alpha = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "alpha"));
res.act1_beta = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "beta"));
res.conv1_w = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "weight"));
res.conv1_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "bias"));
res.act2_alpha = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "alpha"));
res.act2_beta = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "beta"));
res.conv2_w = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "weight"));
res.conv2_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "bias"));
}
}
c2w.dac_post_snake_alpha = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "alpha"));
c2w.dac_post_snake_beta = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "beta"));
c2w.dac_post_conv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_POST_CONV, "weight"));
c2w.dac_post_conv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_POST_CONV, "bias"));
};
// a.gen.vae.*, see mtmd_snake_vae
auto load_snake_vae = [&]() {
auto & vae = model.snake_vae;
const std::string p = "a.gen.vae";
auto load_snake = [&](const std::string & q) {
return mtmd_snake_vae::snake{ get_tensor(q + ".alpha"), get_tensor(q + ".beta") };
};
vae.in_proj_w = get_tensor(p + ".in_proj.weight", false);
vae.in_proj_b = get_tensor(p + ".in_proj.bias", false);
vae.in_w = get_tensor(p + ".conv_in.weight");
vae.in_b = get_tensor(p + ".conv_in.bias");
for (int b = 0; gguf_find_tensor(ctx_gguf.get(), mtmd_format("%s.blk.%d.up.weight", p.c_str(), b).c_str()) >= 0; b++) {
const std::string q = p + ".blk." + std::to_string(b);
mtmd_snake_vae::block blk;
blk.act = load_snake(q + ".snake");
blk.up_w = get_tensor(q + ".up.weight");
blk.up_b = get_tensor(q + ".up.bias");
for (int r = 0; r < 3; r++) {
const std::string rq = q + ".res." + std::to_string(r);
mtmd_snake_vae::res_unit ru;
ru.act1 = load_snake(rq + ".snake1");
ru.conv1_w = get_tensor(rq + ".conv1.weight");
ru.conv1_b = get_tensor(rq + ".conv1.bias");
ru.act2 = load_snake(rq + ".snake2");
ru.conv2_w = get_tensor(rq + ".conv2.weight");
ru.conv2_b = get_tensor(rq + ".conv2.bias");
blk.res.push_back(ru);
}
vae.blocks.push_back(std::move(blk));
}
vae.post = load_snake(p + ".post_snake");
vae.out_w = get_tensor(p + ".conv_out.weight");
vae.out_b = get_tensor(p + ".conv_out.bias", false);
if (vae.blocks.empty()) {
throw std::runtime_error(mtmd_format("%s: no VAE decoder blocks\n", __func__));
}
};
switch (model.proj_type) {
case PROJECTOR_TYPE_MLP:
case PROJECTOR_TYPE_MLP_NORM:
{
// LLaVA projection: mm.0 -> gelu -> mm.2
// Yi-type llava (MLP_NORM): mm.0 -> norm mm.1 -> gelu -> mm.3 -> norm mm.4
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"));
model.mm_3_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 3, "weight"), model.proj_type == PROJECTOR_TYPE_MLP_NORM);
if (model.mm_3_w) {
// TODO: this is a hack to support Yi-type llava
model.proj_type = PROJECTOR_TYPE_MLP_NORM;
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"));
model.mm_3_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 3, "bias"));
model.mm_4_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 4, "weight"));
model.mm_4_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 4, "bias"));
} else {
model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
}
model.image_newline = get_tensor(TN_IMAGE_NEWLINE, false);
} break;
case PROJECTOR_TYPE_LDP:
{
// MobileVLM projection
model.mm_model_mlp_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "weight"));
model.mm_model_mlp_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "bias"));
model.mm_model_mlp_3_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "weight"));
model.mm_model_mlp_3_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "bias"));
model.mm_model_block_1_block_0_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 0, "0.weight"));
model.mm_model_block_1_block_0_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.weight"));
model.mm_model_block_1_block_0_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.bias"));
model.mm_model_block_1_block_1_fc1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.weight"));
model.mm_model_block_1_block_1_fc1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.bias"));
model.mm_model_block_1_block_1_fc2_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.weight"));
model.mm_model_block_1_block_1_fc2_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.bias"));
model.mm_model_block_1_block_2_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 2, "0.weight"));
model.mm_model_block_1_block_2_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.weight"));
model.mm_model_block_1_block_2_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.bias"));
model.mm_model_block_2_block_0_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 0, "0.weight"));
model.mm_model_block_2_block_0_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.weight"));
model.mm_model_block_2_block_0_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.bias"));
model.mm_model_block_2_block_1_fc1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.weight"));
model.mm_model_block_2_block_1_fc1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.bias"));
model.mm_model_block_2_block_1_fc2_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.weight"));
model.mm_model_block_2_block_1_fc2_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.bias"));
model.mm_model_block_2_block_2_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 2, "0.weight"));
model.mm_model_block_2_block_2_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.weight"));
model.mm_model_block_2_block_2_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.bias"));
} break;
case PROJECTOR_TYPE_LDPV2:
{
// MobilVLM_V2 projection
model.mm_model_mlp_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "weight"));
model.mm_model_mlp_0_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "bias"));
model.mm_model_mlp_2_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 2, "weight"));
model.mm_model_mlp_2_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 2, "bias"));
model.mm_model_peg_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_PEG, 0, "weight"));
model.mm_model_peg_0_b = get_tensor(mtmd_format(TN_MVLM_PROJ_PEG, 0, "bias"));
} break;
case PROJECTOR_TYPE_MINICPMV:
{
// model.mm_model_pos_embed = get_tensor(new_clip->ctx_data, TN_MINICPMV_POS_EMBD);
model.mm_model_pos_embed_k = get_tensor(TN_MINICPMV_POS_EMBD_K);
model.mm_model_query = get_tensor(TN_MINICPMV_QUERY);
model.mm_model_proj = get_tensor(TN_MINICPMV_PROJ);
model.mm_model_kv_proj = get_tensor(TN_MINICPMV_KV_PROJ);
model.mm_model_attn_q_w = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "q", "weight"));
model.mm_model_attn_k_w = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "k", "weight"));
model.mm_model_attn_v_w = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "v", "weight"));
model.mm_model_attn_q_b = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "q", "bias"));
model.mm_model_attn_k_b = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "k", "bias"));
model.mm_model_attn_v_b = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "v", "bias"));
model.mm_model_attn_o_w = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "out", "weight"));
model.mm_model_attn_o_b = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "out", "bias"));
model.mm_model_ln_q_w = get_tensor(mtmd_format(TN_MINICPMV_LN, "q", "weight"));
model.mm_model_ln_q_b = get_tensor(mtmd_format(TN_MINICPMV_LN, "q", "bias"));
model.mm_model_ln_kv_w = get_tensor(mtmd_format(TN_MINICPMV_LN, "kv", "weight"));
model.mm_model_ln_kv_b = get_tensor(mtmd_format(TN_MINICPMV_LN, "kv", "bias"));
model.mm_model_ln_post_w = get_tensor(mtmd_format(TN_MINICPMV_LN, "post", "weight"));
model.mm_model_ln_post_b = get_tensor(mtmd_format(TN_MINICPMV_LN, "post", "bias"));
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
const bool merger_required = hparams.n_merge == 4;
auto get_merger_tensor = [&](const std::string & name, bool required = true) {
return get_tensor(name, merger_required && required);
};
// ViT merger: window self-attention
model.vit_merger_ln1_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_LN1, "weight"));
model.vit_merger_ln1_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_LN1, "bias"));
model.vit_merger_attn_q_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_Q, "weight"));
model.vit_merger_attn_q_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_Q, "bias"), false);
model.vit_merger_attn_k_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_K, "weight"));
model.vit_merger_attn_k_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_K, "bias"), false);
model.vit_merger_attn_v_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_V, "weight"));
model.vit_merger_attn_v_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_V, "bias"), false);
model.vit_merger_attn_o_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_O, "weight"));
model.vit_merger_attn_o_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_O, "bias"), false);
// ViT merger: MLP downsample
model.vit_merger_ds_ln_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_LN, "weight"));
model.vit_merger_ds_ln_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_LN, "bias"));
model.vit_merger_ds_up_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_UP, "weight"));
model.vit_merger_ds_up_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_UP, "bias"), false);
model.vit_merger_ds_down_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_DOWN, "weight"));
model.vit_merger_ds_down_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_DOWN, "bias"), false);
// Final Merger (DownsampleMLP)
model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);
model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B, false);
model.mm_ffn_up_w = get_tensor(mtmd_format(TN_MM_UP, "weight"));
model.mm_ffn_up_b = get_tensor(mtmd_format(TN_MM_UP, "bias"), false);
model.mm_ffn_down_w = get_tensor(mtmd_format(TN_MM_DOWN, "weight"));
model.mm_ffn_down_b = get_tensor(mtmd_format(TN_MM_DOWN, "bias"), false);
model.tok_embd_sep = get_tensor(TN_TOK_EMBD_SEP, model.proj_type == PROJECTOR_TYPE_MINICPMV4_7);
} break;
case PROJECTOR_TYPE_GLM_EDGE:
{
model.mm_model_adapter_conv_w = get_tensor(mtmd_format(TN_GLM_ADAPER_CONV, "weight"));
model.mm_model_adapter_conv_b = get_tensor(mtmd_format(TN_GLM_ADAPER_CONV, "bias"));
model.mm_model_mlp_0_w = get_tensor(mtmd_format(TN_GLM_ADAPTER_LINEAR, "weight"));
model.mm_model_ln_q_w = get_tensor(mtmd_format(TN_GLM_ADAPTER_NORM_1, "weight"));
model.mm_model_ln_q_b = get_tensor(mtmd_format(TN_GLM_ADAPTER_NORM_1, "bias"));
model.mm_model_mlp_1_w = get_tensor(mtmd_format(TN_GLM_ADAPTER_D_H_2_4H, "weight"));
model.mm_model_mlp_2_w = get_tensor(mtmd_format(TN_GLM_ADAPTER_GATE, "weight"));
model.mm_model_mlp_3_w = get_tensor(mtmd_format(TN_GLM_ADAPTER_D_4H_2_H, "weight"));
model.mm_boi = get_tensor(mtmd_format(TN_TOK_GLM_BOI));
model.mm_eoi = get_tensor(mtmd_format(TN_TOK_GLM_EOI));
} break;
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_EXAONE4_5:
{
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_MAGEVL:
{
// d_head rope factors: the first half drives the even outputs, the second half the odd ones
model.rope_freqs = get_tensor(TN_ROPE_FREQS);
const int d_head = hparams.n_head > 0 ? hparams.n_embd / hparams.n_head : 0;
if (d_head <= 0 || d_head * hparams.n_head != hparams.n_embd || d_head % 32 != 0 ||
model.rope_freqs->type != GGML_TYPE_F32 || ggml_nelements(model.rope_freqs) != d_head) {
throw std::runtime_error(mtmd_format("%s: hparams do not match the tensor shapes\n", __func__));
}
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_QWEN3VL:
{
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_LFM3VL:
{
model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);
model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B);
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
// each layer must be either a conv block or a complete attention block
for (size_t il = 0; il < model.layers.size(); ++il) {
const auto & layer = model.layers[il];
const bool is_conv = layer.conv2d_in_proj_w != nullptr;
const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w;
if (is_conv == is_attn) {
throw std::runtime_error(mtmd_format("%s: lfm3vl layer %zu is neither a complete conv nor a complete attention block\n", __func__, il));
}
}
} break;
case PROJECTOR_TYPE_LING3VL:
{
model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); // merger.norm
model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); // merger.norm
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); // linear_proj.0
model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); // linear_proj.2
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_MIMOVL:
{
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"), false);
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false);
} break;
case PROJECTOR_TYPE_MINIMAX_M3:
{
// per-patch MLP: mm.1 -> gelu -> mm.2
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
// 2x2 merge MLP: mm.merge.fc1 -> gelu -> mm.merge.fc2
model.mm_merger_fc1_w = get_tensor(mtmd_format(TN_MM_MERGER_FC1, "weight"));
model.mm_merger_fc1_b = get_tensor(mtmd_format(TN_MM_MERGER_FC1, "bias"));
model.mm_merger_fc2_w = get_tensor(mtmd_format(TN_MM_MERGER_FC2, "weight"));
model.mm_merger_fc2_b = get_tensor(mtmd_format(TN_MM_MERGER_FC2, "bias"));
} break;
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
{
model.layers.resize(hparams.n_layer);
model.a_patches.resize(hparams.audio_patch_sizes.size());
for (size_t i = 0; i < model.a_patches.size(); ++i) {
auto & pe = model.a_patches[i];
pe.proj_w = get_tensor(mtmd_format(TN_A_PATCH, (int) i, "weight"));
pe.proj_b = get_tensor(mtmd_format(TN_A_PATCH, (int) i, "bias"));
pe.norm_w = get_tensor(mtmd_format(TN_A_PATCH_NORM, (int) i, "weight"));
if (pe.proj_w->ne[1] != hparams.audio_hidden_sizes[i]) {
throw std::runtime_error(mtmd_format("%s: audio stage %zu projection has width %" PRId64 ", expected %d\n", __func__, i, pe.proj_w->ne[1], hparams.audio_hidden_sizes[i]));
}
}
for (int il = 0; il < hparams.n_layer; ++il) {
auto & layer = model.layers[il];
layer.conv_w = get_tensor(mtmd_format(TN_CONV1D, il, "weight"));
layer.conv_b = get_tensor(mtmd_format(TN_CONV1D, il, "bias"));
layer.conv_norm_w = get_tensor(mtmd_format(TN_A_CONV1D_NORM, il, "weight"));
layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, "a", il, "weight"));
layer.ff_up_b = get_tensor(mtmd_format(TN_FFN_UP, "a", il, "bias"));
layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, "a", il, "weight"));
layer.ff_down_b = get_tensor(mtmd_format(TN_FFN_DOWN, "a", il, "bias"));
}
// pre_ln and post_ln come from the generic path above, where they are optional
if (!model.pre_ln_w || !model.post_ln_w) {
throw std::runtime_error(mtmd_format("%s: missing pre_ln or post_ln weight\n", __func__));
}
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight"));
model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias"));
} break;
case PROJECTOR_TYPE_VIBEVOICE_ASR:
{
const int n_stage = (int) hparams.audio_patch_sizes.size();
for (int e = 0; e < 2; ++e) {
auto & enc = model.cnx_enc[e];
enc.conv_w.resize(n_stage + 1);
enc.conv_b.resize(n_stage + 1);
for (int i = 0; i <= n_stage; ++i) {
enc.conv_w[i] = get_tensor(mtmd_format(TN_A_CNX_CONV, e, i, "weight"));
enc.conv_b[i] = get_tensor(mtmd_format(TN_A_CNX_CONV, e, i, "bias"));
// the receptive field in load_hparams assumes these kernel sizes
const int64_t k_expected = (i == 0 || i == n_stage) ? hparams.audio_conv_kernel_size : 2 * hparams.audio_patch_sizes[i];
if (enc.conv_w[i]->ne[0] != k_expected) {
throw std::runtime_error(mtmd_format("%s: %s has kernel %" PRId64 ", expected %" PRId64 "\n", __func__, ggml_get_name(enc.conv_w[i]), enc.conv_w[i]->ne[0], k_expected));
}
}
enc.blocks.resize(hparams.n_layer);
for (int il = 0; il < hparams.n_layer; ++il) {
auto & layer = enc.blocks[il];
layer.ln_1_w = get_tensor(mtmd_format(TN_A_CNX_NORM, e, il, "weight"));
layer.conv_w = get_tensor(mtmd_format(TN_A_CNX_DW, e, il, "weight"));
layer.conv_b = get_tensor(mtmd_format(TN_A_CNX_DW, e, il, "bias"));
layer.ls_1_w = get_tensor(mtmd_format(TN_A_CNX_LS1, e, il, "weight"));
layer.ln_2_w = get_tensor(mtmd_format(TN_A_CNX_FFN_NORM, e, il, "weight"));
layer.ff_up_w = get_tensor(mtmd_format(TN_A_CNX_FFN_UP, e, il, "weight"));
layer.ff_up_b = get_tensor(mtmd_format(TN_A_CNX_FFN_UP, e, il, "bias"));
layer.ff_down_w = get_tensor(mtmd_format(TN_A_CNX_FFN_DOWN, e, il, "weight"));
layer.ff_down_b = get_tensor(mtmd_format(TN_A_CNX_FFN_DOWN, e, il, "bias"));
layer.ls_2_w = get_tensor(mtmd_format(TN_A_CNX_LS2, e, il, "weight"));
if (layer.conv_w->ne[0] != hparams.audio_conv_kernel_size || layer.conv_w->type != GGML_TYPE_F32) {
throw std::runtime_error(mtmd_format("%s: %s must be F32 with kernel %d\n", __func__, ggml_get_name(layer.conv_w), hparams.audio_conv_kernel_size));
}
}
}
// acoustic connector: mlp 0 -> norm -> mlp 1, semantic connector: mlp 2 -> norm -> mlp 3
model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias"));
model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP_NORM, 0, "weight"));
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias"));
model.mm_norm_mid_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP_NORM, 2, "weight"));
model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight"));
model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias"));
} break;
case PROJECTOR_TYPE_INKLING2V:
{
// these come from the generic path above, where they are optional
if (!model.class_embedding || !model.position_embeddings || !model.pre_ln_w || !model.pre_ln_b) {
throw std::runtime_error(mtmd_format("%s: missing class_embd, position_embd or pre_ln\n", __func__));
}
model.class_position_embedding = get_tensor(TN_CLASS_POS_EMBD);
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"), false);
model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false);
} break;
case PROJECTOR_TYPE_MUSE_GLIMMER:
{
// 3-linear MLP: fc -> erf-GELU -> proj -> erf-GELU -> vision_proj (into LLM residual dim)
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
} break;
case PROJECTOR_TYPE_STEP3VL:
{
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"), false);
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"), false);
model.mm_model_proj = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight"));
} break;
case PROJECTOR_TYPE_YOUTUVL:
{
model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); // merger.ln_q (RMS norm)
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); // merger.mlp.0
model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); // merger.mlp.2
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_YASA2:
{
// reuse tensors already loaded by the common section
// (TN_PATCH_EMBD and TN_PATCH_BIAS have the same tensor names)
model.yasa_patch_w = model.patch_embeddings_0;
model.yasa_patch_b = model.patch_bias;
model.yasa_patch_ln_w = get_tensor(TN_YASA_PATCH_LN_W, false);
model.yasa_patch_ln_b = get_tensor(TN_YASA_PATCH_LN_B, false);
model.yasa_backbone_ln_w = get_tensor(TN_YASA_BACKBONE_LN_W, false);
model.yasa_backbone_ln_b = get_tensor(TN_YASA_BACKBONE_LN_B, false);
model.yasa_vision_pos_embed = get_tensor(TN_YASA_POS_EMBD, false);
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"), false);
model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false);
model.yasa_stages.clear();
for (int s = 0; ; ++s) {
yasa2_stage stage;
stage.down_ln_w = get_tensor(mtmd_format(TN_YASA_STAGE_DOWN_LN, s, "weight"), false);
stage.down_ln_b = get_tensor(mtmd_format(TN_YASA_STAGE_DOWN_LN, s, "bias"), false);
stage.down_conv_w = get_tensor(mtmd_format(TN_YASA_STAGE_DOWN_CONV, s, "weight"), false);
stage.down_conv_b = get_tensor(mtmd_format(TN_YASA_STAGE_DOWN_CONV, s, "bias"), false);
for (int bi = 0; ; ++bi) {
yasa2_block blk;
blk.dw_w = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "dw", "weight"), false);
if (!blk.dw_w) {
break;
}
blk.dw_b = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "dw", "bias"), false);
blk.ln_w = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "ln", "weight"), false);
blk.ln_b = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "ln", "bias"), false);
blk.pw1_w = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "pw1", "weight"), false);
blk.pw1_b = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "pw1", "bias"), false);
blk.grn_w = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "grn", "weight"), false);
blk.grn_b = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "grn", "bias"), false);
blk.pw2_w = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "pw2", "weight"), false);
blk.pw2_b = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "pw2", "bias"), false);
stage.blocks.push_back(blk);
}
if (!stage.down_conv_w && stage.blocks.empty()) {
break;
}
model.yasa_stages.push_back(std::move(stage));
}
} break;
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_GLM5V:
{
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight"));
model.mm_ffn_up_w = get_tensor(mtmd_format(TN_MM_UP, "weight"));
model.mm_ffn_up_b = get_tensor(mtmd_format(TN_MM_UP, "bias"), false);
model.mm_ffn_gate_w = get_tensor(mtmd_format(TN_MM_GATE, "weight"));
model.mm_ffn_gate_b = get_tensor(mtmd_format(TN_MM_GATE, "bias"), false);
model.mm_ffn_down_w = get_tensor(mtmd_format(TN_MM_DOWN, "weight"));
model.mm_ffn_down_b = get_tensor(mtmd_format(TN_MM_DOWN, "bias"), false);
model.mm_post_norm_w = get_tensor(mtmd_format(TN_MM_POST_NORM, "weight"));
model.mm_post_norm_b = get_tensor(mtmd_format(TN_MM_POST_NORM, "bias"), false);
model.mm_patch_merger_w = get_tensor(mtmd_format(TN_MM_PATCH_MERGER, "weight"));
model.mm_patch_merger_b = get_tensor(mtmd_format(TN_MM_PATCH_MERGER, "bias"));
} break;
case PROJECTOR_TYPE_GEMMA3:
{
model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ);
model.mm_soft_emb_norm_w = get_tensor(TN_MM_SOFT_EMB_N);
} break;
case PROJECTOR_TYPE_GEMMA4V:
{
model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ);
model.std_bias = get_tensor(TN_STD_BIAS, false);
model.std_scale = get_tensor(TN_STD_SCALE, false);
// load scalar for Gemma4ClippableLinear
for (auto * tensor : tensors_to_load) {
std::string name = tensor->name;
if (string_ends_with(name, ".weight")) {
std::string name_inp_max = name;
std::string name_inp_min = name;
std::string name_out_max = name;
std::string name_out_min = name;
string_replace_all(name_inp_max, ".weight", ".input_max");
string_replace_all(name_inp_min, ".weight", ".input_min");
string_replace_all(name_out_max, ".weight", ".output_max");
string_replace_all(name_out_min, ".weight", ".output_min");
model.clamp_info_map[name] = {
get_scalar(name_inp_max, FLT_MAX),
get_scalar(name_inp_min, -FLT_MAX),
get_scalar(name_out_max, FLT_MAX),
get_scalar(name_out_min, -FLT_MAX)
};
}
}
} break;
case PROJECTOR_TYPE_GEMMA4UV:
{
model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ);
model.patch_norm_1_w = get_tensor(mtmd_format(TN_PATCH_NORM, 1, "weight"));
model.patch_norm_1_b = get_tensor(mtmd_format(TN_PATCH_NORM, 1, "bias"));
model.patch_norm_2_w = get_tensor(mtmd_format(TN_PATCH_NORM, 2, "weight"));
model.patch_norm_2_b = get_tensor(mtmd_format(TN_PATCH_NORM, 2, "bias"));
model.patch_norm_3_w = get_tensor(mtmd_format(TN_PATCH_NORM, 3, "weight")); // pos_norm
model.patch_norm_3_b = get_tensor(mtmd_format(TN_PATCH_NORM, 3, "bias")); // pos_norm
} break;
case PROJECTOR_TYPE_GEMMA3NV:
{
model.mobilenet_stem_conv_w = get_tensor(TN_MNV5_STEM_CONV, false);
model.mobilenet_stem_conv_b = get_tensor(TN_MNV5_STEM_BIAS, false);
model.mobilenet_stem_norm_w = get_tensor(TN_MNV5_STEM_BN, false);
model.msfa_ffn_expand_w = get_tensor(TN_MNV5_MSFA_FFN_EXP_W, false);
model.msfa_ffn_expand_bn = get_tensor(TN_MNV5_MSFA_FFN_EXP_BN, false); // Consume BN if present but likely folded
model.msfa_ffn_project_w = get_tensor(TN_MNV5_MSFA_FFN_PROJ_W, false);
model.msfa_ffn_project_bn = get_tensor(TN_MNV5_MSFA_FFN_PROJ_BN, false);
model.msfa_concat_norm_w = get_tensor(TN_MNV5_MSFA_NORM, false);
// Dynamically load blocks stage by stage
for (int stage = 0; stage < 4; ++stage) {
int blocks_found_in_stage = 0;
for (int blk_idx = 0; ; ++blk_idx) {
bool found_block = false;
mobilenetv5_block block;
// 1. Check for Edge Residual (S0)
block.s0_conv_exp_w = get_tensor(mtmd_format(TN_MNV5_BLK_S0_EXP_W, stage, blk_idx), false);
if (block.s0_conv_exp_w) {
found_block = true;
block.s0_bn1_w = get_tensor(mtmd_format(TN_MNV5_BLK_S0_BN1_W, stage, blk_idx), false);
block.s0_conv_pwl_w = get_tensor(mtmd_format(TN_MNV5_BLK_S0_PWL_W, stage, blk_idx), false);
block.s0_bn2_w = get_tensor(mtmd_format(TN_MNV5_BLK_S0_BN2_W, stage, blk_idx), false);
}
// 2. Check for UIR (Universal Inverted Residual)
else {
// Check for dw_start OR pw_exp (some UIR blocks skip dw_start)
block.dw_start_w = get_tensor(mtmd_format(TN_MNV5_BLK_DW_START_W, stage, blk_idx), false);
block.pw_exp_w = get_tensor(mtmd_format(TN_MNV5_BLK_PW_EXP_W, stage, blk_idx), false);
if (block.dw_start_w || block.pw_exp_w) {
found_block = true;
if (block.dw_start_w) {
block.dw_start_bn_w = get_tensor(mtmd_format(TN_MNV5_BLK_DW_START_BN, stage, blk_idx), false);
}
if (block.pw_exp_w) {
block.pw_exp_bn_w = get_tensor(mtmd_format(TN_MNV5_BLK_PW_EXP_BN, stage, blk_idx), false);
}
block.dw_mid_w = get_tensor(mtmd_format(TN_MNV5_BLK_DW_MID_W, stage, blk_idx), false);
if (block.dw_mid_w) {
block.dw_mid_bn_w = get_tensor(mtmd_format(TN_MNV5_BLK_DW_MID_BN, stage, blk_idx), false);
}
block.pw_proj_w = get_tensor(mtmd_format(TN_MNV5_BLK_PW_PROJ_W, stage, blk_idx), false);
if (block.pw_proj_w) {
block.pw_proj_bn_w = get_tensor(mtmd_format(TN_MNV5_BLK_PW_PROJ_BN, stage, blk_idx), false);
}
block.layer_scale_w = get_tensor(mtmd_format(TN_MNV5_BLK_LAYER_SCALE, stage, blk_idx), false);
}
}
// 3. Check for Attention (MQA)
// Even if UIR/Edge check failed, this might be a pure attention block
ggml_tensor* attn_q_check = get_tensor(mtmd_format(TN_MNV5_ATTN_Q_W, stage, blk_idx), false);
if (attn_q_check) {
found_block = true;
block.attn_q_w = attn_q_check;
block.attn_k_w = get_tensor(mtmd_format(TN_MNV5_ATTN_K_W, stage, blk_idx), false);
block.attn_v_w = get_tensor(mtmd_format(TN_MNV5_ATTN_V_W, stage, blk_idx), false);
block.attn_o_w = get_tensor(mtmd_format(TN_MNV5_ATTN_O_W, stage, blk_idx), false);
block.attn_k_dw_w = get_tensor(mtmd_format(TN_MNV5_ATTN_K_DW, stage, blk_idx), false);
block.attn_k_norm_w = get_tensor(mtmd_format(TN_MNV5_ATTN_K_NORM, stage, blk_idx), false);
block.attn_v_dw_w = get_tensor(mtmd_format(TN_MNV5_ATTN_V_DW, stage, blk_idx), false);
block.attn_v_norm_w = get_tensor(mtmd_format(TN_MNV5_ATTN_V_NORM, stage, blk_idx), false);
block.attn_norm_w = get_tensor(mtmd_format(TN_MNV5_ATTN_NORM, stage, blk_idx), false);
// Note: Attention blocks also have layer_scale, load it if not already loaded by UIR check
if (!block.layer_scale_w) {
block.layer_scale_w = get_tensor(mtmd_format(TN_MNV5_BLK_LAYER_SCALE, stage, blk_idx), false);
}
}
if (found_block) {
model.mobilenet_blocks.push_back(block);
blocks_found_in_stage++;
} else {
// End of blocks for this stage
break;
}
}
// Track where this stage ends in the flat vector
if (blocks_found_in_stage > 0) {
model.mobilenet_stage_ends.push_back(model.mobilenet_blocks.size() - 1);
LOG_INF("%s: Stage %d ended at global block index %zu\n", __func__, stage, model.mobilenet_blocks.size() - 1);
}
}
model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ);
model.mm_soft_emb_norm_w = get_tensor(TN_MM_SOFT_EMB_N);
} break;
case PROJECTOR_TYPE_IDEFICS3:
{
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight"));
} break;
case PROJECTOR_TYPE_COHERE2V:
{
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_LFM2:
case PROJECTOR_TYPE_D1OMNI_V:
{
model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM, false);
model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B, false);
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_KIMIVL:
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_KIMIK25:
{
model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);
model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B);
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_KIMIK31:
{
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_post_norm_w = get_tensor(mtmd_format(TN_MM_POST_NORM, "weight"));
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
// sentinel token embeddings written into the output block
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
model.token_embd_img_start = get_tensor(TN_TOK_IMG_START);
model.token_embd_img_end = get_tensor(TN_TOK_IMG_END);
model.token_embd_img_pad = get_tensor(TN_TOK_IMG_PAD);
} break;
case PROJECTOR_TYPE_DEEPSEEK41V:
{
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
// image_start / image_end / image_newline, written into the output block
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
model.token_embd_img_start = get_tensor(TN_TOK_IMG_START);
model.token_embd_img_end = get_tensor(TN_TOK_IMG_END);
} break;
case PROJECTOR_TYPE_PIXTRAL:
{
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"), false);
model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false);
// [IMG_BREAK] token embedding
model.token_embd_img_break = get_tensor(TN_TOK_IMG_BREAK);
// for mistral small 3.1
model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM, false);
model.mm_patch_merger_w = get_tensor(mtmd_format(TN_MM_PATCH_MERGER, "weight"), false);
} break;
case PROJECTOR_TYPE_LIGHTONOCR:
{
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"), false);
model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false);
model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM, false);
model.mm_patch_merger_w = get_tensor(mtmd_format(TN_MM_PATCH_MERGER, "weight"), false);
} break;
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
{
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);
model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B);
// post_trunk_norm: applied after all ViT blocks, before the merger
model.post_ln_w = get_tensor(mtmd_format(TN_MM_POST_NORM, "weight"));
} break;
case PROJECTOR_TYPE_DOTS3NOTE_A:
{
model.conv2d_1_w = get_tensor(mtmd_format(TN_CONV2D, 1, "weight"));
model.conv2d_1_b = get_tensor(mtmd_format(TN_CONV2D, 1, "bias"));
model.conv2d_2_w = get_tensor(mtmd_format(TN_CONV2D, 2, "weight"));
model.conv2d_2_b = get_tensor(mtmd_format(TN_CONV2D, 2, "bias"));
model.conv2d_3_w = get_tensor(mtmd_format(TN_CONV2D, 3, "weight"));
model.conv2d_3_b = get_tensor(mtmd_format(TN_CONV2D, 3, "bias"));
model.conv_out_w = get_tensor(mtmd_format(TN_CONV_OUT, "weight")); // no bias
// adapter: LayerNorm -> Linear -> GELU -> Linear
model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_NORM_PRE, "weight"));
model.mm_norm_pre_b = get_tensor(mtmd_format(TN_MM_NORM_PRE, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias"));
} break;
case PROJECTOR_TYPE_ULTRAVOX:
{
model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight"));
model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias"));
model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight"));
model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_NORM_PRE, "weight"));
model.mm_norm_mid_w = get_tensor(mtmd_format(TN_MM_NORM_MID, "weight"));
} break;
case PROJECTOR_TYPE_MERALION:
{
// Whisper encoder conv layers
model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight"));
model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias"));
model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight"));
model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias"));
// MERaLiON adaptor: 4 linear layers + ln_pre
// linear_0 = frame compression (19200->6400) + SiLU
// linear_1 = gate_proj (6400->6400) for GLU
// linear_2 = pool_proj (6400->6400) for GLU
// linear_3 = out_proj (6400->3584)
model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias"));
model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight"));
model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias"));
// ln_speech (LayerNorm before adaptor)
model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_NORM_PRE, "weight"));
model.mm_norm_pre_b = get_tensor(mtmd_format(TN_MM_NORM_PRE, "bias"));
} break;
case PROJECTOR_TYPE_QWEN2A:
{
model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight"));
model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias"));
model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight"));
model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias"));
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight"));
model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias"));
} break;
case PROJECTOR_TYPE_HIGGS_AUDIO:
{
model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight"));
model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias"));
model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight"));
model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias"));
model.downsample_conv_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight"));
model.downsample_conv_b = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias"));
} break;
case PROJECTOR_TYPE_QWEN3A:
{
model.conv2d_1_w = get_tensor(mtmd_format(TN_CONV2D, 1, "weight"));
model.conv2d_1_b = get_tensor(mtmd_format(TN_CONV2D, 1, "bias"));
model.conv2d_2_w = get_tensor(mtmd_format(TN_CONV2D, 2, "weight"));
model.conv2d_2_b = get_tensor(mtmd_format(TN_CONV2D, 2, "bias"));
model.conv2d_3_w = get_tensor(mtmd_format(TN_CONV2D, 3, "weight"));
model.conv2d_3_b = get_tensor(mtmd_format(TN_CONV2D, 3, "bias"));
model.conv_out_w = get_tensor(mtmd_format(TN_CONV_OUT, "weight")); // no bias
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias"));
} break;
case PROJECTOR_TYPE_MIMO_AUDIO:
{
model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight"));
model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias"));
model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight"));
model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias"));
model.downsample_conv_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight"));
model.downsample_norm_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_NORM, "weight"));
model.downsample_norm_b = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_NORM, "bias"));
model.rvq_codebook = get_tensor(mtmd_format(TN_A_RVQ_CODEBOOK, "weight"), false);
model.mm_a_code_embd = get_tensor(mtmd_format(TN_MM_A_CODE_EMBD, "weight"), false);
if (!model.rvq_codebook || !model.mm_a_code_embd) {
throw std::runtime_error(mtmd_format("%s: mimo_audio: missing %s or %s\n", __func__,
TN_A_RVQ_CODEBOOK, TN_MM_A_CODE_EMBD));
}
// hparams.rvq_codebook_size comes from GGUF metadata and is independent of the
// tensors' actual shapes - bound it so codebook/code_embd views built from it
// (mimo-audio.cpp) can never read past either tensor's allocated bins/vocab.
for (int32_t bins : hparams.rvq_codebook_size) {
if (bins <= 0 || bins > model.rvq_codebook->ne[1] || bins > model.mm_a_code_embd->ne[1]) {
throw std::runtime_error(mtmd_format(
"%s: mimo_audio: %s entry (%d) out of range for codebook/code_embd tensors\n",
__func__, KEY_A_RVQ_CODEBOOK_SIZE, bins));
}
}
// LLM-side connector: input_local_transformer + projection
model.mm_a_local_layers.resize(hparams.audio_local_n_layer);
for (int il = 0; il < hparams.audio_local_n_layer; il++) {
auto & layer = model.mm_a_local_layers[il];
layer.q_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_Q, il, "weight"));
layer.q_b = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_Q, il, "bias"));
layer.k_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_K, il, "weight"));
layer.k_b = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_K, il, "bias"));
layer.v_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_V, il, "weight"));
layer.v_b = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_V, il, "bias"));
layer.o_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_OUT, il, "weight"));
layer.ff_gate_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_FFN_GATE, il, "weight"));
layer.ff_up_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_FFN_UP, il, "weight"));
layer.ff_down_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_FFN_DOWN, il, "weight"));
layer.ln_1_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_LN1, il, "weight"));
layer.ln_2_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_LN2, il, "weight"));
}
model.mm_a_local_norm_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_NORM, "weight"));
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
} break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
// stem TDNN (block 0)
model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 0, "weight"));
model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 0, "bias"));
// SE-Res2Net blocks (GGUF bid 1..3, one per hparams.n_layer)
model.layers.resize(hparams.n_layer);
for (int il = 0; il < hparams.n_layer; il++) {
auto & layer = model.layers[il];
int bid = il + 1;
layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, bid, "weight"));
layer.conv_pw1_b = get_tensor(mtmd_format(TN_CONV_PW1, prefix, bid, "bias"));
layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, bid, "weight"));
layer.conv_pw2_b = get_tensor(mtmd_format(TN_CONV_PW2, prefix, bid, "bias"));
layer.se_conv1_w = get_tensor(mtmd_format(TN_A_SE_CONV1, bid, "weight"));
layer.se_conv1_b = get_tensor(mtmd_format(TN_A_SE_CONV1, bid, "bias"));
layer.se_conv2_w = get_tensor(mtmd_format(TN_A_SE_CONV2, bid, "weight"));
layer.se_conv2_b = get_tensor(mtmd_format(TN_A_SE_CONV2, bid, "bias"));
layer.res2_conv_w.resize(7);
layer.res2_conv_b.resize(7);
for (int xid = 0; xid < 7; xid++) {
layer.res2_conv_w[xid] = get_tensor(mtmd_format(TN_A_CONV_RES2, bid, xid, "weight"));
layer.res2_conv_b[xid] = get_tensor(mtmd_format(TN_A_CONV_RES2, bid, xid, "bias"));
}
}
// multi-layer feature aggregation
model.conv_out_w = get_tensor(mtmd_format(TN_CONV_OUT, "weight"));
model.conv_out_b = get_tensor(mtmd_format(TN_CONV_OUT, "bias"));
// attentive statistics pooling
model.spk_asp_attn_w = get_tensor(mtmd_format(TN_A_ASP_ATTN, "weight"));
model.spk_asp_attn_b = get_tensor(mtmd_format(TN_A_ASP_ATTN, "bias"));
model.spk_asp_tdnn_w = get_tensor(mtmd_format(TN_A_ASP_TDNN, "weight"));
model.spk_asp_tdnn_b = get_tensor(mtmd_format(TN_A_ASP_TDNN, "bias"));
// final speaker embedding projection
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight"));
model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias"));
} break;
case PROJECTOR_TYPE_FISH_S2_SPKENC:
{
// codec encoder in the code2wav structs: dac = encoder blocks, upsample = quantizer downsample
// generic a.blk.* layers: 0..3 last encoder block, 4..11 quantizer pre_module
auto & c2w = model.c2w;
c2w.dac_entry_w = get_tensor(mtmd_format(TN_A_CODEC_ENC_ENTRY, "weight"));
c2w.dac_entry_b = get_tensor(mtmd_format(TN_A_CODEC_ENC_ENTRY, "bias"));
c2w.dac.resize(4);
for (int il = 0; il < (int) c2w.dac.size(); il++) {
auto & blk = c2w.dac[il];
blk.snake_alpha = get_tensor(mtmd_format(TN_A_CODEC_ENC_SNAKE, il, "alpha"));
blk.snake_beta = get_tensor(mtmd_format(TN_A_CODEC_ENC_SNAKE, il, "beta"));
blk.conv_w = get_tensor(mtmd_format(TN_A_CODEC_ENC_CONV, il, "weight"));
blk.conv_b = get_tensor(mtmd_format(TN_A_CODEC_ENC_CONV, il, "bias"));
blk.res.resize(3);
for (int ir = 0; ir < (int) blk.res.size(); ir++) {
auto & res = blk.res[ir];
res.act1_alpha = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_ACT1, il, ir, "alpha"));
res.act1_beta = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_ACT1, il, ir, "beta"));
res.conv1_w = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_CONV1, il, ir, "weight"));
res.conv1_b = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_CONV1, il, ir, "bias"));
res.act2_alpha = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_ACT2, il, ir, "alpha"));
res.act2_beta = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_ACT2, il, ir, "beta"));
res.conv2_w = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_CONV2, il, ir, "weight"));
res.conv2_b = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_CONV2, il, ir, "bias"));
}
}
c2w.dac_post_snake_alpha = get_tensor(mtmd_format(TN_A_CODEC_ENC_POST_SNAKE, "alpha"));
c2w.dac_post_snake_beta = get_tensor(mtmd_format(TN_A_CODEC_ENC_POST_SNAKE, "beta"));
c2w.dac_post_conv_w = get_tensor(mtmd_format(TN_A_CODEC_ENC_POST_CONV, "weight"));
c2w.dac_post_conv_b = get_tensor(mtmd_format(TN_A_CODEC_ENC_POST_CONV, "bias"));
c2w.upsample.resize(2);
for (int il = 0; il < (int) c2w.upsample.size(); il++) {
auto & down = c2w.upsample[il];
down.conv_w = get_tensor(mtmd_format(TN_A_CODEC_DOWN_CONV, il, "weight"));
down.conv_b = get_tensor(mtmd_format(TN_A_CODEC_DOWN_CONV, il, "bias"));
down.dwconv_w = get_tensor(mtmd_format(TN_A_CODEC_DOWN_DWCONV, il, "weight"));
down.dwconv_b = get_tensor(mtmd_format(TN_A_CODEC_DOWN_DWCONV, il, "bias"));
down.norm_w = get_tensor(mtmd_format(TN_A_CODEC_DOWN_NORM, il, "weight"));
down.norm_b = get_tensor(mtmd_format(TN_A_CODEC_DOWN_NORM, il, "bias"));
down.pw1_w = get_tensor(mtmd_format(TN_A_CODEC_DOWN_PW1, il, "weight"));
down.pw1_b = get_tensor(mtmd_format(TN_A_CODEC_DOWN_PW1, il, "bias"));
down.pw2_w = get_tensor(mtmd_format(TN_A_CODEC_DOWN_PW2, il, "weight"));
down.pw2_b = get_tensor(mtmd_format(TN_A_CODEC_DOWN_PW2, il, "bias"));
down.gamma = get_tensor(mtmd_format(TN_A_CODEC_DOWN_GAMMA, il));
}
c2w.tfm_output_norm_w = get_tensor(mtmd_format(TN_A_CODEC_PRE_NORM, "weight"));
if (hparams.n_layer != mtmd_graph_fish_s2_spkenc::n_tfm_layer) {
throw std::runtime_error(mtmd_format("%s: fish_s2_spkenc expects %d transformer layers\n", __func__, mtmd_graph_fish_s2_spkenc::n_tfm_layer));
}
// VQ: search with the normalized codebooks, subtract the codec decoder's out_proj tables
c2w.quant_first_in_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_FIRST_IN, "weight"));
c2w.quant_first_in_b = get_tensor(mtmd_format(TN_A_CODEC_VQ_FIRST_IN, "bias"));
c2w.quant_first_cb_norm_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_FIRST_CB, "weight"));
c2w.quant_rest_in_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_REST_IN, "weight"));
c2w.quant_rest_in_b = get_tensor(mtmd_format(TN_A_CODEC_VQ_REST_IN, "bias"));
c2w.quant_rest_cb_norm_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_REST_CB, "weight"));
c2w.quant_first_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight"));
c2w.quant_rest_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_CB, "weight"));
// codes -> backbone rows, same tables as fish_s2_gen
model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight"));
model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
} break;
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
{
load_seanet(model.seanet, false);
model.downsample_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight"));
model.spk_proj_w = get_tensor(mtmd_format(TN_A_SPEAKER_PROJ, "weight"));
} break;
case PROJECTOR_TYPE_POCKETTTS_GEN:
{
auto & flow = model.flow;
flow.input_proj_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_INPUT_PROJ, "weight"));
flow.input_proj_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_INPUT_PROJ, "bias"));
flow.cond_embd_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_COND_EMBD, "weight"));
flow.cond_embd_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_COND_EMBD, "bias"));
flow.final_ada_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_FINAL_ADA, "weight"));
flow.final_ada_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_FINAL_ADA, "bias"));
flow.final_proj_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_FINAL_PROJ, "weight"));
flow.final_proj_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_FINAL_PROJ, "bias"));
flow.time.resize(2);
for (size_t i = 0; i < flow.time.size(); i++) {
auto & t = flow.time[i];
t.freqs = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_FREQS, (int) i));
t.up_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_UP, (int) i, "weight"));
t.up_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_UP, (int) i, "bias"));
t.down_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_DOWN, (int) i, "weight"));
t.down_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_DOWN, (int) i, "bias"));
t.norm = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_NORM, (int) i));
}
// one AdaLN block per flow depth, the count is only known from the tensors
for (int il = 0; ; il++) {
ggml_tensor * probe = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_NORM, il, "weight"), false);
if (probe == nullptr) {
break;
}
mtmd_flow_net::block blk;
blk.norm_w = probe;
blk.norm_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_NORM, il, "bias"));
blk.up_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_UP, il, "weight"));
blk.up_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_UP, il, "bias"));
blk.down_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_DOWN, il, "weight"));
blk.down_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_DOWN, il, "bias"));
blk.ada_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_ADA, il, "weight"));
blk.ada_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_ADA, il, "bias"));
flow.blocks.push_back(blk);
}
model.gen_out_eos_w = get_tensor(mtmd_format(TN_A_GEN_OUT_EOS, "weight"));
model.gen_out_eos_b = get_tensor(mtmd_format(TN_A_GEN_OUT_EOS, "bias"));
model.gen_input_lin_w = get_tensor(mtmd_format(TN_A_GEN_INPUT_LINEAR, "weight"));
model.gen_emb_mean = get_tensor(TN_A_GEN_EMB_MEAN);
model.gen_emb_std = get_tensor(TN_A_GEN_EMB_STD);
// mimi decoder
model.gen_quant_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_OUT, "weight"));
load_mimi_decoder(hparams.n_layer);
for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) {
const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);
if (strncmp(name, "voice.", 6) != 0) {
continue;
}
const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name);
if (t->type != GGML_TYPE_F32 || ggml_n_dims(t) > 2 || t->ne[0] != model.gen_input_lin_w->ne[1]) {
throw std::runtime_error(mtmd_format("%s: invalid voice tensor %s\n", __func__, name));
}
auto & embd = model.gen_voices[name + 6].rows;
embd.resize(ggml_nelements(t));
fin.seekg(tensor_offset.at(name), std::ios::beg);
if (!fin.read(reinterpret_cast<char *>(embd.data()), ggml_nbytes(t))) {
throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name));
}
}
if (!model.gen_voices.empty()) {
LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size());
}
} break;
case PROJECTOR_TYPE_MOSHI:
{
// streaming mimi encoder, its RVQ, then the backbone embeddings of the input codes
load_seanet(model.seanet, false);
model.downsample_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight"));
auto & c2w = model.c2w;
c2w.quant_first_in_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_FIRST_IN, "weight"));
c2w.quant_first_cb_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_FIRST_CB, "weight"));
c2w.quant_rest_in_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_REST_IN, "weight"));
c2w.quant_rest_cb_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_REST_CB, "weight"));
model.mm_a_code_embd = get_tensor(mtmd_format(TN_MM_A_CODE_EMBD, "weight"));
if (model.mm_a_code_embd->ne[2] != c2w.quant_rest_cb_w->ne[2] + 1) {
throw std::runtime_error(mtmd_format("%s: the input code embeddings do not match the codebooks\n", __func__));
}
} break;
case PROJECTOR_TYPE_MOSHI_GEN:
{
// depformer, the generic loop above loaded the a.gen.code.blk.* tensors (per-step weights on dim 2)
model.gen_code_proj_in_w = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "weight"));
model.gen_code_tok_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_TOK_EMBD, "weight"));
model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight"));
model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight"));
model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
const int64_t n_q = model.gen_code_head_w->ne[2];
if (model.gen_code_embd_w->ne[2] != n_q - 1 || model.gen_code_out_embd_w->ne[2] != n_q ||
model.gen_code_proj_in_w->ne[2] > n_q || (int64_t) n_q > mtmd_graph_moshi_gen::n_q_max) {
throw std::runtime_error(mtmd_format("%s: invalid depformer tensors\n", __func__));
}
// RVQ decode, then the mimi decoder
auto & c2w = model.c2w;
c2w.quant_first_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight"));
c2w.quant_first_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight"));
c2w.quant_rest_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_OUT, "weight"));
c2w.quant_rest_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_CB, "weight"));
load_mimi_decoder(mtmd_graph_moshi_gen::mimi_n_layer);
} break;
case PROJECTOR_TYPE_LFM3A_GEN:
{
// must match the "tts" chat template, see conversion/lfm3.py
// note for contributor: this is one-off fix, do not repeat this for future models
model.gen_voices = { {"female", {}}, {"male", {}} };
// depthformer, the generic loop above loaded the a.gen.code.blk.* tensors
for (int il = 0; il < hparams.n_layer; ++il) {
model.layers[il].attn_gate_w = get_tensor(mtmd_format(TN_ATTN_GATE, prefix, il, "weight"));
}
model.gen_code_proj_in_w = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "weight"));
model.gen_code_proj_in_b = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "bias"));
model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight")); // tied head
model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
model.gen_code_norm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_NORM, "weight"));
model.gen_code_head_norm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD_NORM, "weight"));
model.gen_code_mask_embd = get_tensor(TN_A_GEN_CODE_MASK_EMBD);
// codec decoder: FSQ level table, one projection over all codebooks, LFM3 decoder -> polar STFT frames
auto & c2w = model.c2w;
c2w.quant_first_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight"));
c2w.quant_first_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight"));
c2w.quant_first_out_b = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "bias"));
c2w.tfm_in_proj_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_IN_PROJ, "weight"));
c2w.tfm_in_proj_b = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_IN_PROJ, "bias"));
c2w.tfm_out_proj_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "weight"));
c2w.tfm_out_proj_b = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias"));
c2w.tfm_output_norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight"));
// each layer is a short conv block or a gated attention block, both with sandwich norms
c2w.tfm_layers.resize(hparams.wav_tfm_n_layer);
for (int il = 0; il < hparams.wav_tfm_n_layer; il++) {
auto & layer = c2w.tfm_layers[il];
const char * p = "a.gen.wav.tfm";
layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, p, il, "weight"));
layer.attn_post_norm_w = get_tensor(mtmd_format(TN_ATTN_POST_NORM, p, il, "weight"));
layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, p, il, "weight"));
layer.ff_post_norm_w = get_tensor(mtmd_format(TN_FFN_POST_NORM, p, il, "weight"));
layer.ff_gate_w = get_tensor(mtmd_format(TN_FFN_GATE, p, il, "weight"));
layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, p, il, "weight"));
layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, p, il, "weight"));
layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, p, il, "weight"), false);
layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, p, il, "weight"), false);
layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, p, il, "weight"), false);
layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, p, il, "weight"), false);
layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, p, il, "weight"), false);
layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, p, il, "weight"), false);
layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight"), false);
layer.q_norm = get_tensor(mtmd_format(TN_ATTN_Q_NORM, p, il, "weight"), false);
layer.k_norm = get_tensor(mtmd_format(TN_ATTN_K_NORM, p, il, "weight"), false);
layer.attn_gate_w = get_tensor(mtmd_format(TN_ATTN_GATE, p, il, "weight"), false);
const bool is_conv = layer.conv_pw1_w && layer.conv_dw_w && layer.conv_pw2_w;
const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w && layer.q_norm && layer.k_norm && layer.attn_gate_w;
if (is_conv == is_attn) {
throw std::runtime_error(mtmd_format("%s: codec layer %d is neither a conv nor an attention block\n", __func__, il));
}
if (is_attn) {
const int d_head = layer.q_norm->ne[0];
hparams.wav_tfm_n_head = layer.q_w->ne[1] / d_head;
hparams.wav_tfm_n_head_kv = layer.k_w->ne[1] / d_head;
}
}
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
// code_predictor, proj_in is absent when the talker and the predictor share the hidden size
model.gen_code_proj_in_w = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "weight"), false);
model.gen_code_proj_in_b = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "bias"), false);
model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight"));
model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight"));
model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
model.gen_code_norm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_NORM, "weight"));
// code2wav: RVQ codes -> raw PCM, lives in the same ctx as code_predictor
load_code2wav(true);
// reference encoder, absent from files converted before it was added
model.ref_quant_rest_cb = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_REST_CB, "weight"), false);
if (model.ref_quant_rest_cb) {
load_seanet(model.seanet, false);
model.downsample_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight"));
for (int il = 0; ; il++) {
const char * p = "a.gen.enc.tfm";
ggml_tensor * ln_1_w = get_tensor(mtmd_format(TN_LN_1, p, il, "weight"), false);
if (!ln_1_w) {
break;
}
mtmd_layer layer;
layer.ln_1_w = ln_1_w;
layer.ln_1_b = get_tensor(mtmd_format(TN_LN_1, p, il, "bias"));
layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, p, il, "weight"));
layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, p, il, "weight"));
layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, p, il, "weight"));
layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight"));
layer.ls_1_w = get_tensor(mtmd_format(TN_LS_1, p, il, "weight"));
layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, p, il, "weight"));
layer.ln_2_b = get_tensor(mtmd_format(TN_LN_2, p, il, "bias"));
layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, p, il, "weight"));
layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, p, il, "weight"));
layer.ls_2_w = get_tensor(mtmd_format(TN_LS_2, p, il, "weight"));
model.ref_enc_layers.push_back(layer);
}
model.ref_quant_first_in_w = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_FIRST_IN, "weight"));
model.ref_quant_first_cb = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_FIRST_CB, "weight"));
model.ref_quant_rest_in_w = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_REST_IN, "weight"));
// the codes index the code predictor tables, see build_ref_encoder()
const int64_t n_acoustic = model.ref_quant_rest_cb->ne[2];
if (model.ref_enc_layers.empty() ||
n_acoustic != model.gen_code_head_w->ne[2] ||
n_acoustic > model.gen_code_embd_w->ne[2] ||
model.ref_quant_first_cb->ne[1] > model.gen_code_out_embd_w->ne[1] ||
model.ref_quant_rest_cb->ne[1] > model.gen_code_embd_w->ne[1]) {
throw std::runtime_error(mtmd_format("%s: qwen3tts: unexpected reference encoder shape\n", __func__));
}
}
} break;
case PROJECTOR_TYPE_ZONOS3_GEN:
{
model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight"));
model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
if (model.gen_code_out_embd_w->ne[1] % mtmd_graph_zonos3_gen::n_codebooks != 0 ||
model.gen_code_head_w->ne[1] != model.gen_code_out_embd_w->ne[1]) {
throw std::runtime_error(mtmd_format("%s: zonos3: unexpected codebook table shape\n", __func__));
}
// voice.<name>: the voice rows of the prompt
for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) {
const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);
if (strncmp(name, "voice.", 6) != 0) {
continue;
}
const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name);
if (t->type != GGML_TYPE_F32 || ggml_n_dims(t) > 2 || t->ne[0] != model.gen_code_out_embd_w->ne[0]) {
throw std::runtime_error(mtmd_format("%s: invalid voice tensor %s\n", __func__, name));
}
auto & embd = model.gen_voices[name + 6].rows;
embd.resize(ggml_nelements(t));
fin.seekg(tensor_offset.at(name), std::ios::beg);
if (!fin.read(reinterpret_cast<char *>(embd.data()), ggml_nbytes(t))) {
throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name));
}
}
LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size());
// ZAC codec decoder
auto & z = model.zonos3;
const std::string p = "a.gen.wav";
auto has = [&](const std::string & name) {
return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0;
};
auto load_convnext = [&](const std::string & q) {
mtmd_zonos3_wav::convnext b;
b.dw_w = get_tensor(q + ".dw.weight");
b.dw_b = get_tensor(q + ".dw.bias");
b.norm_w = get_tensor(q + ".norm.weight");
b.norm_b = get_tensor(q + ".norm.bias");
b.pw1_w = get_tensor(q + ".pw1.weight");
b.pw1_b = get_tensor(q + ".pw1.bias");
b.pw2_w = get_tensor(q + ".pw2.weight");
b.pw2_b = get_tensor(q + ".pw2.bias");
return b;
};
z.sem_table = get_tensor(p + ".sem.table");
z.sem_bias = get_tensor(p + ".sem.bias");
for (int i = 0; has(mtmd_format("%s.sem.%d.dw.weight", p.c_str(), i)); i++) {
z.sem.push_back(load_convnext(p + ".sem." + std::to_string(i)));
}
z.sem_out_w = get_tensor(p + ".sem.out.weight");
z.sem_out_b = get_tensor(p + ".sem.out.bias");
z.res_table = get_tensor(p + ".res.table");
z.res_bias = get_tensor(p + ".res.bias");
for (int s = 0; has(mtmd_format("%s.stage.%d.in.weight", p.c_str(), s)); s++) {
const std::string q = p + ".stage." + std::to_string(s);
mtmd_zonos3_wav::stage st;
st.in_w = get_tensor(q + ".in.weight");
st.out_w = get_tensor(q + ".out.weight", false);
for (int il = 0; has(mtmd_format("%s.blk.%d.qkv.weight", q.c_str(), il)); il++) {
const std::string b = q + ".blk." + std::to_string(il);
mtmd_zonos3_wav::layer l;
l.norm1_w = get_tensor(b + ".norm1.weight");
l.norm1_b = get_tensor(b + ".norm1.bias");
l.qkv_w = get_tensor(b + ".qkv.weight");
l.wo_w = get_tensor(b + ".wo.weight");
l.attn_scale = get_tensor(b + ".attn_scale");
l.norm2_w = get_tensor(b + ".norm2.weight");
l.norm2_b = get_tensor(b + ".norm2.bias");
l.ffn1_w = get_tensor(b + ".ffn1.weight");
l.ffn2_w = get_tensor(b + ".ffn2.weight");
l.ffn_scale = get_tensor(b + ".ffn_scale");
st.layers.push_back(l);
}
z.stages.push_back(std::move(st));
}
for (int i = 0; has(mtmd_format("%s.head.%d.dw.weight", p.c_str(), i)); i++) {
z.head.push_back(load_convnext(p + ".head." + std::to_string(i)));
}
z.head_norm_w = get_tensor(p + ".head.norm.weight");
z.head_norm_b = get_tensor(p + ".head.norm.bias");
z.head_out_w = get_tensor(p + ".head.out.weight");
z.head_out_b = get_tensor(p + ".head.out.bias");
if (z.stages.size() != mtmd_graph_zonos3_gen::n_stages || z.sem.empty() || z.head.empty() ||
z.head_out_w->ne[1] != mtmd_graph_zonos3_gen::n_fft[0] + mtmd_graph_zonos3_gen::n_fft[1] + 4) {
throw std::runtime_error(mtmd_format("%s: zonos3: unexpected codec decoder layout\n", __func__));
}
// read on the host
auto read_host = [&](const std::string & name, std::vector<float> & dst) -> const ggml_tensor * {
const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name.c_str());
if (!t || t->type != GGML_TYPE_F32 || ggml_n_dims(t) != 2) {
throw std::runtime_error(mtmd_format("%s: invalid or missing tensor %s\n", __func__, name.c_str()));
}
dst.resize(ggml_nelements(t));
fin.seekg(tensor_offset.at(name), std::ios::beg);
if (!fin.read(reinterpret_cast<char *>(dst.data()), ggml_nbytes(t))) {
throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name.c_str()));
}
return t;
};
const ggml_tensor * mixer = read_host(p + ".mixer.weight", z.mixer);
const ggml_tensor * rs = read_host(p + ".resample.weight", z.resample);
if (mixer->ne[1] != 2 || rs->ne[1] != mtmd_graph_zonos3_gen::rs_new ||
rs->ne[0] < mtmd_graph_zonos3_gen::rs_orig || (rs->ne[0] - mtmd_graph_zonos3_gen::rs_orig) % 2 != 0) {
throw std::runtime_error(mtmd_format("%s: zonos3: unexpected mixer or resampler shape\n", __func__));
}
z.fir_kernel = mixer->ne[0];
z.rs_new = rs->ne[1];
z.rs_kernel_len = rs->ne[0];
} break;
case PROJECTOR_TYPE_YUE2_GEN:
{
auto & y = model.yue2;
y.inp_w = get_tensor("a.gen.nar.inp.weight");
y.inp_b = get_tensor("a.gen.nar.inp.bias");
y.out_w = get_tensor("a.gen.nar.out.weight");
y.out_b = get_tensor("a.gen.nar.out.bias");
if (y.inp_w->ne[0] != mtmd_graph_yue2_gen::n_latent || y.out_w->ne[1] != mtmd_graph_yue2_gen::n_latent ||
y.inp_w->ne[1] != hparams.n_embd || y.out_w->ne[0] != hparams.n_embd) {
throw std::runtime_error(mtmd_format("%s: yue2: unexpected latent projection shape\n", __func__));
}
auto read_host = [&](const std::string & name, std::vector<float> & dst, int64_t ne0, int64_t ne1) {
const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name.c_str());
if (!t || t->type != GGML_TYPE_F32 || t->ne[0] != ne0 || t->ne[1] != ne1 || ggml_n_dims(t) > 2) {
throw std::runtime_error(mtmd_format("%s: invalid or missing tensor %s\n", __func__, name.c_str()));
}
dst.resize(ggml_nelements(t));
fin.seekg(tensor_offset.at(name), std::ios::beg);
if (!fin.read(reinterpret_cast<char *>(dst.data()), ggml_nbytes(t))) {
throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name.c_str()));
}
};
const int64_t n_embd = hparams.n_embd;
read_host("a.gen.nar.time.0.weight", y.time0_w, mtmd_graph_yue2_gen::n_freq, n_embd);
read_host("a.gen.nar.time.0.bias", y.time0_b, n_embd, 1);
read_host("a.gen.nar.time.2.weight", y.time2_w, n_embd, n_embd);
read_host("a.gen.nar.time.2.bias", y.time2_b, n_embd, 1);
// the tile cropping in mtmd_gen_wav_yue2() depends on the total stride
load_snake_vae();
const auto & vae = model.snake_vae;
if (vae.ratio() != mtmd_graph_yue2_gen::vae_ratio || vae.n_latent() != mtmd_graph_yue2_gen::n_latent ||
vae.out_w->ne[2] != mtmd_graph_yue2_gen::n_channels) {
throw std::runtime_error(mtmd_format("%s: yue2: unexpected VAE decoder layout\n", __func__));
}
} break;
case PROJECTOR_TYPE_VOXCPM2_GEN:
{
// generic a.gen.code.blk.* tensors are the LocDiT layers, LocEnc has the same hparams
using g = mtmd_graph_voxcpm2_gen;
auto & v = model.voxcpm2;
v.rope_factors = get_tensor("a.gen.rope_factors");
v.enc_layers.resize(hparams.n_layer);
for (int il = 0; il < hparams.n_layer; il++) {
auto & layer = v.enc_layers[il];
const char * p = "a.gen.enc";
layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, p, il, "weight"));
layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, p, il, "weight"));
layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, p, il, "weight"));
layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, p, il, "weight"));
layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight"));
layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, p, il, "weight"));
layer.ff_gate_w = get_tensor(mtmd_format(TN_FFN_GATE, p, il, "weight"));
layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, p, il, "weight"));
layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, p, il, "weight"));
}
v.enc_special = get_tensor("a.gen.enc.special_token");
v.enc_in_w = get_tensor("a.gen.enc.in_proj.weight");
v.enc_in_b = get_tensor("a.gen.enc.in_proj.bias");
v.enc_norm_w = get_tensor("a.gen.enc.post_ln.weight");
v.enc_to_lm_w = get_tensor("a.gen.enc_to_lm.weight");
v.enc_to_lm_b = get_tensor("a.gen.enc_to_lm.bias");
v.dit_in_w = get_tensor("a.gen.dit.in_proj.weight");
v.dit_in_b = get_tensor("a.gen.dit.in_proj.bias");
v.dit_cond_w = get_tensor("a.gen.dit.cond_proj.weight");
v.dit_cond_b = get_tensor("a.gen.dit.cond_proj.bias");
v.dit_out_w = get_tensor("a.gen.dit.out_proj.weight");
v.dit_out_b = get_tensor("a.gen.dit.out_proj.bias");
v.time0_w = get_tensor("a.gen.dit.time_mlp.0.weight");
v.time0_b = get_tensor("a.gen.dit.time_mlp.0.bias");
v.time1_w = get_tensor("a.gen.dit.time_mlp.1.weight");
v.time1_b = get_tensor("a.gen.dit.time_mlp.1.bias");
v.lm_to_dit_w = get_tensor("a.gen.lm_to_dit.weight");
v.lm_to_dit_b = get_tensor("a.gen.lm_to_dit.bias");
v.res_to_dit_w = get_tensor("a.gen.res_to_dit.weight");
v.res_to_dit_b = get_tensor("a.gen.res_to_dit.bias");
v.fsq_in_w = get_tensor("a.gen.fsq.in.weight");
v.fsq_in_b = get_tensor("a.gen.fsq.in.bias");
v.fsq_out_w = get_tensor("a.gen.fsq.out.weight");
v.fsq_out_b = get_tensor("a.gen.fsq.out.bias");
v.fusion_w = get_tensor("a.gen.fusion.weight");
v.fusion_b = get_tensor("a.gen.fusion.bias");
v.stop_proj_w = get_tensor("a.gen.stop_proj.weight");
v.stop_proj_b = get_tensor("a.gen.stop_proj.bias");
v.stop_head_w = get_tensor("a.gen.stop_head.weight");
const std::string p = "a.gen.vae";
v.dec_in_dw_w = get_tensor(p + ".conv_in_dw.weight");
v.dec_in_dw_b = get_tensor(p + ".conv_in_dw.bias");
v.dec_in_w = get_tensor(p + ".conv_in.weight");
v.dec_in_b = get_tensor(p + ".conv_in.bias");
int64_t ratio = 1;
for (int b = 0; gguf_find_tensor(ctx_gguf.get(), mtmd_format("%s.blk.%d.up.weight", p.c_str(), b).c_str()) >= 0; b++) {
const std::string q = p + ".blk." + std::to_string(b);
mtmd_code2wav::dac_block blk;
blk.snake_alpha = get_tensor(q + ".snake.alpha");
blk.snake_beta = get_tensor(q + ".snake.beta");
blk.conv_w = get_tensor(q + ".up.weight");
blk.conv_b = get_tensor(q + ".up.bias");
for (int r = 0; r < 3; r++) {
blk.res.push_back(load_dac_res(q + ".res." + std::to_string(r)));
}
ratio *= blk.conv_w->ne[0] / 2;
v.dec_sr_scale.push_back(get_tensor(q + ".sr_scale"));
v.dec_sr_bias.push_back(get_tensor(q + ".sr_bias"));
v.dec.push_back(std::move(blk));
}
v.dec_post_alpha = get_tensor(p + ".post_snake.alpha");
v.dec_post_beta = get_tensor(p + ".post_snake.beta");
v.dec_out_w = get_tensor(p + ".conv_out.weight");
v.dec_out_b = get_tensor(p + ".conv_out.bias");
// the graphs hard-code these shapes
if (v.rope_factors->ne[0] != g::head_dim / 2 || v.dit_in_w->ne[0] != g::n_latent ||
v.dec_in_dw_w->ne[2] != g::n_latent || ratio != g::dec_ratio || v.dec_out_w->ne[2] != 1 ||
model.layers.empty() || model.layers[0].q_w->ne[1] != (int64_t) g::head_dim * hparams.n_head) {
throw std::runtime_error(mtmd_format("%s: voxcpm2: unexpected tensor shapes\n", __func__));
}
} break;
case PROJECTOR_TYPE_VOXCPM2_SPKENC:
{
using g = mtmd_graph_voxcpm2_gen;
auto & v = model.voxcpm2;
const std::string p = "a.vae.enc";
v.enc_conv_in_w = get_tensor(p + ".conv_in.weight");
v.enc_conv_in_b = get_tensor(p + ".conv_in.bias");
int64_t hop = 1;
for (int b = 0; gguf_find_tensor(ctx_gguf.get(), mtmd_format("%s.blk.%d.down.weight", p.c_str(), b).c_str()) >= 0; b++) {
const std::string q = p + ".blk." + std::to_string(b);
mtmd_code2wav::dac_block blk;
for (int r = 0; r < 3; r++) {
blk.res.push_back(load_dac_res(q + ".res." + std::to_string(r)));
}
blk.snake_alpha = get_tensor(q + ".snake.alpha");
blk.snake_beta = get_tensor(q + ".snake.beta");
blk.conv_w = get_tensor(q + ".down.weight");
blk.conv_b = get_tensor(q + ".down.bias");
hop *= blk.conv_w->ne[0] / 2;
v.enc.push_back(std::move(blk));
}
v.enc_mu_w = get_tensor(p + ".fc_mu.weight");
v.enc_mu_b = get_tensor(p + ".fc_mu.bias");
if (hop * g::n_patch != hparams.audio_samples_per_token() || v.enc_mu_w->ne[2] != g::n_latent) {
throw std::runtime_error(mtmd_format("%s: voxcpm2: unexpected AudioVAE encoder layout\n", __func__));
}
} break;
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
{
// generic a.gen.code.* tensors (depth decoder layers, position_embd, post_ln) are loaded above
using g = mtmd_graph_music3_gen;
model.gen_code_proj_in_w = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "weight"));
model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight"));
model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight"));
model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
// the depth decoder sees the backbone hidden, then one row per code (all but the last)
const int64_t n_embd = hparams.n_embd;
if (!model.post_ln_w || !model.position_embeddings || n_embd != hparams.projection_dim ||
model.gen_code_proj_in_w->ne[0] != n_embd || model.gen_code_proj_in_w->ne[1] != n_embd ||
model.gen_code_embd_w->ne[0] != n_embd || model.gen_code_head_w->ne[0] != n_embd ||
model.gen_code_out_embd_w->ne[0] != n_embd || model.position_embeddings->ne[0] != n_embd ||
model.gen_code_embd_w->ne[2] != model.gen_code_head_w->ne[2] ||
model.gen_code_embd_w->ne[1] != model.gen_code_head_w->ne[1] ||
model.position_embeddings->ne[1] < model.gen_code_head_w->ne[2] + 1) {
throw std::runtime_error(mtmd_format("%s: minimax_music3: unexpected depth decoder layout\n", __func__));
}
auto & m = model.music3;
m.cond_layer_w = get_tensor("a.gen.cond.layer_w");
m.cond_w = get_tensor("a.gen.cond.proj.weight");
m.cond_b = get_tensor("a.gen.cond.proj.bias");
m.time_freq = get_tensor("a.gen.flow.time_freq");
m.time_in_w = get_tensor("a.gen.flow.time_in.weight");
m.time_in_b = get_tensor("a.gen.flow.time_in.bias");
m.time_out_w = get_tensor("a.gen.flow.time_out.weight");
m.time_out_b = get_tensor("a.gen.flow.time_out.bias");
m.pre_w = get_tensor("a.gen.flow.pre.weight");
m.proj_in_w = get_tensor("a.gen.flow.proj_in.weight");
m.proj_out_w = get_tensor("a.gen.flow.proj_out.weight");
m.post_w = get_tensor("a.gen.flow.post.weight");
for (int il = 0; gguf_find_tensor(ctx_gguf.get(), mtmd_format(TN_ATTN_Q, "a.gen.flow", il, "weight").c_str()) >= 0; il++) {
const char * q = "a.gen.flow";
mtmd_layer layer;
layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, q, il, "weight"));
layer.ln_1_b = get_tensor(mtmd_format(TN_LN_1, q, il, "bias"));
layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, q, il, "weight"));
layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, q, il, "weight"));
layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, q, il, "weight"));
layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, q, il, "weight"));
layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, q, il, "weight"));
layer.ln_2_b = get_tensor(mtmd_format(TN_LN_2, q, il, "bias"));
layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, q, il, "weight"));
layer.ff_up_b = get_tensor(mtmd_format(TN_FFN_UP, q, il, "bias"));
layer.ff_gate_w = get_tensor(mtmd_format(TN_FFN_GATE, q, il, "weight"));
layer.ff_gate_b = get_tensor(mtmd_format(TN_FFN_GATE, q, il, "bias"));
layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, q, il, "weight"));
layer.ff_down_b = get_tensor(mtmd_format(TN_FFN_DOWN, q, il, "bias"));
m.dit.push_back(layer);
}
load_snake_vae();
const auto & vae = model.snake_vae;
const int64_t n_dit_embd = m.proj_in_w->ne[1];
const int64_t n_cond = m.cond_w->ne[2];
if (m.dit.empty() || m.cond_layer_w->ne[0] != model.gen_code_head_w->ne[2] + 1 ||
m.cond_w->ne[0] != 3 || m.cond_w->ne[1] != hparams.n_embd ||
m.pre_w->ne[0] != 2 * g::n_latent + n_cond || m.pre_w->ne[1] != m.pre_w->ne[0] ||
m.proj_in_w->ne[0] != m.pre_w->ne[0] || m.proj_out_w->ne[0] != n_dit_embd || m.proj_out_w->ne[1] != g::n_latent ||
m.post_w->ne[0] != g::n_latent || m.post_w->ne[1] != g::n_latent || m.cond_b->ne[0] != n_cond ||
m.time_freq->ne[0] * 2 != m.time_in_w->ne[0] || m.time_in_w->ne[1] != m.time_out_w->ne[0] ||
m.time_in_b->ne[0] != m.time_in_w->ne[1] || m.time_out_w->ne[1] != n_dit_embd || m.time_out_b->ne[0] != n_dit_embd ||
n_dit_embd % g::dit_d_head != 0 ||
vae.ratio() != g::vae_ratio || vae.n_latent() * 2 != g::n_latent || vae.out_w->ne[2] != 1) {
throw std::runtime_error(mtmd_format("%s: minimax_music3: unexpected DiT or vocoder layout\n", __func__));
}
} break;
case PROJECTOR_TYPE_DRAMA3_GEN:
{
// generic a.gen.code.blk.* tensors are loaded by the loop above
model.drama3_layers.resize(hparams.n_layer);
for (int il = 0; il < hparams.n_layer; ++il) {
auto & tl = model.drama3_layers[il];
model.layers[il].attn_gate_w = get_tensor(mtmd_format(TN_ATTN_GATE, prefix, il, "weight"));
tl.enorm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_ENORM, il, "weight"));
tl.hnorm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HNORM, il, "weight"));
tl.eh_proj_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EH_PROJ, il, "weight"));
tl.embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_BLK_EMBD, il, "weight"));
tl.out_norm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_BLK_OUT_NORM, il, "weight"));
}
model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight")); // merged 3D, one table per codebook
model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight")); // merged 3D, codebooks 1..
model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); // backbone rows of the semantic band
load_code2wav(false);
} break;
case PROJECTOR_TYPE_FISH_S2_GEN:
{
// generic a.gen.code.blk.* tensors are loaded by the loop above
model.gen_code_tok_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_TOK_EMBD, "weight")); // fast AR input
model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight")); // fast AR head, residual codebooks
model.gen_code_norm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_NORM, "weight"));
model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight")); // merged 3D, backbone input tables
model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); // backbone rows of the semantic band
load_code2wav(false);
// the graph reads model.layers[0]
if (hparams.n_layer < 1) {
throw std::runtime_error(mtmd_format("%s: fish_s2_gen has no fast AR layer\n", __func__));
}
} break;
case PROJECTOR_TYPE_CHATTERBOX:
{
auto & c = model.cbx;
auto has = [&](const std::string & name) {
return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0;
};
auto load_enc_layer = [&](const std::string & p, mtmd_chatterbox::enc_layer & l) {
l.norm_mha_w = get_tensor(p + ".norm_mha.weight");
l.norm_mha_b = get_tensor(p + ".norm_mha.bias");
l.attn_q_w = get_tensor(p + ".self_attn.linear_q.weight");
l.attn_q_b = get_tensor(p + ".self_attn.linear_q.bias");
l.attn_k_w = get_tensor(p + ".self_attn.linear_k.weight");
l.attn_k_b = get_tensor(p + ".self_attn.linear_k.bias");
l.attn_v_w = get_tensor(p + ".self_attn.linear_v.weight");
l.attn_v_b = get_tensor(p + ".self_attn.linear_v.bias");
l.attn_pos_w = get_tensor(p + ".self_attn.linear_pos.weight");
l.attn_pos_bias_u = get_tensor(p + ".self_attn.pos_bias_u");
l.attn_pos_bias_v = get_tensor(p + ".self_attn.pos_bias_v");
l.attn_out_w = get_tensor(p + ".self_attn.linear_out.weight");
l.attn_out_b = get_tensor(p + ".self_attn.linear_out.bias");
l.norm_ff_w = get_tensor(p + ".norm_ff.weight");
l.norm_ff_b = get_tensor(p + ".norm_ff.bias");
l.ffn_1_w = get_tensor(p + ".feed_forward.w_1.weight");
l.ffn_1_b = get_tensor(p + ".feed_forward.w_1.bias");
l.ffn_2_w = get_tensor(p + ".feed_forward.w_2.weight");
l.ffn_2_b = get_tensor(p + ".feed_forward.w_2.bias");
};
auto load_causal = [&](const std::string & p, mtmd_chatterbox::causal_block & b) {
b.conv_w = get_tensor(p + ".block.0.weight");
b.conv_b = get_tensor(p + ".block.0.bias");
b.norm_w = get_tensor(p + ".block.2.weight");
b.norm_b = get_tensor(p + ".block.2.bias");
};
auto load_resnet = [&](const std::string & p, mtmd_chatterbox::resnet & r) {
load_causal(p + ".block1", r.block1);
load_causal(p + ".block2", r.block2);
r.mlp_w = get_tensor(p + ".mlp.1.weight");
r.mlp_b = get_tensor(p + ".mlp.1.bias");
r.res_conv_w = get_tensor(p + ".res_conv.weight");
r.res_conv_b = get_tensor(p + ".res_conv.bias");
};
auto load_tfm = [&](const std::string & p, mtmd_chatterbox::tfm_block & b) {
b.norm1_w = get_tensor(p + ".norm1.weight");
b.norm1_b = get_tensor(p + ".norm1.bias");
b.attn_q_w = get_tensor(p + ".attn1.to_q.weight");
b.attn_k_w = get_tensor(p + ".attn1.to_k.weight");
b.attn_v_w = get_tensor(p + ".attn1.to_v.weight");
b.attn_out_w = get_tensor(p + ".attn1.to_out.0.weight");
b.attn_out_b = get_tensor(p + ".attn1.to_out.0.bias");
b.norm3_w = get_tensor(p + ".norm3.weight");
b.norm3_b = get_tensor(p + ".norm3.bias");
b.ff_in_w = get_tensor(p + ".ff.net.0.proj.weight");
b.ff_in_b = get_tensor(p + ".ff.net.0.proj.bias");
b.ff_out_w = get_tensor(p + ".ff.net.2.weight");
b.ff_out_b = get_tensor(p + ".ff.net.2.bias");
};
auto load_stage = [&](const std::string & p, mtmd_chatterbox::est_stage & s, bool boundary) {
load_resnet(p + ".0", s.res);
for (int j = 0; has(p + ".1." + std::to_string(j) + ".norm1.weight"); j++) {
mtmd_chatterbox::tfm_block b;
load_tfm(p + ".1." + std::to_string(j), b);
s.tfm.push_back(b);
}
if (boundary) {
s.conv_w = get_tensor(p + ".2.weight");
s.conv_b = get_tensor(p + ".2.bias");
}
};
auto load_hres = [&](const std::string & p, mtmd_chatterbox::hift_res & r) {
for (int j = 0; has(p + ".convs1." + std::to_string(j) + ".weight"); j++) {
const std::string js = std::to_string(j);
mtmd_chatterbox::hift_res_unit u;
u.act1_alpha = get_tensor(p + ".activations1." + js + ".alpha");
u.act2_alpha = get_tensor(p + ".activations2." + js + ".alpha");
u.conv1_w = get_tensor(p + ".convs1." + js + ".weight");
u.conv1_b = get_tensor(p + ".convs1." + js + ".bias");
u.conv2_w = get_tensor(p + ".convs2." + js + ".weight");
u.conv2_b = get_tensor(p + ".convs2." + js + ".bias");
r.units.push_back(u);
}
};
// flow encoder
c.input_embedding_w = get_tensor("a.gen.flow.input_embedding.weight");
c.spk_affine_w = get_tensor("a.gen.flow.spk_embed_affine_layer.weight");
c.spk_affine_b = get_tensor("a.gen.flow.spk_embed_affine_layer.bias");
c.embed_linear_w = get_tensor("a.gen.fenc.embed.out.0.weight");
c.embed_linear_b = get_tensor("a.gen.fenc.embed.out.0.bias");
c.embed_norm_w = get_tensor("a.gen.fenc.embed.out.1.weight");
c.embed_norm_b = get_tensor("a.gen.fenc.embed.out.1.bias");
c.pre_conv1_w = get_tensor("a.gen.fenc.pre_lookahead_layer.conv1.weight");
c.pre_conv1_b = get_tensor("a.gen.fenc.pre_lookahead_layer.conv1.bias");
c.pre_conv2_w = get_tensor("a.gen.fenc.pre_lookahead_layer.conv2.weight");
c.pre_conv2_b = get_tensor("a.gen.fenc.pre_lookahead_layer.conv2.bias");
for (int i = 0; has("a.gen.fenc.encoders." + std::to_string(i) + ".norm_mha.weight"); i++) {
mtmd_chatterbox::enc_layer l;
load_enc_layer("a.gen.fenc.encoders." + std::to_string(i), l);
c.enc.push_back(l);
}
c.up_conv_w = get_tensor("a.gen.fenc.up_layer.conv.weight");
c.up_conv_b = get_tensor("a.gen.fenc.up_layer.conv.bias");
c.up_embed_linear_w = get_tensor("a.gen.fenc.up_embed.out.0.weight");
c.up_embed_linear_b = get_tensor("a.gen.fenc.up_embed.out.0.bias");
c.up_embed_norm_w = get_tensor("a.gen.fenc.up_embed.out.1.weight");
c.up_embed_norm_b = get_tensor("a.gen.fenc.up_embed.out.1.bias");
for (int i = 0; has("a.gen.fenc.up_encoders." + std::to_string(i) + ".norm_mha.weight"); i++) {
mtmd_chatterbox::enc_layer l;
load_enc_layer("a.gen.fenc.up_encoders." + std::to_string(i), l);
c.up_enc.push_back(l);
}
c.after_norm_w = get_tensor("a.gen.fenc.after_norm.weight");
c.after_norm_b = get_tensor("a.gen.fenc.after_norm.bias");
c.encoder_proj_w = get_tensor("a.gen.flow.encoder_proj.weight");
c.encoder_proj_b = get_tensor("a.gen.flow.encoder_proj.bias");
// cfm estimator
c.time_mlp_1_w = get_tensor("a.gen.est.time_mlp.linear_1.weight");
c.time_mlp_1_b = get_tensor("a.gen.est.time_mlp.linear_1.bias");
c.time_mlp_2_w = get_tensor("a.gen.est.time_mlp.linear_2.weight");
c.time_mlp_2_b = get_tensor("a.gen.est.time_mlp.linear_2.bias");
c.time_embed_mixer_w = get_tensor("a.gen.est.time_embed_mixer.weight", false);
load_stage("a.gen.est.down_blocks.0", c.est_down, true);
for (int i = 0; has("a.gen.est.mid_blocks." + std::to_string(i) + ".0.block1.block.0.weight"); i++) {
mtmd_chatterbox::est_stage s;
load_stage("a.gen.est.mid_blocks." + std::to_string(i), s, false);
c.est_mid.push_back(std::move(s));
}
load_stage("a.gen.est.up_blocks.0", c.est_up, true);
load_causal("a.gen.est.final_block", c.est_final_block);
c.est_final_proj_w = get_tensor("a.gen.est.final_proj.weight");
c.est_final_proj_b = get_tensor("a.gen.est.final_proj.bias");
// hift vocoder
c.hift_pre_w = get_tensor("a.gen.hift.conv_pre.weight");
c.hift_pre_b = get_tensor("a.gen.hift.conv_pre.bias");
c.hift_post_w = get_tensor("a.gen.hift.conv_post.weight");
c.hift_post_b = get_tensor("a.gen.hift.conv_post.bias");
for (int i = 0; has("a.gen.hift.ups." + std::to_string(i) + ".weight"); i++) {
const std::string is = std::to_string(i);
mtmd_chatterbox::hift_up up;
up.up_w = get_tensor("a.gen.hift.ups." + is + ".weight");
up.up_b = get_tensor("a.gen.hift.ups." + is + ".bias");
up.source_down_w = get_tensor("a.gen.hift.source_downs." + is + ".weight");
up.source_down_b = get_tensor("a.gen.hift.source_downs." + is + ".bias");
load_hres("a.gen.hift.source_resblocks." + is, up.source_res);
load_hres("a.gen.hift.resblocks." + std::to_string(3 * i), up.res_0);
load_hres("a.gen.hift.resblocks." + std::to_string(3 * i + 1), up.res_1);
load_hres("a.gen.hift.resblocks." + std::to_string(3 * i + 2), up.res_2);
c.hift_ups.push_back(std::move(up));
}
for (int i = 0; has("a.gen.hift.f0_predictor.condnet." + std::to_string(i) + ".weight"); i += 2) {
const std::string is = std::to_string(i);
mtmd_chatterbox::f0_conv fc;
fc.w = get_tensor("a.gen.hift.f0_predictor.condnet." + is + ".weight");
fc.b = get_tensor("a.gen.hift.f0_predictor.condnet." + is + ".bias");
c.f0_condnet.push_back(fc);
}
c.f0_classifier_w = get_tensor("a.gen.hift.f0_predictor.classifier.weight");
c.f0_classifier_b = get_tensor("a.gen.hift.f0_predictor.classifier.bias");
// s3 tokenizer
c.s3tok_conv1_w = get_tensor("a.s3tok.encoder.conv1.weight");
c.s3tok_conv1_b = get_tensor("a.s3tok.encoder.conv1.bias");
c.s3tok_conv2_w = get_tensor("a.s3tok.encoder.conv2.weight");
c.s3tok_conv2_b = get_tensor("a.s3tok.encoder.conv2.bias");
for (int i = 0; has("a.s3tok.encoder.blocks." + std::to_string(i) + ".attn_ln.weight"); i++) {
const std::string p = "a.s3tok.encoder.blocks." + std::to_string(i);
mtmd_chatterbox::s3tok_block b;
b.attn_ln_w = get_tensor(p + ".attn_ln.weight");
b.attn_ln_b = get_tensor(p + ".attn_ln.bias");
b.attn_q_w = get_tensor(p + ".attn.query.weight");
b.attn_q_b = get_tensor(p + ".attn.query.bias");
b.attn_k_w = get_tensor(p + ".attn.key.weight");
b.attn_v_w = get_tensor(p + ".attn.value.weight");
b.attn_v_b = get_tensor(p + ".attn.value.bias");
b.fsmn_w = get_tensor(p + ".attn.fsmn_block.weight");
b.attn_out_w = get_tensor(p + ".attn.out.weight");
b.attn_out_b = get_tensor(p + ".attn.out.bias");
b.mlp_ln_w = get_tensor(p + ".mlp_ln.weight");
b.mlp_ln_b = get_tensor(p + ".mlp_ln.bias");
b.mlp_in_w = get_tensor(p + ".mlp.0.weight");
b.mlp_in_b = get_tensor(p + ".mlp.0.bias");
b.mlp_out_w = get_tensor(p + ".mlp.2.weight");
b.mlp_out_b = get_tensor(p + ".mlp.2.bias");
c.s3tok_blocks.push_back(b);
}
c.s3tok_down_w = get_tensor("a.s3tok.quantizer._codebook.project_down.weight");
c.s3tok_down_b = get_tensor("a.s3tok.quantizer._codebook.project_down.bias");
// read on the host: embedding tables, source module
for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta.get()); t; t = ggml_get_next_tensor(ctx_meta.get(), t)) {
const std::string name = t->name;
if (name.rfind("a.gen.code.", 0) == 0 || name.rfind("a.gen.t3.", 0) == 0 ||
name.rfind("a.gen.hift.m_source.", 0) == 0) {
model.cbx_tensors[name] = get_tensor(name);
}
}
// voice.<name> holds the talker rows, voice.<name>.ref_* the flow decoder reference
const int64_t n_embd_talker = model.cbx_tensors.at("a.gen.code.out_embd.weight")->ne[0];
for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) {
const std::string name = gguf_get_tensor_name(ctx_gguf.get(), i);
if (name.rfind("voice.", 0) != 0) {
continue;
}
const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name.c_str());
std::string voice = name.substr(6);
std::vector<float> ids; // the converter stores 1D tensors as F32, ids are exact
void * dst = nullptr;
bool ok = false;
auto take = [&](const char * suffix) {
const size_t n = strlen(suffix);
if (voice.size() > n && voice.compare(voice.size() - n, n, suffix) == 0) {
voice.resize(voice.size() - n);
return true;
}
return false;
};
if (take(".ref_tokens")) {
ids.resize(ggml_nelements(t));
dst = ids.data();
ok = t->type == GGML_TYPE_F32;
} else if (take(".ref_feat")) {
auto & v = model.gen_voices[voice].ref.feat;
v.resize(ggml_nelements(t));
dst = v.data();
ok = t->type == GGML_TYPE_F32 && t->ne[0] == 80;
} else if (take(".ref_spk")) {
auto & v = model.gen_voices[voice].ref.spk;
v.resize(ggml_nelements(t));
dst = v.data();
ok = t->type == GGML_TYPE_F32 && t->ne[0] == 192;
} else {
auto & v = model.gen_voices[voice].rows;
v.resize(ggml_nelements(t));
dst = v.data();
ok = t->type == GGML_TYPE_F32 && ggml_n_dims(t) <= 2 && t->ne[0] == n_embd_talker;
}
if (!ok) {
throw std::runtime_error(mtmd_format("%s: invalid voice tensor %s\n", __func__, name.c_str()));
}
fin.seekg(tensor_offset.at(name), std::ios::beg);
if (!fin.read(reinterpret_cast<char *>(dst), ggml_nbytes(t))) {
throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name.c_str()));
}
if (!ids.empty()) {
model.gen_voices[voice].ref.tokens.assign(ids.begin(), ids.end());
}
}
for (const auto & [voice, v] : model.gen_voices) {
if (v.rows.empty() || v.ref.tokens.empty() || v.ref.feat.size() != 2 * 80 * v.ref.tokens.size() || v.ref.spk.empty()) {
throw std::runtime_error(mtmd_format("%s: incomplete voice %s\n", __func__, voice.c_str()));
}
}
} break;
case PROJECTOR_TYPE_ZONOS3_SPKENC:
{
// the graph reads the tensors by name
for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) {
const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);
if (strncmp(name, "a.spk.", 6) == 0) {
model.zonos3_spk[name] = get_tensor(name);
}
}
if (!model.zonos3_spk.count("a.spk.proj.weight") || !model.zonos3_spk.count("a.spk.linear.weight")) {
throw std::runtime_error(mtmd_format("%s: zonos3: missing speaker encoder tensors\n", __func__));
}
} break;
case PROJECTOR_TYPE_CHATTERBOX_SPKENC:
{
auto & c = model.cbx;
auto has = [&](const std::string & name) {
return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0;
};
auto load_bn = [&](const std::string & p, mtmd_chatterbox::bn & n) {
n.mean = get_tensor(p + ".running_mean");
n.var = get_tensor(p + ".running_var");
n.w = get_tensor(p + ".weight", false);
n.b = get_tensor(p + ".bias", false);
};
auto load_res2d = [&](const std::string & p, mtmd_chatterbox::spk_res2d & r) {
r.conv1_w = get_tensor(p + ".conv1.weight");
load_bn(p + ".bn1", r.bn1);
r.conv2_w = get_tensor(p + ".conv2.weight");
load_bn(p + ".bn2", r.bn2);
r.shortcut_w = get_tensor(p + ".shortcut.0.weight", false);
if (r.shortcut_w) {
load_bn(p + ".shortcut.1", r.shortcut_bn);
}
};
auto load_cam_block = [&](const std::string & bp, const std::string & tp, mtmd_chatterbox::spk_cam_block & blk) {
for (int li = 1; has(bp + ".tdnnd" + std::to_string(li) + ".linear1.weight"); li++) {
const std::string p = bp + ".tdnnd" + std::to_string(li);
mtmd_chatterbox::spk_cam_layer l;
load_bn(p + ".nonlinear1.batchnorm", l.nl1_bn);
l.linear1_w = get_tensor(p + ".linear1.weight");
load_bn(p + ".nonlinear2.batchnorm", l.nl2_bn);
l.local_w = get_tensor(p + ".cam_layer.linear_local.weight");
l.ctx1_w = get_tensor(p + ".cam_layer.linear1.weight");
l.ctx1_b = get_tensor(p + ".cam_layer.linear1.bias");
l.ctx2_w = get_tensor(p + ".cam_layer.linear2.weight");
l.ctx2_b = get_tensor(p + ".cam_layer.linear2.bias");
blk.layers.push_back(l);
}
load_bn(tp + ".nonlinear.batchnorm", blk.transit_bn);
blk.transit_w = get_tensor(tp + ".linear.weight");
};
c.spk_conv1_w = get_tensor("a.spk.head.conv1.weight");
load_bn("a.spk.head.bn1", c.spk_bn1);
load_res2d("a.spk.head.layer1.0", c.spk_layer1_0);
load_res2d("a.spk.head.layer1.1", c.spk_layer1_1);
load_res2d("a.spk.head.layer2.0", c.spk_layer2_0);
load_res2d("a.spk.head.layer2.1", c.spk_layer2_1);
c.spk_conv2_w = get_tensor("a.spk.head.conv2.weight");
load_bn("a.spk.head.bn2", c.spk_bn2);
c.spk_tdnn_w = get_tensor("a.spk.xvector.tdnn.linear.weight");
load_bn("a.spk.xvector.tdnn.nonlinear.batchnorm", c.spk_tdnn_bn);
load_cam_block("a.spk.xvector.block1", "a.spk.xvector.transit1", c.spk_block1);
load_cam_block("a.spk.xvector.block2", "a.spk.xvector.transit2", c.spk_block2);
load_cam_block("a.spk.xvector.block3", "a.spk.xvector.transit3", c.spk_block3);
load_bn("a.spk.xvector.out_nonlinear.batchnorm", c.spk_out_bn);
c.spk_dense_w = get_tensor("a.spk.xvector.dense.linear.weight");
load_bn("a.spk.xvector.dense.nonlinear.batchnorm", c.spk_dense_bn);
// read on the host: voice encoder lstm, conditioning encoder
for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta.get()); t; t = ggml_get_next_tensor(ctx_meta.get(), t)) {
const std::string name = t->name;
if (name.rfind("a.ve.", 0) == 0 || name.rfind("a.cenc.", 0) == 0) {
model.cbx_tensors[name] = get_tensor(name);
}
}
} break;
case PROJECTOR_TYPE_KOKORO:
{
// the graphs and the host code look the tensors up by name, see models/kokoro.cpp
// a voice is a table of style rows, the audio pipeline reads the row it needs
for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta.get()); t; t = ggml_get_next_tensor(ctx_meta.get(), t)) {
const std::string name = t->name;
if (name.rfind("a.gen.", 0) == 0) {
model.cbx_tensors[name] = get_tensor(name);
} else if (name.rfind("voice.", 0) == 0) {
model.cbx_tensors[name] = get_tensor(name);
model.gen_voices[name.substr(6)] = {};
}
}
LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size());
} break;
case PROJECTOR_TYPE_TINY_VOCA:
{
using g = mtmd_graph_tiny_voca;
auto & tv = model.tiny_voca;
tv.dur_conv1_w = get_tensor("a.gen.dur.conv1.weight");
tv.dur_conv1_b = get_tensor("a.gen.dur.conv1.bias");
tv.dur_norm1_w = get_tensor("a.gen.dur.norm1.weight");
tv.dur_norm1_b = get_tensor("a.gen.dur.norm1.bias");
tv.dur_conv2_w = get_tensor("a.gen.dur.conv2.weight");
tv.dur_conv2_b = get_tensor("a.gen.dur.conv2.bias");
tv.dur_norm2_w = get_tensor("a.gen.dur.norm2.weight");
tv.dur_norm2_b = get_tensor("a.gen.dur.norm2.bias");
tv.dur_out_w = get_tensor("a.gen.dur.out.weight");
tv.dur_out_b = get_tensor("a.gen.dur.out.bias");
tv.harm_w = get_tensor("a.gen.voc.harmonic.weight");
tv.harm_b = get_tensor("a.gen.voc.harmonic.bias");
tv.noise_w = get_tensor("a.gen.voc.noise.weight");
tv.noise_b = get_tensor("a.gen.voc.noise.bias");
if (tv.harm_w->ne[0] != hparams.n_embd || tv.harm_w->ne[1] != g::n_cep_causal + g::n_cep_anti + g::harm_steps ||
tv.noise_w->ne[0] != hparams.n_embd || tv.noise_w->ne[1] != g::n_noise_cep + g::noise_steps ||
tv.dur_out_w->ne[1] != 2) {
throw std::runtime_error(mtmd_format("%s: tiny_voca: unexpected head shape\n", __func__));
}
auto read_host = [&](const std::string & name, std::vector<float> & dst, int64_t ne0, int64_t ne1) {
const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name.c_str());
if (!t || t->type != GGML_TYPE_F32 || t->ne[0] != ne0 || t->ne[1] != ne1 || ggml_n_dims(t) > 2) {
throw std::runtime_error(mtmd_format("%s: invalid or missing tensor %s\n", __func__, name.c_str()));
}
dst.resize(ggml_nelements(t));
fin.seekg(tensor_offset.at(name), std::ios::beg);
if (!fin.read(reinterpret_cast<char *>(dst.data()), ggml_nbytes(t))) {
throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name.c_str()));
}
};
const int64_t n_embd = hparams.n_embd;
const ggml_tensor * tok = ggml_get_tensor(ctx_meta.get(), "a.gen.tok_embd.weight");
if (!tok) {
throw std::runtime_error(mtmd_format("%s: tiny_voca: missing token table\n", __func__));
}
read_host("a.gen.token_class", tv.token_class, tok->ne[1], 1);
read_host("a.gen.frame_pos.weight", tv.frame_pos_w, 3, n_embd);
read_host("a.gen.frame_pos.bias", tv.frame_pos_b, n_embd, 1);
read_host("a.gen.prosody_head.weight", tv.prosody_head_w, n_embd, 3);
read_host("a.gen.prosody_head.bias", tv.prosody_head_b, 3, 1);
read_host("a.gen.prosody_proj.weight", tv.prosody_proj_w, 3, n_embd);
read_host("a.gen.prosody_proj.bias", tv.prosody_proj_b, n_embd, 1);
// unknown-word model, optional
if (const ggml_tensor * e = ggml_get_tensor(ctx_meta.get(), "a.gen.g2p.embed.weight")) {
const ggml_tensor * hh = ggml_get_tensor(ctx_meta.get(), "a.gen.g2p.gru.weight_hh_l0");
if (!hh || e->ne[1] != g::n_g2p_letter + 1) {
throw std::runtime_error(mtmd_format("%s: tiny_voca: unexpected g2p model\n", __func__));
}
const int64_t w = e->ne[0];
const int64_t h = hh->ne[0];
tv.g2p_n_width = w;
tv.g2p_n_hidden = h;
read_host("a.gen.g2p.embed.weight", tv.g2p_embd, w, g::n_g2p_letter + 1);
read_host("a.gen.g2p.slot", tv.g2p_slot, w, g::n_g2p_slot);
for (int il = 0; ggml_get_tensor(ctx_meta.get(), ("a.gen.g2p.gru.weight_ih_l" + std::to_string(il)).c_str()); il++) {
for (const char * dir : { "", "_reverse" }) {
const std::string sfx = "_l" + std::to_string(il) + dir;
mtmd_tiny_voca::g2p_gru gru;
read_host("a.gen.g2p.gru.weight_ih" + sfx, gru.w_ih, il == 0 ? w : 2 * h, 3 * h);
read_host("a.gen.g2p.gru.weight_hh" + sfx, gru.w_hh, h, 3 * h);
read_host("a.gen.g2p.gru.bias_ih" + sfx, gru.b_ih, 3 * h, 1);
read_host("a.gen.g2p.gru.bias_hh" + sfx, gru.b_hh, 3 * h, 1);
tv.g2p_layers.push_back(std::move(gru));
}
}
read_host("a.gen.g2p.out.weight", tv.g2p_out_w, 2 * h, g::n_g2p_symbol + 1);
read_host("a.gen.g2p.out.bias", tv.g2p_out_b, g::n_g2p_symbol + 1, 1);
}
// read by the audio pipeline: input tables and voices (encoder row, then frame row)
for (const char * name : { "a.gen.tok_embd.weight", "a.gen.stress_embd.weight" }) {
model.cbx_tensors[name] = get_tensor(name);
}
for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta.get()); t; t = ggml_get_next_tensor(ctx_meta.get(), t)) {
const std::string name = t->name;
if (name.rfind("voice.", 0) == 0) {
if (t->ne[0] != 2 * n_embd) {
throw std::runtime_error(mtmd_format("%s: tiny_voca: unexpected voice shape\n", __func__));
}
model.cbx_tensors[name] = get_tensor(name);
model.gen_voices[name.substr(6)] = {};
}
}
LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size());
} break;
case PROJECTOR_TYPE_SOPRANO:
{
auto & v = model.vocos;
v.input_w = get_tensor(mtmd_format(TN_A_GEN_WAV_INPUT, "weight"));
v.input_b = get_tensor(mtmd_format(TN_A_GEN_WAV_INPUT, "bias"));
v.norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_NORM, "weight"));
v.norm_b = get_tensor(mtmd_format(TN_A_GEN_WAV_NORM, "bias"));
v.output_norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_OUTPUT_NORM, "weight"));
v.output_norm_b = get_tensor(mtmd_format(TN_A_GEN_WAV_OUTPUT_NORM, "bias"));
v.output_w = get_tensor(mtmd_format(TN_A_GEN_WAV_OUTPUT, "weight"));
v.output_b = get_tensor(mtmd_format(TN_A_GEN_WAV_OUTPUT, "bias"));
v.blocks.resize(hparams.n_layer);
for (int il = 0; il < hparams.n_layer; ++il) {
auto & b = v.blocks[il];
b.dwconv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight"));
b.dwconv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias"));
b.norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_NORM, il, "weight"));
b.norm_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_NORM, il, "bias"));
b.pw1_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW1, il, "weight"));
b.pw1_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW1, il, "bias"));
b.pw2_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW2, il, "weight"));
b.pw2_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW2, il, "bias"));
b.gamma = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_GAMMA, il));
}
} break;
case PROJECTOR_TYPE_VOXTRAL:
case PROJECTOR_TYPE_VOXTRAL_RT:
{
model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight"));
model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias"));
model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight"));
model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
} break;
case PROJECTOR_TYPE_MUSIC_FLAMINGO:
{
model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight"));
model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias"));
model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight"));
model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias"));
} break;
case PROJECTOR_TYPE_INTERNVL:
{
model.mm_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "bias"));
model.mm_3_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "weight"));
model.mm_3_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "bias"));
} break;
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
case PROJECTOR_TYPE_NEMOTRON_OMNI:
{
model.mm_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "weight"));
model.mm_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "weight"));
model.mm_3_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "weight"));
} break;
case PROJECTOR_TYPE_GLMA:
{
model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight"));
model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias"));
model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight"));
model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias"));
model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_NORM_PRE, "weight"));
model.mm_norm_pre_b = get_tensor(mtmd_format(TN_MM_NORM_PRE, "bias"));
model.mm_boi = get_tensor(mtmd_format(TN_TOK_BOI));
model.mm_eoi = get_tensor(mtmd_format(TN_TOK_EOI));
} break;
case PROJECTOR_TYPE_LLAMA4:
{
model.mm_model_proj = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight"));
model.mm_model_mlp_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "weight"));
model.mm_model_mlp_2_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 2, "weight"));
} break;
case PROJECTOR_TYPE_COGVLM:
{
model.mm_model_proj = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight"));
model.mm_post_fc_norm_w = get_tensor(mtmd_format(TN_MM_POST_FC_NORM, "weight"));
model.mm_post_fc_norm_b = get_tensor(mtmd_format(TN_MM_POST_FC_NORM, "bias"));
model.mm_h_to_4h_w = get_tensor(mtmd_format(TN_MM_H_TO_4H, "weight"));
model.mm_gate_w = get_tensor(mtmd_format(TN_MM_GATE, "weight"));
model.mm_4h_to_h_w = get_tensor(mtmd_format(TN_MM_4H_TO_H, "weight"));
model.mm_boi = get_tensor(TN_TOK_BOI);
model.mm_eoi = get_tensor(TN_TOK_EOI);
} break;
case PROJECTOR_TYPE_HUNYUANVL:
{
// proj.0 -> mm.0 (conv1), proj.2 -> mm.2 (conv2), mlp -> mm.model.fc (linear)
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
model.mm_model_proj = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight"));
model.mm_model_proj_b = get_tensor(mtmd_format(TN_MM_PROJECTOR, "bias"));
model.mm_pre_norm_w = get_tensor(mtmd_format(TN_MM_PRE_NORM, "weight"));
model.mm_post_norm_w = get_tensor(mtmd_format(TN_MM_POST_NORM, "weight"));
model.mm_img_begin = get_tensor(TN_MM_IMG_BEGIN);
model.mm_img_end = get_tensor(TN_MM_IMG_END);
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR, false);
// the position embedding is resampled on the host as F32
if (!model.position_embeddings || model.position_embeddings->type != GGML_TYPE_F32) {
throw std::runtime_error(mtmd_format("%s: HunyuanVL requires an F32 %s\n", __func__, mtmd_format(TN_POS_EMBD, prefix).c_str()));
}
} break;
case PROJECTOR_TYPE_JANUS_PRO:
{
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"));
} break;
case PROJECTOR_TYPE_PHI4:
{
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
{
model.pos_embed = get_tensor(mtmd_format(TN_SAM_POS_EMBD, "weight"));
model.patch_embed_proj_w = get_tensor(mtmd_format(TN_SAM_PATCH_EMBD, "weight"));
model.patch_embed_proj_b = get_tensor(mtmd_format(TN_SAM_PATCH_EMBD, "bias"));
model.n_sam_layers = hparams.sam_n_layer;
model.sam_layers.resize(model.n_sam_layers);
for (int il = 0; il < model.n_sam_layers; ++il) {
auto & layer = model.sam_layers[il];
layer.qkv_w = get_tensor(mtmd_format(TN_SAM_ATTN_QKV, il, "weight"));
layer.qkv_b = get_tensor(mtmd_format(TN_SAM_ATTN_QKV, il, "bias"));
layer.o_w = get_tensor(mtmd_format(TN_SAM_ATTN_OUT, il, "weight"));
layer.o_b = get_tensor(mtmd_format(TN_SAM_ATTN_OUT, il, "bias"));
layer.ln_1_w = get_tensor(mtmd_format(TN_SAM_PRE_NORM, il, "weight"));
layer.ln_1_b = get_tensor(mtmd_format(TN_SAM_PRE_NORM, il, "bias"));
layer.ln_2_w = get_tensor(mtmd_format(TN_SAM_POST_NORM, il, "weight"));
layer.ln_2_b = get_tensor(mtmd_format(TN_SAM_POST_NORM, il, "bias"));
layer.rel_pos_h = get_tensor(mtmd_format(TN_SAM_ATTN_POS_H, il, "weight"));
layer.rel_pos_w = get_tensor(mtmd_format(TN_SAM_ATTN_POS_W, il, "weight"));
layer.ff_up_w = get_tensor(mtmd_format(TN_SAM_FFN_UP, il, "weight"));
layer.ff_up_b = get_tensor(mtmd_format(TN_SAM_FFN_UP, il, "bias"));
layer.ff_down_w = get_tensor(mtmd_format(TN_SAM_FFN_DOWN, il, "weight"));
layer.ff_down_b = get_tensor(mtmd_format(TN_SAM_FFN_DOWN, il, "bias"));
}
model.neck_0_w = get_tensor(mtmd_format(TN_SAM_NECK, 0, "weight"));
model.neck_1_b = get_tensor(mtmd_format(TN_SAM_NECK, 1, "bias"));
model.neck_1_w = get_tensor(mtmd_format(TN_SAM_NECK, 1, "weight"));
model.neck_2_w = get_tensor(mtmd_format(TN_SAM_NECK, 2, "weight"));
model.neck_3_b = get_tensor(mtmd_format(TN_SAM_NECK, 3, "bias"));
model.neck_3_w = get_tensor(mtmd_format(TN_SAM_NECK, 3, "weight"));
model.net_2 = get_tensor(mtmd_format(TN_SAM_NET, 2, "weight"));
model.net_3 = get_tensor(mtmd_format(TN_SAM_NET, 3, "weight"));
model.image_newline = get_tensor(TN_IMAGE_NEWLINE, false);
model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR);
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight"));
model.mm_fc_b = get_tensor(mtmd_format(TN_MM_PROJECTOR, "bias"));
model.resample_query_768 = get_tensor(mtmd_format(TN_RESMPL_QUERY, 768, "weight"), false);
model.resample_query_1024 = get_tensor(mtmd_format(TN_RESMPL_QUERY, 1024, "weight"), false);
} break;
case PROJECTOR_TYPE_GEMMA4A:
{
for (int i = 0; i < 2; i++) {
model.sscp_conv_w[i] = get_tensor(mtmd_format(TN_A_CONV1D, i, "weight"));
model.sscp_conv_b[i] = get_tensor(mtmd_format(TN_A_CONV1D, i, "bias"), false);
model.sscp_norm_w[i] = get_tensor(mtmd_format(TN_A_CONV1D_NORM, i, "weight"), false);
}
model.sscp_inp_proj_w = get_tensor(mtmd_format(TN_A_INP_PROJ, "weight"));
model.sscp_inp_proj_b = get_tensor(mtmd_format(TN_A_INP_PROJ, "bias"), false);
model.audio_out_proj_w = get_tensor(mtmd_format(TN_A_OUT_PROJ, "weight"), false);
model.audio_out_proj_b = get_tensor(mtmd_format(TN_A_OUT_PROJ, "bias"), false);
// audio multimodal embedder (mm.a.* namespace, not mm.*)
model.mm_soft_emb_norm_w = get_tensor(mtmd_format(TN_A_MM_SOFT_EMB_N, "weight"), false);
model.mm_input_proj_w = get_tensor(mtmd_format(TN_A_MM_INP_PROJ, "weight"), false);
// Per-layer tensors NOT loaded by the generic loop above
for (int il = 0; il < hparams.n_layer; ++il) {
auto & layer = model.layers[il];
// Gemma4 audio conformer-specific tensors
layer.ff_norm_w = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "weight"));
layer.attn_pre_norm_w = get_tensor(mtmd_format(TN_A_ATTN_PRE_NORM, prefix, il, "weight"), false);
layer.per_dim_scale_w = get_tensor(mtmd_format(TN_A_PER_DIM_SCALE, prefix, il, "weight"), false);
layer.per_dim_k_scale_w = get_tensor(mtmd_format(TN_A_PER_DIM_K_SCALE, prefix, il, "weight"), false);
layer.attn_k_rel_w = get_tensor(mtmd_format(TN_A_ATTN_K_REL, prefix, il, "weight"), false);
// Convolution module
// Note: conv_norm / norm_conv are swapped in GGUF due to
// upstream tensor_mapping.py, so we load them in reverse order
layer.norm_conv_w = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "weight"), false);
layer.norm_conv_b = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "bias"), false);
layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "weight"));
layer.conv_pw1_b = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "bias"), false);
layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "weight"));
layer.conv_dw_b = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "bias"), false);
layer.conv_norm_w = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "weight"), false);
layer.conv_norm_b = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "bias"), false);
layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "weight"));
layer.conv_pw2_b = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "bias"), false);
// FFN2 (second half-step)
layer.ff_norm_1_w = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "weight"));
layer.ff_up_1_w = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "weight"));
layer.ff_up_1_b = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "bias"), false);
layer.ff_down_1_w = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "weight"));
layer.ff_down_1_b = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "bias"), false);
layer.ff_post_norm_1_w = get_tensor(mtmd_format(TN_A_FFN_POST_NORM_1, prefix, il, "weight"), false);
}
// Load clamp info for ClippableLinear AFTER all tensors are loaded
for (auto * tensor : tensors_to_load) {
std::string name = tensor->name;
if (string_ends_with(name, ".weight")) {
std::string name_inp_max = name;
std::string name_inp_min = name;
std::string name_out_max = name;
std::string name_out_min = name;
string_replace_all(name_inp_max, ".weight", ".input_max");
string_replace_all(name_inp_min, ".weight", ".input_min");
string_replace_all(name_out_max, ".weight", ".output_max");
string_replace_all(name_out_min, ".weight", ".output_min");
model.clamp_info_map[name] = {
get_scalar(name_inp_max, FLT_MAX),
get_scalar(name_inp_min, -FLT_MAX),
get_scalar(name_out_max, FLT_MAX),
get_scalar(name_out_min, -FLT_MAX)
};
}
}
} break;
case PROJECTOR_TYPE_GEMMA4UA:
{
model.mm_input_proj_w = get_tensor(mtmd_format(TN_A_MM_INP_PROJ, "weight"));
} break;
case PROJECTOR_TYPE_LFM2A:
case PROJECTOR_TYPE_D1OMNI_A:
{
for (int i : {0, 2, 3, 5, 6}) {
model.pre_encode_conv_X_w[i] = get_tensor(mtmd_format(TN_CONV1D, i, "weight"));
model.pre_encode_conv_X_b[i] = get_tensor(mtmd_format(TN_CONV1D, i, "bias"));
}
model.pre_encode_out_w = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "weight"));
model.pre_encode_out_b = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "bias"));
model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight"));
model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias"));
// residual block after the projector: norm, down, up
if (model.proj_type == PROJECTOR_TYPE_D1OMNI_A) {
model.mm_4_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 4, "weight"));
model.mm_4_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 4, "bias"));
model.mm_5_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 5, "weight"));
model.mm_5_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 5, "bias"));
model.mm_6_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 6, "weight"));
model.mm_6_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 6, "bias"));
}
for (int il = 0; il < hparams.n_layer; ++il) {
auto & layer = model.layers[il];
layer.ff_norm_w = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "weight"));
layer.ff_norm_b = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "bias"));
layer.ff_norm_1_w = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "weight"));
layer.ff_norm_1_b = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "bias"));
layer.ff_up_1_w = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "weight"));
layer.ff_up_1_b = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "bias"));
layer.ff_down_1_w = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "weight"));
layer.ff_down_1_b = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "bias"));
layer.pos_bias_u = get_tensor(mtmd_format(TN_POS_BIAS_U, prefix, il));
layer.pos_bias_v = get_tensor(mtmd_format(TN_POS_BIAS_V, prefix, il));
layer.norm_conv_w = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "weight"));
layer.norm_conv_b = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "bias"));
layer.linear_pos_w = get_tensor(mtmd_format(TN_LINEAR_POS, prefix, il, "weight"));
layer.conv_norm_w = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "weight"));
layer.conv_norm_b = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "bias"));
layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "weight"));
layer.conv_dw_b = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "bias"));
layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "weight"));
layer.conv_pw1_b = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "bias"));
layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "weight"));
layer.conv_pw2_b = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "bias"));
}
} break;
case PROJECTOR_TYPE_LFM3A:
{
for (int i : {0, 2, 3, 5, 6}) {
model.pre_encode_conv_X_w[i] = get_tensor(mtmd_format(TN_CONV1D, i, "weight"));
model.pre_encode_conv_X_b[i] = get_tensor(mtmd_format(TN_CONV1D, i, "bias"));
}
model.pre_encode_out_w = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "weight"));
model.pre_encode_out_b = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "bias"));
model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight"));
model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias"));
if (!model.post_ln_w) {
throw std::runtime_error(mtmd_format("%s: missing tensor %s\n", __func__, mtmd_format(TN_LN_POST, prefix, "weight").c_str()));
}
// each layer is either a short conv block or a complete attention block
for (int il = 0; il < hparams.n_layer; ++il) {
auto & layer = model.layers[il];
layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "weight"), false);
layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "weight"), false);
layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "weight"), false);
const bool is_conv = layer.conv_pw1_w && layer.conv_dw_w && layer.conv_pw2_w;
const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w && layer.q_norm && layer.k_norm;
if (is_conv == is_attn) {
throw std::runtime_error(mtmd_format("%s: lfm3a layer %d is neither a complete conv nor a complete attention block\n", __func__, il));
}
}
} break;
case PROJECTOR_TYPE_PARAKEET:
{
hparams.mel_filters = get_vector(TN_MEL_FILTERS);
hparams.window = get_vector(TN_WINDOW);
// Subsampling layers (conv1d)
for (int i : {0, 2, 3, 5, 6}) {
model.pre_encode_conv_X_w[i] = get_tensor(mtmd_format(TN_CONV1D, i, "weight"));
model.pre_encode_conv_X_b[i] = get_tensor(mtmd_format(TN_CONV1D, i, "bias"));
}
model.pre_encode_out_w = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "weight"));
model.pre_encode_out_b = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "bias"));
// Projection layers
model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_NORM_PRE, "weight"), false);
model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"), false);
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"), false);
// Encoder layers
for (int il = 0; il < hparams.n_layer; ++il) {
auto & layer = model.layers[il];
// Attention (from shared above)
// Relative position encoding
layer.linear_pos_w = get_tensor(mtmd_format(TN_LINEAR_POS, prefix, il, "weight"));
layer.pos_bias_u = get_tensor(mtmd_format(TN_POS_BIAS_U, prefix, il));
layer.pos_bias_v = get_tensor(mtmd_format(TN_POS_BIAS_V, prefix, il));
// Convolution module
layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "weight"));
layer.conv_pw1_b = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "bias"), false);
layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "weight"));
layer.conv_dw_b = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "bias"), false);
layer.conv_norm_w = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "weight"));
layer.conv_norm_b = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "bias"));
layer.conv_norm_mean = get_tensor(mtmd_format(TN_CONV_NORM_MEAN, prefix, il));
layer.conv_norm_var = get_tensor(mtmd_format(TN_CONV_NORM_VAR, prefix, il));
layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "weight"));
layer.conv_pw2_b = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "bias"), false);
// Feed-forward networks
layer.ff_norm_w = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "weight"));
layer.ff_norm_b = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "bias"));
layer.ff_norm_1_w = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "weight"));
layer.ff_norm_1_b = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "bias"));
layer.ff_up_1_w = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "weight"));
layer.ff_up_1_b = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "bias"), false);
layer.ff_down_1_w = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "weight"));
layer.ff_down_1_b = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "bias"), false);
// Layer norms
layer.norm_conv_w = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "weight"));
layer.norm_conv_b = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "bias"));
}
model.mm_model_mlp_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "weight"));
model.mm_model_mlp_2_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "weight"));
model.mm_model_mlp_3_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "weight"));
} break;
case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
{
hparams.mel_filters = get_vector(TN_MEL_FILTERS);
hparams.window = get_vector(TN_WINDOW);
// ASR expert: blocks, pre_ln and post_ln come from the generic loader
model.inp_proj_w = get_tensor(mtmd_format(TN_INP_PROJ, "weight"));
for (int il = 0; il < hparams.n_layer; ++il) {
auto & layer = model.layers[il];
layer.q_norm_b = get_tensor(mtmd_format(TN_ATTN_Q_NORM, prefix, il, "bias"));
layer.k_norm_b = get_tensor(mtmd_format(TN_ATTN_K_NORM, prefix, il, "bias"));
}
// speaker expert
const char * sp = "a.spk";
model.spk_inp_proj_w = get_tensor(mtmd_format(TN_A_SPK_INP_PROJ, "weight"));
model.spk_pre_ln_w = get_tensor(mtmd_format(TN_LN_PRE, sp, "weight"));
model.spk_pre_ln_b = get_tensor(mtmd_format(TN_LN_PRE, sp, "bias"));
model.spk_post_ln_w = get_tensor(mtmd_format(TN_LN_POST, sp, "weight"));
model.spk_post_ln_b = get_tensor(mtmd_format(TN_LN_POST, sp, "bias"));
model.spk_layers.resize(hparams.spk_n_layer);
for (int il = 0; il < hparams.spk_n_layer; ++il) {
auto & layer = model.spk_layers[il];
layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, sp, il, "weight"));
layer.ln_1_b = get_tensor(mtmd_format(TN_LN_1, sp, il, "bias"));
layer.qkv_w = get_tensor(mtmd_format(TN_ATTN_QKV, sp, il, "weight"));
layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, sp, il, "weight"));
layer.o_b = get_tensor(mtmd_format(TN_ATTN_OUTPUT, sp, il, "bias"));
layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, sp, il, "weight"));
layer.ln_2_b = get_tensor(mtmd_format(TN_LN_2, sp, il, "bias"));
layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, sp, il, "weight"));
layer.ff_up_b = get_tensor(mtmd_format(TN_FFN_UP, sp, il, "bias"));
layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, sp, il, "weight"));
layer.ff_down_b = get_tensor(mtmd_format(TN_FFN_DOWN, sp, il, "bias"));
}
// Sortformer head, fusion
model.spk_head_proj_w = get_tensor(mtmd_format(TN_A_SPK_HEAD, "proj", "weight"));
model.spk_head_proj_b = get_tensor(mtmd_format(TN_A_SPK_HEAD, "proj", "bias"));
model.spk_head_up_w = get_tensor(mtmd_format(TN_A_SPK_HEAD, "upsample", "weight"));
model.spk_head_up_b = get_tensor(mtmd_format(TN_A_SPK_HEAD, "upsample", "bias"));
model.spk_head_fc1_w = get_tensor(mtmd_format(TN_A_SPK_HEAD, "fc1", "weight"));
model.spk_head_fc1_b = get_tensor(mtmd_format(TN_A_SPK_HEAD, "fc1", "bias"));
model.spk_head_fc2_w = get_tensor(mtmd_format(TN_A_SPK_HEAD, "fc2", "weight"));
model.spk_head_fc2_b = get_tensor(mtmd_format(TN_A_SPK_HEAD, "fc2", "bias"));
model.fusion_asr_norm_w = get_tensor(mtmd_format(TN_A_FUSION_NORM, "asr_norm", "weight"));
model.fusion_asr_norm_b = get_tensor(mtmd_format(TN_A_FUSION_NORM, "asr_norm", "bias"));
model.fusion_spk_norm_w = get_tensor(mtmd_format(TN_A_FUSION_NORM, "spk_norm", "weight"));
model.fusion_spk_norm_b = get_tensor(mtmd_format(TN_A_FUSION_NORM, "spk_norm", "bias"));
model.fusion_spk_kernel = get_tensor(TN_A_FUSION_KERNEL);
model.spk_sil_emb = get_vector(TN_A_SPK_SIL_EMB);
// the head layouts and the widths the graph and the speaker state use must match the tensors
hparams.spk_n_embd = model.spk_inp_proj_w->ne[1];
const int64_t d_head = hparams.n_head > 0 ? hparams.n_embd / hparams.n_head : 0;
const int64_t d_head_spk = hparams.spk_n_head > 0 ? hparams.spk_n_embd / hparams.spk_n_head : 0;
const int64_t n_spk = model.spk_head_fc2_w->ne[1];
// the graph splits each fused QKV into three views of the layer width
bool qkv_ok = true;
for (const auto & layer : model.layers) {
qkv_ok = qkv_ok && layer.qkv_w && layer.qkv_w->ne[0] == hparams.n_embd && layer.qkv_w->ne[1] == 3 * hparams.n_embd;
}
for (const auto & layer : model.spk_layers) {
qkv_ok = qkv_ok && layer.qkv_w->ne[0] == hparams.spk_n_embd && layer.qkv_w->ne[1] == 3 * hparams.spk_n_embd;
}
if (!qkv_ok || model.inp_proj_w->ne[1] != hparams.n_embd || d_head <= 0 || d_head * hparams.n_head != hparams.n_embd ||
(hparams.n_embd_head > 0 && hparams.n_embd_head != d_head) ||
hparams.audio_rope_dims <= 0 || hparams.audio_rope_dims % 2 != 0 || hparams.audio_rope_dims > d_head ||
d_head_spk <= 0 || d_head_spk % 2 != 0 || d_head_spk * hparams.spk_n_head != hparams.spk_n_embd ||
(int64_t) model.spk_sil_emb.size() != hparams.spk_n_embd || n_spk <= 0 ||
model.fusion_spk_kernel->ne[0] != n_spk || model.fusion_spk_kernel->ne[1] != hparams.n_embd ||
hparams.mel_filters.size() != (size_t) hparams.n_mel_bins * (hparams.audio_n_fft / 2 + 1) ||
hparams.window.size() != (size_t) hparams.audio_window_len) {
throw std::runtime_error(mtmd_format("%s: hparams do not match the tensor shapes\n", __func__));
}
// projection to the text model
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight"));
model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias"));
} break;
case PROJECTOR_TYPE_GRANITE_SPEECH:
{
model.inp_proj_w = get_tensor(mtmd_format(TN_INP_PROJ, "weight"));
model.inp_proj_b = get_tensor(mtmd_format(TN_INP_PROJ, "bias"));
model.ctc_out_w = get_tensor(mtmd_format(TN_CTC_OUT, "weight"));
model.ctc_out_b = get_tensor(mtmd_format(TN_CTC_OUT, "bias"));
model.ctc_out_mid_w = get_tensor(mtmd_format(TN_CTC_OUT_MID, "weight"));
model.ctc_out_mid_b = get_tensor(mtmd_format(TN_CTC_OUT_MID, "bias"));
// per-layer tensors not loaded by the generic loop above
for (int il = 0; il < hparams.n_layer; ++il) {
auto & layer = model.layers[il];
layer.attn_rel_pos_emb = get_tensor(mtmd_format(TN_ATTN_REL_POS_EMB, prefix, il));
layer.ff_norm_w = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "weight"));
layer.ff_norm_b = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "bias"));
layer.ff_norm_1_w = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "weight"));
layer.ff_norm_1_b = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "bias"));
layer.ff_up_1_w = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "weight"));
layer.ff_up_1_b = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "bias"));
layer.ff_down_1_w = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "weight"));
layer.ff_down_1_b = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "bias"));
layer.norm_conv_w = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "weight"));
layer.norm_conv_b = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "bias"));
layer.conv_norm_w = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "weight"));
layer.conv_norm_b = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "bias"));
layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "weight"));
layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "weight"));
layer.conv_pw1_b = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "bias"));
layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "weight"));
layer.conv_pw2_b = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "bias"));
}
model.qf_proj_blocks.resize(1);
auto & qf = model.qf_proj_blocks[0];
qf.qf_proj_query = get_tensor(mtmd_format(TN_QF_PROJ_QUERY, prefix));
qf.qf_proj_norm_w = get_tensor(mtmd_format(TN_QF_PROJ_NORM, prefix, "weight"));
qf.qf_proj_norm_b = get_tensor(mtmd_format(TN_QF_PROJ_NORM, prefix, "bias"));
qf.qf_proj_linear_w = get_tensor(mtmd_format(TN_QF_PROJ_LINEAR, prefix, "weight"));
qf.qf_proj_linear_b = get_tensor(mtmd_format(TN_QF_PROJ_LINEAR, prefix, "bias"));
const int n_proj_layers = 2;
qf.qf_proj_layers.resize(n_proj_layers);
for (int il = 0; il < n_proj_layers; ++il) {
auto & pl = qf.qf_proj_layers[il];
pl.q_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_Q, prefix, il, "weight"));
pl.q_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_Q, prefix, il, "bias"));
pl.k_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_K, prefix, il, "weight"));
pl.k_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_K, prefix, il, "bias"));
pl.v_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_V, prefix, il, "weight"));
pl.v_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_V, prefix, il, "bias"));
pl.o_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_O, prefix, il, "weight"));
pl.o_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_O, prefix, il, "bias"));
pl.ln_1_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_N, prefix, il, "weight"));
pl.ln_1_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_N, prefix, il, "bias"));
pl.cross_attn_q_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_Q, prefix, il, "weight"));
pl.cross_attn_q_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_Q, prefix, il, "bias"));
pl.cross_attn_k_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_K, prefix, il, "weight"));
pl.cross_attn_k_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_K, prefix, il, "bias"));
pl.cross_attn_v_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_V, prefix, il, "weight"));
pl.cross_attn_v_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_V, prefix, il, "bias"));
pl.cross_attn_o_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_O, prefix, il, "weight"));
pl.cross_attn_o_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_O, prefix, il, "bias"));
pl.cross_attn_norm_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_N, prefix, il, "weight"));
pl.cross_attn_norm_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_N, prefix, il, "bias"));
pl.ff_up_w = get_tensor(mtmd_format(TN_QF_FFN_UP, prefix, il, "weight"));
pl.ff_up_b = get_tensor(mtmd_format(TN_QF_FFN_UP, prefix, il, "bias"));
pl.ff_down_w = get_tensor(mtmd_format(TN_QF_FFN_DOWN, prefix, il, "weight"));
pl.ff_down_b = get_tensor(mtmd_format(TN_QF_FFN_DOWN, prefix, il, "bias"));
pl.ln_2_w = get_tensor(mtmd_format(TN_QF_FFN_NORM, prefix, il, "weight"));
pl.ln_2_b = get_tensor(mtmd_format(TN_QF_FFN_NORM, prefix, il, "bias"));
}
} break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
// image_newline lives at the top-level.
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
// Load separate layerwise and spatial projector tensors
const auto projector_count = hparams.feature_layers.size();
model.qf_proj_blocks.resize(projector_count);
for (size_t bid = 0; bid < projector_count; ++bid) {
auto & b = model.qf_proj_blocks[bid];
// non-layerwise tensors
b.qf_proj_img_pos = get_tensor(mtmd_format(TN_MULTI_PROJ_IMG_POS, bid));
b.qf_proj_query = get_tensor(mtmd_format(TN_MULTI_PROJ_QUERY, prefix, bid));
b.qf_proj_linear_w = get_tensor(mtmd_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "weight"));
b.qf_proj_linear_b = get_tensor(mtmd_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "bias"));
b.qf_proj_norm_w = get_tensor(mtmd_format(TN_MULTI_PROJ_NORM, prefix, bid, "weight"));
b.qf_proj_norm_b = get_tensor(mtmd_format(TN_MULTI_PROJ_NORM, prefix, bid, "bias"));
b.qf_proj_post_norm_w = get_tensor(mtmd_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "weight"));
b.qf_proj_post_norm_b = get_tensor(mtmd_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "bias"));
// laywerwise tensors
// NOTE: If any model uses multi-layer qformers, this will need to change
b.qf_proj_layers.resize(1);
auto & pl = b.qf_proj_layers[0];
pl.q_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_Q, prefix, bid, "weight"));
pl.q_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_Q, prefix, bid, "bias"));
pl.k_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_K, prefix, bid, "weight"));
pl.k_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_K, prefix, bid, "bias"));
pl.v_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_V, prefix, bid, "weight"));
pl.v_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_V, prefix, bid, "bias"));
pl.o_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_O, prefix, bid, "weight"));
pl.o_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_O, prefix, bid, "bias"));
pl.ln_1_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_N, prefix, bid, "weight"));
pl.ln_1_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_N, prefix, bid, "bias"));
pl.cross_attn_q_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "weight"));
pl.cross_attn_q_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "bias"));
pl.cross_attn_k_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_K, prefix, bid, "weight"));
pl.cross_attn_k_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_K, prefix, bid, "bias"));
pl.cross_attn_v_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_V, prefix, bid, "weight"));
pl.cross_attn_v_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_V, prefix, bid, "bias"));
pl.cross_attn_o_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_O, prefix, bid, "weight"));
pl.cross_attn_o_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_O, prefix, bid, "bias"));
pl.cross_attn_norm_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_N, prefix, bid, "weight"));
pl.cross_attn_norm_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_N, prefix, bid, "bias"));
pl.ff_up_w = get_tensor(mtmd_format(TN_QF_FFN_UP, prefix, bid, "weight"));
pl.ff_up_b = get_tensor(mtmd_format(TN_QF_FFN_UP, prefix, bid, "bias"));
pl.ff_down_w = get_tensor(mtmd_format(TN_QF_FFN_DOWN, prefix, bid, "weight"));
pl.ff_down_b = get_tensor(mtmd_format(TN_QF_FFN_DOWN, prefix, bid, "bias"));
pl.ln_2_w = get_tensor(mtmd_format(TN_QF_FFN_NORM, prefix, bid, "weight"));
pl.ln_2_b = get_tensor(mtmd_format(TN_QF_FFN_NORM, prefix, bid, "bias"));
}
} break;
case PROJECTOR_TYPE_SENSENOVA_U1:
{
model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"));
} break;
default:
GGML_ASSERT(false && "unknown projector type");
}
// load data
{
std::vector<uint8_t> read_buf;
// start loading event
if (progress_callback){
progress_callback(0.0, progress_callback_user_data);
}
// compute total tensor data size for progress reporting
size_t total_data_size = 0;
for (auto & t : tensors_to_load) {
total_data_size += ggml_nbytes(t);
}
// alloc memory and offload data
model.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(model.ctx_data.get(), buft));
ggml_backend_buffer_set_usage(model.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
// read the weight from file
if (!no_alloc) {
size_t data_loaded = 0;
for (auto & t : tensors_to_load) {
ggml_tensor * cur = ggml_get_tensor(model.ctx_data.get(), t->name);
GGML_ASSERT(cur && "tensor not found in ctx_data");
auto it_off = tensor_offset.find(t->name);
GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor");
const size_t offset = it_off->second;
fin.seekg(offset, std::ios::beg);
if (!fin) {
throw std::runtime_error(mtmd_format("%s: failed to seek for tensor %s\n", __func__, t->name));
}
size_t num_bytes = ggml_nbytes(cur);
if (ggml_backend_buft_is_host(buft)) {
// for the CPU and Metal backend, we can read directly into the tensor
fin.read(reinterpret_cast<char *>(cur->data), num_bytes);
} else {
// read into a temporary buffer first, then copy to device memory
read_buf.resize(num_bytes);
fin.read(reinterpret_cast<char *>(read_buf.data()), num_bytes);
ggml_backend_tensor_set(cur, read_buf.data(), 0, num_bytes);
}
data_loaded += num_bytes;
if (progress_callback && total_data_size > 0) {
const float progress = (float)data_loaded / (float)total_data_size;
if (!progress_callback(progress, progress_callback_user_data)) {
throw std::runtime_error(mtmd_format("%s: model loading cancelled by progress_callback\n", __func__));
}
}
}
LOG_DBG("%s: loaded %zu tensors from %s\n", __func__, tensors_to_load.size(), fname.c_str());
#if defined(__linux__)
// the weights are copied into our buffer: drop the file's page cache (only a hint)
const int fd = ::open(fname.c_str(), O_RDONLY);
if (fd != -1) {
posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
::close(fd);
}
#endif
} else {
LOG_DBG("%s: no_alloc is set, skipping tensor data loading (%zu tensors)\n", __func__, tensors_to_load.size());
}
fin.close();
}
}
void get_bool(const std::string & key, bool & output, bool required = true) const {
const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
if (i < 0) {
if (required) {
throw std::runtime_error("Key not found: " + key);
}
return;
}
output = gguf_get_val_bool(ctx_gguf.get(), i);
}
void get_i32(const std::string & key, int & output, bool required = true) const {
const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
if (i < 0) {
if (required) {
throw std::runtime_error("Key not found: " + key);
}
return;
}
output = gguf_get_val_i32(ctx_gguf.get(), i);
}
void get_u32(const std::string & key, int & output, bool required = true) const {
const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
if (i < 0) {
if (required) {
throw std::runtime_error("Key not found: " + key);
}
return;
}
const uint32_t val = gguf_get_val_u32(ctx_gguf.get(), i);
// sanity check
if (val > (uint32_t) INT32_MAX) {
throw std::runtime_error(mtmd_format("%s: value %u for key '%s' exceeds INT32_MAX\n",
__func__, val, key.c_str()));
}
output = (int) val;
}
void get_f32(const std::string & key, float & output, bool required = true) const {
const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
if (i < 0) {
if (required) {
throw std::runtime_error("Key not found: " + key);
}
return;
}
output = gguf_get_val_f32(ctx_gguf.get(), i);
}
void get_arr_f32(const std::string & key, std::vector<float> & output, bool required = true) const {
const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
if (i < 0) {
if (required) {
throw std::runtime_error("Key not found: " + key);
}
return;
}
if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) {
throw std::runtime_error(mtmd_format("%s: key '%s' is not an array\n", __func__, key.c_str()));
}
const auto type = gguf_get_arr_type(ctx_gguf.get(), i);
if (type != GGUF_TYPE_FLOAT32) {
throw std::runtime_error(mtmd_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_FLOAT32)\n", __func__, key.c_str(), type, GGUF_TYPE_FLOAT32));
}
const size_t n = gguf_get_arr_n(ctx_gguf.get(), i);
if (n > (size_t) std::numeric_limits<int>::max()) {
throw std::runtime_error(mtmd_format("%s: array '%s' is too large (%zu elements)\n", __func__, key.c_str(), n));
}
output.resize(n);
const float * values = (const float *)gguf_get_arr_data(ctx_gguf.get(), i);
for (size_t j = 0; j < n; ++j) {
output[j] = values[j];
}
}
void get_string(const std::string & key, std::string & output, bool required = true) const {
const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
if (i < 0) {
if (required) {
throw std::runtime_error("Key not found: " + key);
}
return;
}
output = std::string(gguf_get_val_str(ctx_gguf.get(), i));
}
void get_arr_int(const std::string & key, std::vector<int> & output, bool required = true) const {
const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
if (i < 0) {
if (required) {
throw std::runtime_error("Key not found: " + key);
}
return;
}
if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) {
throw std::runtime_error(mtmd_format("%s: key '%s' is not an array\n", __func__, key.c_str()));
}
const auto type = gguf_get_arr_type(ctx_gguf.get(), i);
if (type != GGUF_TYPE_INT32) {
throw std::runtime_error(mtmd_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_INT32)\n", __func__, key.c_str(), type, GGUF_TYPE_INT32));
}
const size_t n = gguf_get_arr_n(ctx_gguf.get(), i);
if (n > (size_t) std::numeric_limits<int>::max()) {
throw std::runtime_error(mtmd_format("%s: array '%s' is too large (%zu elements)\n", __func__, key.c_str(), n));
}
output.resize(n);
const int32_t * values = (const int32_t *)gguf_get_arr_data(ctx_gguf.get(), i);
for (size_t j = 0; j < n; ++j) {
output[j] = values[j];
}
}
static void set_llava_uhd_res_candidates(mtmd_model & model, const int max_patches_per_side) {
auto & hparams = model.hparams;
for (int x = 1; x <= max_patches_per_side; x++) {
for (int y = 1; y <= max_patches_per_side; y++) {
if (x == 1 && y == 1) {
continue; // skip the first point
}
hparams.image_res_candidates.push_back(mtmd_image_size{
x*hparams.image_size,
y*hparams.image_size,
});
}
}
}
static void set_internvl_dhr_res_candidates(mtmd_model & model) {
auto & hparams = model.hparams;
int min_num = hparams.preproc_min_tiles;
int max_num = hparams.preproc_max_tiles;
if (min_num < 1) {
return; // avoid divide by 0
}
for (int a = min_num; a <= max_num; ++a) {
int b_lo = (min_num + a - 1) / a;
int b_hi = max_num / a;
b_lo = std::max(b_lo, min_num);
b_hi = std::min(b_hi, max_num);
for (int b = b_lo; b <= b_hi; ++b) {
hparams.image_res_candidates.push_back(mtmd_image_size {
a*hparams.image_size,
b*hparams.image_size,
});
}
}
}
};
//
// loading
//
mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params, ggml_backend_buffer_type_t buft) {
mtmd_model_loader loader(fname,
/* skip_tensors */ false,
params.progress_callback,
params.progress_callback_user_data);
auto load = [&](mtmd_modality modality) {
auto model = std::make_unique<mtmd_model>();
model->hparams.custom_image_min_tokens = params.image_min_tokens;
model->hparams.custom_image_max_tokens = params.image_max_tokens;
model->no_alloc = params.no_alloc;
loader.load_hparams(*model, modality);
loader.load_tensors(*model, buft, params.no_alloc);
const std::string & def = model->hparams.gen_default_voice;
if (!def.empty() && model->gen_voices.find(def) == model->gen_voices.end()) {
throw std::runtime_error(mtmd_format("%s: default voice \"%s\" is not a predefined voice\n", __func__, def.c_str()));
}
return model;
};
mtmd_models res;
if (loader.has_vision) {
res.v = load(MTMD_MODALITY_VISION);
}
// TODO: we don't support audio for Gemma 3N, but GGUF contains audio tensors
// we can remove this check when we implement audio support for Gemma 3N
const bool skip_audio = res.v && res.v->proj_type == PROJECTOR_TYPE_GEMMA3NV;
if (loader.has_audio && !skip_audio) {
res.a = load(MTMD_MODALITY_AUDIO);
}
if (loader.has_gen_audio) {
res.gen_a = load(MTMD_MODALITY_GEN_AUDIO);
}
return res;
}
void mtmd_model_release_device(mtmd_model & model) {
if (model.dev_released || model.no_alloc) {
return;
}
ggml_backend_buffer_t buf = model.buf.get();
if (buf == nullptr || ggml_backend_buffer_is_host(buf) || ggml_backend_buffer_get_size(buf) == 0) {
return; // CPU-backed encoder: nothing in VRAM to free
}
model.dev_buft = ggml_backend_buffer_get_type(buf);
ggml_context * cd = model.ctx_data.get();
// capture the host shadow once (weights are read-only): compact, stable iteration order,
// skipping view tensors (which alias a base and are restored implicitly)
if (model.dev_shadow.empty()) {
size_t total = 0;
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
if (t->view_src == nullptr) { total += ggml_nbytes(t); }
}
model.dev_shadow.resize(total);
size_t off = 0;
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
if (t->view_src != nullptr) { continue; }
const size_t n = ggml_nbytes(t);
ggml_backend_tensor_get(t, model.dev_shadow.data() + off, 0, n);
off += n;
}
}
// free the device buffer and clear the now-dangling tensor pointers so restore reallocates cleanly
model.buf.reset();
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
t->buffer = nullptr;
t->data = nullptr;
}
model.dev_released = true;
}
bool mtmd_model_restore_device(mtmd_model & model) {
if (!model.dev_released) {
return true;
}
ggml_context * cd = model.ctx_data.get();
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(cd, model.dev_buft);
if (buf == nullptr) {
LOG_ERR("%s: failed to reallocate encoder device buffer (out of VRAM?)\n", __func__);
return false;
}
ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
size_t off = 0;
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
if (t->view_src != nullptr) { continue; }
const size_t n = ggml_nbytes(t);
ggml_backend_tensor_set(t, model.dev_shadow.data() + off, 0, n);
off += n;
}
model.buf.reset(buf);
model.dev_released = false;
return true;
}
mtmd_model_caps mtmd_model_get_caps(const char * fname) {
mtmd_model_caps res;
mtmd_model_loader loader(fname, /* skip_tensors= */ true);
res.has_vision = loader.has_vision;
res.has_audio = loader.has_audio;
res.has_gen_audio = loader.has_gen_audio;
return res;
}
//
// mtmd_model
//
int mtmd_model::image_max_tokens() const {
if (proj_type == PROJECTOR_TYPE_DEEPSEEK4V || proj_type == PROJECTOR_TYPE_DEEPSEEK41V) {
return hparams.dsv4_max_n_token;
}
if (hparams.image_max_pixels <= 0) {
return -1;
}
const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
return hparams.image_max_pixels / patch_area;
}
int mtmd_model::n_output_tokens_x(const mtmd_image_f32 * img) const {
const auto & params = hparams;
const int n_total = n_output_tokens(img);
const auto & proj = proj_type;
switch (proj) {
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_MAGEVL:
case PROJECTOR_TYPE_SENSENOVA_U1:
case PROJECTOR_TYPE_LING3VL:
case PROJECTOR_TYPE_EXAONE4_5:
case PROJECTOR_TYPE_MIMOVL:
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_GLM5V:
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_HUNYUANVL:
case PROJECTOR_TYPE_YOUTUVL:
case PROJECTOR_TYPE_MUSE_GLIMMER:
case PROJECTOR_TYPE_LFM3VL:
return (img->nx() / params.patch_size) / 2;
case PROJECTOR_TYPE_STEP3VL:
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
case PROJECTOR_TYPE_INKLING2V:
return img->nx() / (params.patch_size * params.n_merge);
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
return (img->nx() / params.patch_size) / 4;
default:
break;
}
return n_total;
}
int mtmd_model::n_output_tokens_y(const mtmd_image_f32 * img) const {
const auto & params = hparams;
const auto & proj = proj_type;
switch (proj) {
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_MAGEVL:
case PROJECTOR_TYPE_SENSENOVA_U1:
case PROJECTOR_TYPE_LING3VL:
case PROJECTOR_TYPE_EXAONE4_5:
case PROJECTOR_TYPE_MIMOVL:
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_GLM5V:
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_HUNYUANVL:
case PROJECTOR_TYPE_YOUTUVL:
case PROJECTOR_TYPE_MUSE_GLIMMER:
case PROJECTOR_TYPE_LFM3VL:
return (img->ny() / params.patch_size) / 2;
case PROJECTOR_TYPE_STEP3VL:
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
case PROJECTOR_TYPE_INKLING2V:
return img->ny() / (params.patch_size * params.n_merge);
default:
break;
}
return 1;
}
int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
const auto & params = hparams;
// for models with fixed size image, the input image is already pre-processed and resized to square
int patch_size = params.patch_size;
int n_patches = (img->nx() / patch_size) * (img->ny() / patch_size);
projector_type proj = proj_type;
switch (proj) {
case PROJECTOR_TYPE_MLP:
case PROJECTOR_TYPE_MLP_NORM:
case PROJECTOR_TYPE_JANUS_PRO:
case PROJECTOR_TYPE_PHI4:
{
// do nothing
} break;
case PROJECTOR_TYPE_YASA2:
{
n_patches = 64; // adaptive average pooling to 8x8 tokens
} break;
case PROJECTOR_TYPE_LDP:
case PROJECTOR_TYPE_LDPV2:
case PROJECTOR_TYPE_GLM_EDGE:
{
n_patches /= 4;
if (mm_boi) {
n_patches += 2; // for BOI and EOI token embeddings
}
} break;
case PROJECTOR_TYPE_MINICPMV:
{
// Use actual config value if available, otherwise fall back to hardcoded values
if (params.minicpmv_query_num > 0) {
n_patches = params.minicpmv_query_num;
} else {
// Fallback to hardcoded values for legacy models
if (params.minicpmv_version == 2) {
n_patches = 96;
} else if (params.minicpmv_version == 3) {
n_patches = 64;
} else if (params.minicpmv_version == 4) {
n_patches = 64;
} else if (params.minicpmv_version == 5) {
// MiniCPM-V 4.0
n_patches = 64;
} else if (params.minicpmv_version == 6) {
// MiniCPM-V 4.5
n_patches = 64;
} else if (params.minicpmv_version == 100045) {
// MiniCPM-o 4.5
n_patches = 64;
} else {
GGML_ABORT("Unknown minicpmv version");
}
}
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
n_patches /= params.n_merge * params.n_merge;
} break;
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_MAGEVL:
case PROJECTOR_TYPE_SENSENOVA_U1:
case PROJECTOR_TYPE_LING3VL:
case PROJECTOR_TYPE_EXAONE4_5:
case PROJECTOR_TYPE_MIMOVL:
case PROJECTOR_TYPE_MINIMAX_M3:
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_GLM5V:
case PROJECTOR_TYPE_YOUTUVL:
case PROJECTOR_TYPE_MUSE_GLIMMER:
case PROJECTOR_TYPE_LFM3VL:
{
// dynamic size (2 conv, so double patch size)
int x_patch = img->nx() / (params.patch_size * 2);
int y_patch = img->ny() / (params.patch_size * 2);
n_patches = x_patch * y_patch;
} break;
case PROJECTOR_TYPE_STEP3VL:
case PROJECTOR_TYPE_INKLING2V:
{
int x_patch = img->nx() / (params.patch_size * params.n_merge);
int y_patch = img->ny() / (params.patch_size * params.n_merge);
n_patches = x_patch * y_patch;
} break;
case PROJECTOR_TYPE_GEMMA3:
case PROJECTOR_TYPE_GEMMA4V:
case PROJECTOR_TYPE_GEMMA4UV:
case PROJECTOR_TYPE_IDEFICS3:
case PROJECTOR_TYPE_COHERE2V:
case PROJECTOR_TYPE_INTERNVL:
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
case PROJECTOR_TYPE_NEMOTRON_OMNI:
case PROJECTOR_TYPE_LLAMA4:
{
// both X and Y are downscaled by the scale factor
int scale_factor = hparams.n_merge;
n_patches /= (scale_factor * scale_factor);
} break;
case PROJECTOR_TYPE_GEMMA3NV:
{
// MobileNetV5 MSFA adapter always outputs fixed 16x16 resolution
// regardless of input size (see architecture description)
n_patches = hparams.image_size / hparams.patch_size;
} break;
case PROJECTOR_TYPE_LFM2:
case PROJECTOR_TYPE_D1OMNI_V:
case PROJECTOR_TYPE_KIMIVL:
case PROJECTOR_TYPE_KIMIK25:
case PROJECTOR_TYPE_KIMIK31:
{
// dynamic size
int out_patch_size = params.patch_size * hparams.n_merge;
int x_patch = MTMD_ALIGN(img->nx(), out_patch_size) / out_patch_size;
int y_patch = MTMD_ALIGN(img->ny(), out_patch_size) / out_patch_size;
n_patches = x_patch * y_patch;
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
const int out_patch_size = params.patch_size * params.n_merge;
const int n_llm_w = MTMD_ALIGN(img->nx(), out_patch_size) / out_patch_size;
const int n_llm_h = MTMD_ALIGN(img->ny(), out_patch_size) / out_patch_size;
n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out;
} break;
case PROJECTOR_TYPE_DEEPSEEK41V:
{
const int out_patch_size = params.patch_size * params.n_merge;
const int n_llm_w = MTMD_ALIGN(img->nx(), out_patch_size) / out_patch_size;
const int n_llm_h = MTMD_ALIGN(img->ny(), out_patch_size) / out_patch_size;
n_patches = dsv41_n_block_tokens(n_llm_w, n_llm_h);
} break;
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
{
// dynamic size
int n_merge = hparams.n_merge;
int stride = n_merge * n_merge;
n_patches = MTMD_ALIGN(n_patches, stride) / stride;
} break;
case PROJECTOR_TYPE_DOTS3NOTE_A:
{
// 3x stride-2 conv2d over mel frames
n_patches = (img->nx() + 7) / 8;
} break;
case PROJECTOR_TYPE_PIXTRAL:
case PROJECTOR_TYPE_LIGHTONOCR:
{
// dynamic size
int n_merge = hparams.n_merge;
int n_patches_x = img->nx() / patch_size / n_merge;
int n_patches_y = img->ny() / patch_size / n_merge;
if (token_embd_img_break) {
n_patches = n_patches_y * n_patches_x + n_patches_y - 1; // + one [IMG_BREAK] per row, except the last row
} else {
n_patches = n_patches_y * n_patches_x;
}
} break;
case PROJECTOR_TYPE_VOXTRAL_RT:
{
// 2 context frames, conv stride 2, then stacked frames
n_patches = (img->nx() - 2) / 2 / hparams.proj_stack_factor;
} break;
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
{
const int64_t fold = hparams.audio_samples_per_token();
n_patches = (int) ((img->nx() + fold - 1) / fold);
} break;
case PROJECTOR_TYPE_VIBEVOICE_ASR:
{
n_patches = (img->nx() - img->audio_n_ctx) / hparams.audio_samples_per_token();
} break;
case PROJECTOR_TYPE_VOXTRAL:
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_QWEN2A:
case PROJECTOR_TYPE_MERALION:
case PROJECTOR_TYPE_MUSIC_FLAMINGO:
{
n_patches = img->nx();
const int proj_stack_factor = hparams.proj_stack_factor;
if (audio_has_stack_frames()) {
GGML_ASSERT(proj_stack_factor > 0);
const int n_len = MTMD_ALIGN(n_patches, proj_stack_factor);
n_patches = n_len / proj_stack_factor;
}
// whisper downscales input token by half after conv1d
n_patches /= 2;
if (audio_has_avgpool()) {
// divide by 2 because of nn.AvgPool1d(2, stride=2)
n_patches /= 2;
}
} break;
case PROJECTOR_TYPE_HIGGS_AUDIO:
{
// stride-2 conv1d, avgpool(2), stride-2 depthwise conv1d
n_patches = (img->nx() + 1) / 2;
n_patches = n_patches / 2;
n_patches = (n_patches + 1) / 2;
} break;
case PROJECTOR_TYPE_QWEN3A:
{
// chunk_size=100 frames --> 3x stride-2 conv2d --> 13 tokens per chunk
const int chunk_size = 100;
const int tokens_per_chunk = 13;
n_patches = (img->nx() / chunk_size) * tokens_per_chunk;
} break;
case PROJECTOR_TYPE_GLMA:
{
n_patches = img->nx();
// whisper downscales input token by half after conv1d
n_patches /= 2;
// reshape by merge_factor
n_patches /= hparams.proj_stack_factor;
// for BOI and EOI token embeddings
n_patches += 2;
} break;
case PROJECTOR_TYPE_COGVLM:
{
n_patches += 2; // for BOI and EOI token embeddings
} break;
case PROJECTOR_TYPE_DEEPSEEKOCR:
{
// SAM encoder applies two stride-2 convolutions (net_2 and net_3)
// that reduce spatial dimensions by 4x in each direction (16x total)
// E.g., 64x64 -> 16x16 patches
n_patches /= 16;
if (img->add_viewsep) {
// global view: one image-newline per token-row + trailing view separator
const int h = static_cast<int>(std::sqrt(static_cast<float>(n_patches)));
n_patches = h * (h + 1) + 1;
} else if (img->ny() >= img->nx() && img->ny() % img->nx() == 0) {
// tile row: one image-newline per token-row
const int grid_w = img->ny() / img->nx();
const int tile_patches = img->nx() / (patch_size * 4); // patches per tile side (SAM divides by 4)
const int h = tile_patches;
n_patches = (tile_patches * grid_w + 1) * h;
}
} break;
case PROJECTOR_TYPE_HUNYUANVL:
{
int merge = hparams.n_merge;
int ow = (img->nx() / patch_size) / merge;
int oh = (img->ny() / patch_size) / merge;
n_patches = (ow + 1) * oh + 2;
} break;
case PROJECTOR_TYPE_DEEPSEEKOCR2:
{
// 1024 global view -> 256 query tokens + 1 view separator = 257;
// 768 local tile -> 144 query tokens, no separator.
n_patches /= 16;
if (img->add_viewsep) {
n_patches += 1; // view separator, appended only after the global view
}
} break;
case PROJECTOR_TYPE_LFM2A:
case PROJECTOR_TYPE_D1OMNI_A:
case PROJECTOR_TYPE_LFM3A:
{
n_patches = ((((img->nx() + 1) / 2) + 1) / 2 + 1) / 2;
} break;
case PROJECTOR_TYPE_GEMMA4A:
{
// Two Conv2D stride-2: O = floor((I + 2p - k) / s) + 1, p=1, k=3, s=2
// O = floor((I - 1) / 2) + 1
int n = img->nx();
for (int i = 0; i < 2; i++) {
n = (n - 1) / 2 + 1;
}
n_patches = n;
} break;
case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
case PROJECTOR_TYPE_PARAKEET:
{
n_patches = (img->nx() + (params.subsampling_factor - 1)) / params.subsampling_factor;
} break;
case PROJECTOR_TYPE_GEMMA4UA:
{
n_patches = img->nx(); // no downsampling: one token per raw waveform frame
} break;
case PROJECTOR_TYPE_MIMO_AUDIO:
{
// conv1(s=1) + conv2(s=2) -> RVQ-encoder downsample conv(k=2,s=2)
int n = img->nx();
n = (n - 1) / 2 + 1; // conv1 + conv2
n = (n - 2) / 2 + 1; // downsample conv
const int group_size = params.audio_local_group_size;
n_patches = (n + group_size - 1) / group_size;
} break;
case PROJECTOR_TYPE_GRANITE_SPEECH:
{
const int ws = hparams.audio_proj_window_size;
const int ds = hparams.audio_proj_downsample_rate;
n_patches = ((img->nx() + ws - 1) / ws) * (ws / ds);
} break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
case PROJECTOR_TYPE_ZONOS3_SPKENC:
{
// pooling gives one speaker embedding, whatever the clip length is
n_patches = 1;
} break;
case PROJECTOR_TYPE_CHATTERBOX_SPKENC:
{
// statistics pooling gives one speaker embedding, whatever the clip length is
n_patches = 1;
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
case PROJECTOR_TYPE_LFM3A_GEN:
case PROJECTOR_TYPE_DRAMA3_GEN:
case PROJECTOR_TYPE_ZONOS3_GEN:
case PROJECTOR_TYPE_YUE2_GEN:
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
case PROJECTOR_TYPE_FISH_S2_GEN:
case PROJECTOR_TYPE_VOXCPM2_GEN:
{
// one hidden-state vector fed back to the talker per call
n_patches = 1;
} break;
case PROJECTOR_TYPE_FISH_S2_SPKENC:
case PROJECTOR_TYPE_VOXCPM2_SPKENC:
{
// one backbone row per codec frame, the preprocessor pads to whole frames
n_patches = (int) (img->nx() / hparams.audio_samples_per_token());
} break;
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
{
// one conditioning row per 12.5Hz frame
const int hop = hparams.mimi_downsample * 120;
n_patches = img->nx() / hop;
} break;
case PROJECTOR_TYPE_POCKETTTS_GEN:
{
// one latent per call for GEN_CODE, GEN_WAV sizes its input from the caller
n_patches = 1;
} break;
case PROJECTOR_TYPE_MOSHI:
{
// one row per codebook and frame, then the rows of the initial codes, see mtmd_graph_moshi
const int n_q = (int) mm_a_code_embd->ne[2];
n_patches = n_q * (img->nx() / mtmd_graph_moshi::n_samples_per_frame + 1);
} break;
case PROJECTOR_TYPE_MOSHI_GEN:
{
n_patches = 1;
} break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
// Per-tile output token count: each projector block outputs
// query_side^2 tokens per window x n^2 windows.
// For 384x384 input: n = 24/8 = 3, query_side = 4 -> 144.
const int window_side = hparams.downsample_window_side;
const int query_side = hparams.downsample_query_side;
const int side = img->nx() / params.patch_size;
const int n = side / window_side;
const int out_side = query_side * n;
n_patches = out_side * out_side;
if (img->anyres.is_tiled()) {
// overview tile, then the unpadded tile grid with one newline per row
int off_x, off_y, w, h;
mtmd_anyres_unpad(img->anyres.grid_x * out_side, img->anyres.grid_y * out_side,
img->anyres.orig_nx, img->anyres.orig_ny, off_x, off_y, w, h);
n_patches += h * (w + 1);
}
} break;
default:
GGML_ABORT("unsupported projector type");
}
n_patches += (int) mtmd_suffix_rows(img->suffix_type).size();
return n_patches;
}
size_t mtmd_model::cbx_read_tensor(const char * name, float * out, size_t n_max) const {
const auto & tensors = cbx_tensors;
auto it = tensors.find(name);
if (it == tensors.end() || !it->second) {
return 0;
}
ggml_tensor * t = it->second;
const size_t n = (size_t) ggml_nelements(t);
if (!out) {
return n;
}
if (n_max < n) {
return 0;
}
if (t->type == GGML_TYPE_F32) {
ggml_backend_tensor_get(t, out, 0, n * sizeof(float));
} else if (t->type == GGML_TYPE_F16) {
std::vector<ggml_fp16_t> tmp(n);
ggml_backend_tensor_get(t, tmp.data(), 0, n * sizeof(ggml_fp16_t));
for (size_t i = 0; i < n; i++) out[i] = ggml_fp16_to_fp32(tmp[i]);
} else {
return 0;
}
return n;
}
int mtmd_model::n_mmproj_embd() const {
switch (proj_type) {
case PROJECTOR_TYPE_LDP:
return mm_model_block_1_block_2_1_b->ne[0];
case PROJECTOR_TYPE_LDPV2:
return mm_model_peg_0_b->ne[0];
case PROJECTOR_TYPE_MLP:
case PROJECTOR_TYPE_PHI4:
case PROJECTOR_TYPE_PIXTRAL:
case PROJECTOR_TYPE_LIGHTONOCR:
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
case PROJECTOR_TYPE_DOTS3NOTE_A:
return mm_2_w->ne[1];
case PROJECTOR_TYPE_MLP_NORM:
return mm_3_b->ne[0];
case PROJECTOR_TYPE_MINICPMV:
return mm_model_proj->ne[1];
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
return mm_ffn_down_w->ne[1];
case PROJECTOR_TYPE_GLM_EDGE:
return mm_model_mlp_3_w->ne[1];
case PROJECTOR_TYPE_MINIMAX_M3:
return mm_merger_fc2_b->ne[0];
case PROJECTOR_TYPE_MUSE_GLIMMER:
case PROJECTOR_TYPE_INKLING2V:
return mm_2_w->ne[1];
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_MAGEVL:
case PROJECTOR_TYPE_LFM3VL:
case PROJECTOR_TYPE_EXAONE4_5:
case PROJECTOR_TYPE_JANUS_PRO:
case PROJECTOR_TYPE_YOUTUVL:
return mm_1_b->ne[0];
case PROJECTOR_TYPE_SENSENOVA_U1:
return hparams.projection_dim;
case PROJECTOR_TYPE_QWEN3VL:
// main path + deepstack paths
return mm_1_b->ne[0] * (1 + n_deepstack_layers);
case PROJECTOR_TYPE_LING3VL:
return mm_1_b->ne[0];
case PROJECTOR_TYPE_MIMOVL:
return mm_1_w->ne[1];
case PROJECTOR_TYPE_STEP3VL:
return mm_model_proj->ne[1];
case PROJECTOR_TYPE_GEMMA3:
case PROJECTOR_TYPE_GEMMA3NV:
return mm_input_proj_w->ne[0];
case PROJECTOR_TYPE_GEMMA4V:
case PROJECTOR_TYPE_GEMMA4UV:
case PROJECTOR_TYPE_GEMMA4A:
case PROJECTOR_TYPE_GEMMA4UA:
return mm_input_proj_w->ne[1];
case PROJECTOR_TYPE_IDEFICS3:
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
return mm_fc_w->ne[1];
case PROJECTOR_TYPE_VIBEVOICE_ASR:
return mm_1_w->ne[1];
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_VOXTRAL:
case PROJECTOR_TYPE_VOXTRAL_RT:
case PROJECTOR_TYPE_MUSIC_FLAMINGO:
return mm_2_w->ne[1];
case PROJECTOR_TYPE_MERALION:
return mm_3_w->ne[1]; // out_proj output dim
case PROJECTOR_TYPE_INTERNVL:
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
case PROJECTOR_TYPE_NEMOTRON_OMNI:
return mm_3_w->ne[1];
case PROJECTOR_TYPE_LLAMA4:
return mm_model_proj->ne[1];
case PROJECTOR_TYPE_QWEN2A:
case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
return mm_fc_w->ne[1];
case PROJECTOR_TYPE_QWEN3A:
case PROJECTOR_TYPE_HIGGS_AUDIO:
return mm_2_w->ne[1];
case PROJECTOR_TYPE_GLMA:
case PROJECTOR_TYPE_LFM2:
case PROJECTOR_TYPE_D1OMNI_V:
case PROJECTOR_TYPE_KIMIVL:
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_KIMIK25:
case PROJECTOR_TYPE_KIMIK31:
case PROJECTOR_TYPE_YASA2:
case PROJECTOR_TYPE_DEEPSEEK4V:
case PROJECTOR_TYPE_DEEPSEEK41V:
case PROJECTOR_TYPE_COHERE2V:
return mm_2_w->ne[1];
case PROJECTOR_TYPE_HUNYUANVL:
return mm_model_proj->ne[1];
case PROJECTOR_TYPE_COGVLM:
return mm_4h_to_h_w->ne[1];
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
return mm_fc_w->ne[1];
case PROJECTOR_TYPE_LFM2A:
return position_embeddings->ne[0];
case PROJECTOR_TYPE_D1OMNI_A:
case PROJECTOR_TYPE_LFM3A:
return mm_3_w->ne[1];
case PROJECTOR_TYPE_GRANITE_SPEECH:
return qf_proj_blocks[0].qf_proj_linear_w->ne[1];
case PROJECTOR_TYPE_GRANITE4_VISION:
return qf_proj_blocks.size() * hparams.projection_dim;
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_GLM5V:
return mm_ffn_down_w->ne[1];
case PROJECTOR_TYPE_MIMO_AUDIO:
return mm_2_w->ne[1];
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
return mm_fc_w->ne[2];
case PROJECTOR_TYPE_ZONOS3_SPKENC:
return zonos3_spk.at("a.spk.proj.weight")->ne[1];
case PROJECTOR_TYPE_QWEN3TTS_GEN:
case PROJECTOR_TYPE_LFM3A_GEN:
case PROJECTOR_TYPE_DRAMA3_GEN:
case PROJECTOR_TYPE_ZONOS3_GEN:
case PROJECTOR_TYPE_FISH_S2_GEN:
case PROJECTOR_TYPE_FISH_S2_SPKENC:
return gen_code_out_embd_w->ne[0];
case PROJECTOR_TYPE_YUE2_GEN:
return yue2.inp_w->ne[1];
case PROJECTOR_TYPE_VOXCPM2_GEN:
return voxcpm2.enc_to_lm_w->ne[1];
case PROJECTOR_TYPE_VOXCPM2_SPKENC:
// the rows come from LocEnc of the gen model, see voxcpm2_ref_encode()
return hparams.projection_dim;
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
return gen_code_out_embd_w->ne[0];
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
return spk_proj_w->ne[1];
case PROJECTOR_TYPE_POCKETTTS_GEN:
return gen_input_lin_w->ne[1];
case PROJECTOR_TYPE_MOSHI:
return mm_a_code_embd->ne[0];
case PROJECTOR_TYPE_MOSHI_GEN:
return gen_code_out_embd_w->ne[0];
case PROJECTOR_TYPE_CHATTERBOX:
// talker width, carried by the speech table
return cbx_tensors.at("a.gen.code.out_embd.weight")->ne[0];
case PROJECTOR_TYPE_CHATTERBOX_SPKENC:
// talker conditioning rows, the speaker projection carries their width
return cbx_tensors.at("a.cenc.spkr_enc.weight")->ne[1];
case PROJECTOR_TYPE_SOPRANO:
case PROJECTOR_TYPE_KOKORO:
case PROJECTOR_TYPE_TINY_VOCA:
return hparams.projection_dim;
case PROJECTOR_TYPE_PARAKEET:
return mm_1_w->ne[1];
default:
GGML_ABORT("Unknown projector type");
}
}
// TODO @ngxson : this is no longer correct with mtmd_batch API
// this was only meant to be used by qwen-vl-based models, to fuse 2 input images into one (qwen-vl video support)
// this logic should be refactored in near future to distinctly handle "merge frames" and "batching"
int mtmd_model::n_temporal_merge() const {
switch (proj_type) {
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_LFM3VL:
case PROJECTOR_TYPE_NEMOTRON_OMNI:
case PROJECTOR_TYPE_LING3VL:
return 2;
default:
return 1;
}
}