#include "mtmd-impl.h" #include "mtmd-model.h" #include "models/models.h" #include "ggml.h" #include "ggml-cpp.h" #include "ggml-alloc.h" #include "ggml-backend.h" #include "gguf.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // // mtmd_model_loader // struct mtmd_model_loader { ggml_context_ptr ctx_meta; gguf_context_ptr ctx_gguf; std::string fname; size_t model_size = 0; // in bytes bool has_vision = false; bool has_audio = false; bool has_gen_audio = false; mtmd_progress_callback progress_callback = nullptr; void * progress_callback_user_data = nullptr; mtmd_model_loader(const char * fname, bool skip_tensors = false, mtmd_progress_callback progress_cb = nullptr, void * progress_user_data = nullptr) : fname(fname), progress_callback(progress_cb), progress_callback_user_data(progress_user_data) { struct ggml_context * meta = nullptr; struct gguf_init_params params = { /*.no_alloc = */ true, /*.ctx = */ &meta, }; ctx_gguf = gguf_context_ptr(gguf_init_from_file(fname, params)); if (!ctx_gguf.get()) { throw std::runtime_error(mtmd_format("%s: failed to load mmproj model from %s. Does this file exist?\n", __func__, fname)); } ctx_meta.reset(meta); const int n_tensors = gguf_get_n_tensors(ctx_gguf.get()); // print gguf info { std::string name; get_string(KEY_NAME, name, false); std::string description; get_string(KEY_DESCRIPTION, description, false); LOG_INF("%s: model name: %s\n", __func__, name.c_str()); LOG_INF("%s: description: %s\n", __func__, description.c_str()); LOG_INF("%s: GGUF version: %d\n", __func__, gguf_get_version(ctx_gguf.get())); LOG_INF("%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx_gguf.get())); LOG_INF("%s: n_tensors: %d\n", __func__, n_tensors); LOG_INF("%s: n_kv: %d\n", __func__, (int)gguf_get_n_kv(ctx_gguf.get())); LOG_INF("\n"); } // modalities { get_bool(KEY_HAS_VISION_ENC, has_vision, false); get_bool(KEY_HAS_AUDIO_ENC, has_audio, false); get_bool(KEY_HAS_GEN_AUDIO_ENC, has_gen_audio, false); if (has_vision) { LOG_INF("%s: has vision encoder\n", __func__); } if (has_audio) { LOG_INF("%s: has audio encoder\n", __func__); } if (has_gen_audio) { LOG_INF("%s: has audio generation (gen) encoder\n", __func__); } } // tensors if (!skip_tensors) { for (int i = 0; i < n_tensors; ++i) { const char * name = gguf_get_tensor_name(ctx_gguf.get(), i); const size_t offset = gguf_get_tensor_offset(ctx_gguf.get(), i); enum ggml_type type = gguf_get_tensor_type(ctx_gguf.get(), i); ggml_tensor * cur = ggml_get_tensor(meta, name); size_t tensor_size = ggml_nbytes(cur); model_size += tensor_size; LOG_DBG("%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, offset=%zu, shape:[%" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "], type = %s\n", __func__, i, ggml_n_dims(cur), cur->name, tensor_size, offset, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3], ggml_type_name(type)); } } } void load_hparams(mtmd_model & model, mtmd_modality modality) { auto & hparams = model.hparams; std::string log_ffn_op; // for logging // sanity check if (modality == MTMD_MODALITY_VISION) { GGML_ASSERT(has_vision); } else if (modality == MTMD_MODALITY_AUDIO) { GGML_ASSERT(has_audio); } else if (modality == MTMD_MODALITY_GEN_AUDIO) { GGML_ASSERT(has_gen_audio); } model.modality = modality; // projector type std::string proj_type; { // default key get_string(KEY_PROJ_TYPE, proj_type, false); // for models with mixed modalities if (proj_type.empty()) { if (modality == MTMD_MODALITY_VISION) { get_string(KEY_VISION_PROJ_TYPE, proj_type, false); } else if (modality == MTMD_MODALITY_AUDIO) { get_string(KEY_AUDIO_PROJ_TYPE, proj_type, false); } else if (modality == MTMD_MODALITY_GEN_AUDIO) { get_string(KEY_GEN_AUDIO_PROJ_TYPE, proj_type, false); } else { GGML_ABORT("unknown modality"); } } model.proj_type = projector_type_from_string(proj_type); if (model.proj_type == PROJECTOR_TYPE_UNKNOWN) { throw std::runtime_error(mtmd_format("%s: unknown projector type: %s\n", __func__, proj_type.c_str())); } // correct arch for multimodal models (legacy method) if (model.proj_type == PROJECTOR_TYPE_QWEN25O) { model.proj_type = modality == MTMD_MODALITY_VISION ? PROJECTOR_TYPE_QWEN25VL : PROJECTOR_TYPE_QWEN2A; } } const bool is_vision = model.modality == MTMD_MODALITY_VISION; const bool is_audio = model.modality == MTMD_MODALITY_AUDIO; const bool is_gen_audio = model.modality == MTMD_MODALITY_GEN_AUDIO; // other hparams { const char * prefix = is_vision ? "vision" : (is_audio ? "audio" : "gen.audio"); get_u32(mtmd_format(KEY_N_EMBD, prefix), hparams.n_embd); get_u32(mtmd_format(KEY_N_HEAD, prefix), hparams.n_head); get_u32(mtmd_format(KEY_N_EMBD_HEAD, prefix), hparams.n_embd_head, false); get_u32(mtmd_format(KEY_N_FF, prefix), hparams.n_ff); get_u32(mtmd_format(KEY_N_BLOCK, prefix), hparams.n_layer); get_u32(mtmd_format(KEY_PROJ_DIM, prefix), hparams.projection_dim); get_f32(mtmd_format(KEY_LAYER_NORM_EPS, prefix), hparams.eps); // n_head_kv is optional (for GQA), default to n_head hparams.n_head_kv = hparams.n_head; get_u32(mtmd_format(KEY_N_HEAD_KV, prefix), hparams.n_head_kv, false); if (is_vision) { get_u32(KEY_IMAGE_SIZE, hparams.image_size); get_u32(KEY_PATCH_SIZE, hparams.patch_size); get_u32(KEY_MAX_SLICE_NUMS, hparams.max_slice_nums, false); get_i32(KEY_MINICPMV_VERSION, hparams.minicpmv_version, false); // legacy get_u32(KEY_MINICPMV_QUERY_NUM, hparams.minicpmv_query_num, false); if (hparams.minicpmv_query_num == 0) { // Fallback to hardcoded values for legacy models if (hparams.minicpmv_version == 3) { hparams.minicpmv_query_num = 64; } else if (hparams.minicpmv_version == 4) { hparams.minicpmv_query_num = 64; } else if (hparams.minicpmv_version == 5) { hparams.minicpmv_query_num = 64; } else if (hparams.minicpmv_version == 6) { hparams.minicpmv_query_num = 64; } else if (hparams.minicpmv_version == 100045) { hparams.minicpmv_query_num = 64; } else { hparams.minicpmv_query_num = 96; } } } else if (is_audio) { get_u32(KEY_A_NUM_MEL_BINS, hparams.n_mel_bins); // some hparams are unused, but still need to set to avoid issues hparams.image_size = 0; hparams.patch_size = 1; } else if (is_gen_audio) { // these are unused, but still need to be set to avoid issues hparams.image_size = 0; hparams.patch_size = 1; get_string(KEY_GEN_AUDIO_VARIANT, hparams.gen_model_variant, false); get_string(KEY_GEN_AUDIO_DEFAULT_VOICE, hparams.gen_default_voice, false); } else { GGML_ASSERT(false && "unknown modality"); } // for pinpoints, we need to convert it into a list of resolution candidates { std::vector pinpoints; get_arr_int(KEY_IMAGE_GRID_PINPOINTS, pinpoints, false); if (pinpoints.size() % 2 != 0) { throw std::runtime_error(mtmd_format("%s: image_grid_pinpoints must have an even number of elements, got %zu\n", __func__, pinpoints.size())); } if (!pinpoints.empty()) { for (size_t i = 0; i < pinpoints.size(); i += 2) { hparams.image_res_candidates.push_back({ pinpoints[i], pinpoints[i+1], }); } } } // default warmup value hparams.warmup_image_size = hparams.image_size; { bool use_gelu = false; bool use_silu = false; get_bool(KEY_USE_GELU, use_gelu, false); get_bool(KEY_USE_SILU, use_silu, false); if (use_gelu && use_silu) { throw std::runtime_error(mtmd_format("%s: both use_gelu and use_silu are set to true\n", __func__)); } if (use_gelu) { hparams.ffn_op = FFN_GELU; log_ffn_op = "gelu"; } else if (use_silu) { hparams.ffn_op = FFN_SILU; log_ffn_op = "silu"; } else { hparams.ffn_op = FFN_GELU_QUICK; log_ffn_op = "gelu_quick"; } } { std::string mm_patch_merge_type; get_string(KEY_MM_PATCH_MERGE_TYPE, mm_patch_merge_type, false); if (mm_patch_merge_type == "spatial_unpad") { hparams.mm_patch_merge_type = PATCH_MERGE_SPATIAL_UNPAD; } } if (is_vision) { std::vector image_mean; std::vector image_std; get_arr_f32(KEY_IMAGE_MEAN, image_mean, false); get_arr_f32(KEY_IMAGE_STD , image_std, false); if (image_mean.size() < 3 || image_std.size() < 3) { throw std::runtime_error(mtmd_format("%s: image_mean/image_std arrays must have at least 3 elements, got %zu and %zu\n", __func__, image_mean.size(), image_std.size())); } for (int i = 0; i < 3; ++i) { hparams.image_mean[i] = image_mean[i]; hparams.image_std[i] = image_std[i]; } } // Load the vision/audio feature layer indices if they are explicitly provided // NOTE: gguf conversions should standardize the values of the vision feature layer to be non-negative, since we use -1 to mark values as unset here. get_arr_int(mtmd_format(KEY_FEATURE_LAYERS, prefix), hparams.feature_layers, false); for (const auto & v : hparams.feature_layers) { if (v > (int) hparams.n_layer) { throw std::runtime_error(mtmd_format("%s: feature layer index %d is out of range (n_layer: %d)", __func__, v, hparams.n_layer)); } } // model-specific params switch (model.proj_type) { case PROJECTOR_TYPE_MLP: case PROJECTOR_TYPE_MLP_NORM: case PROJECTOR_TYPE_LDP: case PROJECTOR_TYPE_LDPV2: case PROJECTOR_TYPE_COGVLM: { hparams.has_llava_projector = model.proj_type != PROJECTOR_TYPE_COGVLM; hparams.image_pad_color = {122, 116, 104}; if (!hparams.image_res_candidates.empty()) { hparams.image_resize_pad = PAD_CEIL; hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; } else { // llava-1.6 default params hparams.image_pad_ov = PAD_NONE; hparams.image_pad_rf = PAD_CEIL; hparams.image_pad_color_rf = {122, 116, 104}; } } break; case PROJECTOR_TYPE_GLM_EDGE: { hparams.image_resize_pad = PAD_CEIL; hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; } break; case PROJECTOR_TYPE_MINICPMV: { // use default llava-uhd preprocessing params if (hparams.minicpmv_version == 0) { hparams.minicpmv_version = 2; // default to 2 if not set } } break; case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: case PROJECTOR_TYPE_MINICPM_OCR_1_0: { // MiniCPM-V 4.6/4.7 unified merger projector // ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension hparams.n_merge = 4; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); GGML_ASSERT(hparams.n_merge == 2 || hparams.n_merge == 4); // no padding: the reference stretches the refined image to the target size hparams.image_pad_ov = PAD_NONE; hparams.image_pad_rf = PAD_NONE; // borrow wa_layer_indexes for vit_merger insertion point std::vector wa_layer_indexes_vec; get_arr_int(KEY_WIN_ATTN_LAYER_INDEXES, wa_layer_indexes_vec, false); if (!wa_layer_indexes_vec.empty()) { const int insert_lid = wa_layer_indexes_vec[0]; if (insert_lid < 0 || insert_lid >= (int) hparams.n_layer) { throw std::runtime_error(mtmd_format("%s: layer index %d is out of range (n_layer: %d)", __func__, insert_lid, hparams.n_layer)); } hparams.insert_layer_id = insert_lid; } } break; case PROJECTOR_TYPE_INTERNVL: { // use default llava-uhd preprocessing params // older version of internvl doesn't have min/max tiles, we need to provide default values for them to avoid issues hparams.preproc_min_tiles = 1; hparams.preproc_max_tiles = 12; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false); get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false); GGML_ASSERT(hparams.preproc_min_tiles <= hparams.preproc_max_tiles && hparams.preproc_max_tiles < INT32_MAX); set_internvl_dhr_res_candidates(model); } break; case PROJECTOR_TYPE_NEMOTRON_V2_VL: { get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); } break; case PROJECTOR_TYPE_NEMOTRON_OMNI: { hparams.ffn_op = FFN_GELU_ERF; log_ffn_op = "gelu_erf"; hparams.image_resize_pad = PAD_NONE; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); hparams.warmup_image_size = hparams.image_size; } break; case PROJECTOR_TYPE_PARAKEET: { get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor); GGML_ASSERT(hparams.subsampling_factor == 8 && "subsampling_factor must match the conv strides in mtmd_graph_parakeet::build()"); get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size); GGML_ASSERT(hparams.audio_conv_kernel_size > 0 && hparams.audio_conv_kernel_size % 2 == 1 && "audio_conv_kernel_size must be a positive odd integer"); hparams.audio_chunk_len = 0; hparams.audio_sample_rate = 16000; hparams.audio_n_fft = 512; hparams.audio_window_len = 400; hparams.audio_hop_len = 160; } break; case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: { get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor); get_u32(KEY_A_ROPE_DIMS, hparams.audio_rope_dims); get_u32(KEY_A_SPK_N_BLOCK, hparams.spk_n_layer); get_u32(KEY_A_SPK_N_HEAD, hparams.spk_n_head); if (hparams.subsampling_factor != 8) { throw std::runtime_error(mtmd_format("%s: unsupported subsampling factor %d\n", __func__, hparams.subsampling_factor)); } if (hparams.spk_n_layer <= 0 || hparams.spk_n_layer > (int) gguf_get_n_tensors(ctx_gguf.get())) { throw std::runtime_error(mtmd_format("%s: invalid speaker block count %d\n", __func__, hparams.spk_n_layer)); } hparams.audio_chunk_len = 0; hparams.audio_sample_rate = 16000; hparams.audio_n_fft = 512; hparams.audio_window_len = 400; hparams.audio_hop_len = 160; // one full window using nt = mtmd_graph_nemotron_transcribe; hparams.warmup_audio_size = (nt::window_left + nt::window_core + nt::window_right) * hparams.subsampling_factor; } break; case PROJECTOR_TYPE_IDEFICS3: { // use default llava-uhd preprocessing params hparams.image_resize_algo = RESIZE_ALGO_LANCZOS; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false); hparams.set_limit_image_tokens(); } break; case PROJECTOR_TYPE_COHERE2V: { hparams.image_pad_rf = PAD_NONE; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles); if (hparams.preproc_max_tiles <= 0 || hparams.preproc_max_tiles > 256) { throw std::runtime_error(mtmd_format("%s: preproc_max_tiles (%d) must be in range [1, 256]\n", __func__, hparams.preproc_max_tiles)); } } break; case PROJECTOR_TYPE_LFM2: case PROJECTOR_TYPE_D1OMNI_V: { // default for older GGUFs std::string resize_algo = "bilinear"; get_string(KEY_IMAGE_RESIZE_ALGO, resize_algo, false); // Lfm2VlImageProcessor and d1-omni vision.py resize bilinear and bicubic with torchvision if (resize_algo == "bilinear") { hparams.set_image_resize_algo(RESIZE_ALGO_BILINEAR, true); } else if (resize_algo == "bicubic") { hparams.set_image_resize_algo(RESIZE_ALGO_BICUBIC, true); } else if (resize_algo == "lanczos") { hparams.set_image_resize_algo(RESIZE_ALGO_LANCZOS); } else { throw std::runtime_error(mtmd_format("%s: unsupported image resize algo: %s\n", __func__, resize_algo.c_str())); } hparams.image_resize_algo_rf = hparams.image_resize_algo; hparams.image_resize_algo_ov = hparams.image_resize_algo; // the tiles stretch the image to the grid (Lfm2VlImageProcessor, d1-omni vision.py) hparams.image_pad_rf = PAD_NONE; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); // defaults for GGUFs converted before tiling params were written // ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json hparams.preproc_min_tiles = 2; hparams.preproc_max_tiles = 10; hparams.preproc_tile_size = 512; hparams.preproc_max_pixels_tolerance = 2.0f; get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false); get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false); get_u32(KEY_PREPROC_TILE_SIZE, hparams.preproc_tile_size, false); get_f32(KEY_PREPROC_MAX_PIXELS_TOLERANCE, hparams.preproc_max_pixels_tolerance, false); GGML_ASSERT(hparams.preproc_min_tiles >= 1 && hparams.preproc_min_tiles <= hparams.preproc_max_tiles); hparams.set_limit_image_tokens(64, 256); } break; case PROJECTOR_TYPE_PHI4: { hparams.n_merge = 1; hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); hparams.set_warmup_n_tokens(16*16); } break; case PROJECTOR_TYPE_PIXTRAL: { // ref: https://huggingface.co/mistral-community/pixtral-12b/blob/main/preprocessor_config.json // TODO: verify the image_min_tokens hparams.n_merge = 1; // the original pixtral does not use patch merging hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.rope_theta = 10000.0f; get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false); if (hparams.image_longest_edge > 0) { const int64_t patch_size = int64_t(hparams.patch_size) * hparams.n_merge; if (patch_size <= 0) { throw std::runtime_error("Pixtral merged patch size must be positive"); } const int64_t n_tokens_per_side = (hparams.image_longest_edge + patch_size - 1) / patch_size; if (n_tokens_per_side * patch_size > std::sqrt(std::numeric_limits::max())) { throw std::runtime_error("Pixtral preprocessing image size is too large"); } hparams.image_resize_pad = PAD_NONE; hparams.image_floor_before_align = true; hparams.set_limit_image_tokens(0, n_tokens_per_side * n_tokens_per_side); } else { hparams.set_limit_image_tokens(8, 1024); } hparams.set_warmup_n_tokens(256); // avoid OOM on warmup } break; case PROJECTOR_TYPE_LIGHTONOCR: { hparams.n_merge = 1; hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.rope_theta = 10000.0f; get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); hparams.image_longest_edge = hparams.image_size; get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false); hparams.set_limit_image_tokens(); hparams.set_warmup_n_tokens(256); // avoid OOM on warmup } break; case PROJECTOR_TYPE_DOTS_OCR: { hparams.rope_theta = 10000.0f; hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding hparams.image_round_half_even = true; // smart_resize uses Python round() get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup } break; case PROJECTOR_TYPE_DOTS3NOTE_V: { hparams.rope_theta = 10000.0f; hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge); get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); get_u32(KEY_VISION_N_EXPERT_USED, hparams.n_expert_used); hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup } break; case PROJECTOR_TYPE_DOTS3NOTE_A: { hparams.rope_theta = 10000.0f; hparams.audio_chunk_len = 60; // in seconds hparams.audio_sample_rate = 16000; hparams.audio_n_fft = 400; hparams.audio_window_len = 400; hparams.audio_hop_len = 160; } break; case PROJECTOR_TYPE_KIMIVL: { hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.rope_theta = 10000.0f; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); // TODO: check kimivl preprocessor for exact values hparams.set_limit_image_tokens(8, 1024); hparams.set_warmup_n_tokens(256); // avoid OOM on warmup } break; case PROJECTOR_TYPE_KIMIK25: case PROJECTOR_TYPE_KIMIK31: { hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.rope_theta = 10000.0f; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); int min_pixels = 0, max_pixels = 0; get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels, false); get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels, false); if (min_pixels > 0 && max_pixels > 0) { hparams.image_min_pixels = min_pixels; hparams.image_max_pixels = max_pixels; hparams.warmup_image_size = static_cast(std::sqrt(max_pixels)); } else { hparams.set_limit_image_tokens(2, 4096); } if (model.proj_type == PROJECTOR_TYPE_KIMIK31) { hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup } } break; case PROJECTOR_TYPE_DEEPSEEK4V: { hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.image_pad_color = {127, 127, 127}; hparams.rope_theta = 10000.0f; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); hparams.dsv4_max_n_token = 384; hparams.dsv4_max_wh_ratio = 8; const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; // handle min/max token counts from CLI if (hparams.custom_image_min_tokens > 0) { hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area; } if (hparams.custom_image_max_tokens > 0) { // the cap is on the whole token block, keep some room for the resize solver hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16); } hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area; // a small custom max token count also lowers the min-pixel upscale threshold hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels); // avoid OOM on warmup const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token)); hparams.set_warmup_n_tokens(warmup_side * warmup_side); } break; case PROJECTOR_TYPE_DEEPSEEK41V: { // ref: load_image in inference/image_processor.py (V4.1), PIL ImageOps.pad on gray 127 hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.image_pad_color = {127, 127, 127}; hparams.rope_theta = 10000.0f; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); // vision_max_n_token = 1024, vision_max_wh_ratio = null (converter asserts both) hparams.dsv4_max_n_token = 1024; hparams.dsv4_max_wh_ratio = 0; const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; if (hparams.custom_image_min_tokens > 0) { hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area; } if (hparams.custom_image_max_tokens > 0) { // the cap is on the whole token block (newlines + start/end included) hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16); } hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area; hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels); const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token)); hparams.set_warmup_n_tokens(warmup_side * warmup_side); } break; case PROJECTOR_TYPE_GEMMA3: { // default value (used by all model sizes in gemma 3 family) // number of patches for each **side** is reduced by a factor of 4 hparams.n_merge = 4; hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; // test model (tinygemma3) has a different value, we optionally read it get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); } break; case PROJECTOR_TYPE_GEMMA4V: case PROJECTOR_TYPE_GEMMA4UV: { hparams.rope_theta = 100.0f; hparams.ffn_op = FFN_GELU; log_ffn_op = "gelu"; hparams.n_merge = 3; // pooling_kernel_size hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.image_resize_pad = PAD_NONE; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); if (model.proj_type == PROJECTOR_TYPE_GEMMA4UV) { // for "unified" variant, we directly use a bigger patch size, because the "token merging" is done directly on conv layer hparams.patch_size = hparams.patch_size * hparams.n_merge; hparams.n_merge = 1; } hparams.set_limit_image_tokens(70, 1120); hparams.set_warmup_n_tokens(256); // avoid OOM on warmup } break; case PROJECTOR_TYPE_GEMMA3NV: { // Gemma3n uses MobileNetV5 which produces 256 tokens (16x16) // Similar configuration to Gemma3 hparams.n_merge = 1; // MobileNetV5 handles resizing internally hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); } break; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: case PROJECTOR_TYPE_LING3VL: { hparams.n_merge = 2; // default value for Qwen 2 and 2.5 hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding hparams.image_round_half_even = true; // smart_resize uses Python round() get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); if (hparams.n_merge != 2) { // the temporal merge input and the merger use a 2x2 patch grid throw std::runtime_error(mtmd_format("%s: spatial_merge_size (%d) must be 2\n", __func__, hparams.n_merge)); } get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, model.proj_type == PROJECTOR_TYPE_QWEN25VL); // only 2.5 requires it // ref: https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct/blob/main/preprocessor_config.json hparams.set_limit_image_tokens(8, 4096); // optional limits of the model, the custom values take precedence if (hparams.custom_image_min_tokens <= 0) { get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels, false); } if (hparams.custom_image_max_tokens <= 0) { get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels, false); } hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup const int warn_min_pixels = 1024 * hparams.n_merge * hparams.n_merge * hparams.patch_size * hparams.patch_size; if (hparams.image_min_pixels < warn_min_pixels) { LOG_WRN("%s: Qwen-VL models require at minimum 1024 image tokens to function correctly on grounding tasks\n", __func__); LOG_WRN("%s: if you encounter problems with accuracy, try adding --image-min-tokens 1024\n", __func__); LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__); } } break; case PROJECTOR_TYPE_MAGEVL: { hparams.ffn_op = FFN_GELU_ERF; log_ffn_op = "gelu_erf"; hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding hparams.image_round_half_even = true; // smart_resize uses Python round() get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge); get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup } break; case PROJECTOR_TYPE_SENSENOVA_U1: { hparams.n_merge = 2; hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); hparams.rope_theta = 10000.0f; hparams.set_limit_image_tokens(16, 4096); if (hparams.n_merge != 2) { throw std::runtime_error(mtmd_format("%s: SenseNova U1 requires spatial_merge_size=2, got %u", __func__, hparams.n_merge)); } hparams.set_warmup_n_tokens(16*16); } break; case PROJECTOR_TYPE_LFM3VL: { std::string resize_algo; get_string(KEY_IMAGE_RESIZE_ALGO, resize_algo); // Lfm3VlImageProcessor resizes with torchvision if (resize_algo == "bilinear") { hparams.set_image_resize_algo(RESIZE_ALGO_BILINEAR, true); } else if (resize_algo == "bicubic") { hparams.set_image_resize_algo(RESIZE_ALGO_BICUBIC, true); } else { throw std::runtime_error(mtmd_format("%s: unsupported image resize algo: %s\n", __func__, resize_algo.c_str())); } // smart_resize stretches to the aligned size, no letterbox hparams.image_resize_pad = PAD_NONE; get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge); // token counts and the projector graph assume a 2x2 merge if (hparams.n_merge != 2) { throw std::runtime_error(mtmd_format("%s: unsupported spatial_merge_size %d for lfm3vl, only 2 is supported\n", __func__, hparams.n_merge)); } get_u32(mtmd_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv); hparams.rope_theta = 1000000.0f; // vision_rope_theta // HF processor size; the max is also capped at 4096 tokens as an OOM guard int min_pixels = 0; int max_pixels = 0; get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels); get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels); if (min_pixels <= 0 || max_pixels < min_pixels) { throw std::runtime_error(mtmd_format("%s: invalid image min/max pixels %d/%d\n", __func__, min_pixels, max_pixels)); } const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; hparams.set_limit_image_tokens(min_pixels / patch_area, std::min(4096, max_pixels / patch_area)); hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup } break; case PROJECTOR_TYPE_MINIMAX_M3: { hparams.n_merge = 2; // spatial_merge_size hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.image_resize_pad = PAD_NONE; get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); // n_merge is used as a divisor in mtmd_subcontext::encode() // (gh / n_merge); reject 0 to avoid int div-by-zero (DoS). GGML_ASSERT(hparams.n_merge > 0); hparams.rope_theta = 10000.0f; // vision_config.rope_theta // MiniMax-M3: max_pixels 451584 (=672^2) -> 576 merged tokens (image_seq_length) hparams.set_limit_image_tokens(8, 576); hparams.set_warmup_n_tokens(16*16); } break; case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: { hparams.audio_sample_rate = 24000; // mono waveform, no frontend get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size, true); get_arr_int(KEY_A_HIDDEN_SIZES, hparams.audio_hidden_sizes, true); get_arr_int(KEY_A_PATCH_SIZES, hparams.audio_patch_sizes, true); get_arr_int(KEY_A_DEPTHS, hparams.audio_depths, true); // the graph indexes the three arrays by stage and model.layers by the running depth sum const size_t n_stage = hparams.audio_patch_sizes.size(); GGML_ASSERT(n_stage > 0); GGML_ASSERT(hparams.audio_hidden_sizes.size() == n_stage); GGML_ASSERT(hparams.audio_depths.size() == n_stage); GGML_ASSERT(hparams.n_mel_bins == 1); GGML_ASSERT(hparams.audio_conv_kernel_size >= 1); int64_t n_blocks = 0; for (size_t i = 0; i < n_stage; ++i) { GGML_ASSERT(hparams.audio_hidden_sizes[i] >= 1); GGML_ASSERT(hparams.audio_depths[i] >= 0); n_blocks += hparams.audio_depths[i]; } if (n_blocks != hparams.n_layer) { throw std::runtime_error(mtmd_format("%s: sum of audio depths (%" PRId64 ") does not match n_layer (%d)\n", __func__, n_blocks, hparams.n_layer)); } hparams.warmup_audio_size = hparams.audio_samples_per_token(); } break; case PROJECTOR_TYPE_VIBEVOICE_ASR: { hparams.audio_sample_rate = 24000; // mono waveform, no frontend hparams.audio_chunk_len = 30; // in seconds, any length works as the convs are causal get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size, true); get_arr_int(KEY_A_PATCH_SIZES, hparams.audio_patch_sizes, true); get_arr_int(KEY_A_DEPTHS, hparams.audio_depths, true); // bounds keep the receptive field below from overflowing const size_t n_stage = hparams.audio_patch_sizes.size(); if (n_stage == 0 || n_stage > 16 || hparams.audio_depths.size() != n_stage || hparams.n_mel_bins != 1 || hparams.audio_conv_kernel_size < 1 || hparams.audio_conv_kernel_size > 64) { throw std::runtime_error(mtmd_format("%s: invalid vibevoice audio hparams\n", __func__)); } // receptive field: stem conv (kernel), downsample convs (2 * stride), blocks (kernel), head conv (kernel) const int64_t k = hparams.audio_conv_kernel_size; int64_t hop = 1; int64_t rf = 0; int64_t n_blocks = 0; for (size_t i = 0; i < n_stage; ++i) { const int64_t stride = hparams.audio_patch_sizes[i]; if (stride < 1 || stride > 64 || hop * stride > (1 << 20) || hparams.audio_depths[i] < 0 || hparams.audio_depths[i] > 64) { throw std::runtime_error(mtmd_format("%s: invalid vibevoice audio hparams\n", __func__)); } rf += ((i == 0 ? k : 2 * stride) - 1) * hop; hop *= stride; rf += hparams.audio_depths[i] * (k - 1) * hop; n_blocks += hparams.audio_depths[i]; } rf += (k - 1) * hop; if (rf > 60 * hparams.audio_sample_rate) { throw std::runtime_error(mtmd_format("%s: vibevoice receptive field of %" PRId64 " samples is too long\n", __func__, rf)); } if (n_blocks != hparams.n_layer) { throw std::runtime_error(mtmd_format("%s: sum of audio depths (%" PRId64 ") does not match n_layer (%d)\n", __func__, n_blocks, hparams.n_layer)); } hparams.audio_n_ctx = (int32_t) ((rf + hop - 1) / hop * hop); hparams.warmup_audio_size = (int32_t) ((int64_t) hparams.audio_chunk_len * hparams.audio_sample_rate / hop * hop) + hparams.audio_n_ctx; } break; case PROJECTOR_TYPE_INKLING2V: { hparams.ffn_op = FFN_GELU_ERF; log_ffn_op = "gelu_erf"; get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge); hparams.rope_theta = 10000.0f; hparams.set_limit_image_tokens(1, 4096); hparams.set_warmup_n_tokens(32*32); // avoid OOM on warmup } break; case PROJECTOR_TYPE_MUSE_GLIMMER: { hparams.n_merge = 2; // pixel-shuffle downsample after the ViT hparams.image_resize_algo = RESIZE_ALGO_LANCZOS; hparams.rope_theta = 10000.0f; hparams.muse_glimmer_patch_temporal = 2; hparams.muse_glimmer_sparse_factor = 4; // 3 sparse layers + 1 global, repeating get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); hparams.set_limit_image_tokens(1, 4096); hparams.set_warmup_n_tokens(32*32); } break; case PROJECTOR_TYPE_MIMOVL: { hparams.n_merge = 2; // spatial_merge_size hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding hparams.image_round_half_even = true; // smart_resize uses Python round() get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); get_u32(mtmd_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv); // 1D banded sliding-window radius (visual_token_window_size); required get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size); std::vector pat; get_arr_int(KEY_WA_PATTERN_MODE, pat, true); GGML_ASSERT((int) pat.size() == hparams.n_layer && "mimovl wa_pattern_mode length must equal n_layer"); hparams.wa_pattern_mode.assign(pat.begin(), pat.end()); get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup } break; case PROJECTOR_TYPE_STEP3VL: { hparams.n_merge = 4; // two stride-2 downsamplers after patching get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); hparams.rope_theta = 10000.0f; get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false); if (hparams.image_longest_edge == 0) { hparams.image_longest_edge = 3024; } // note: the step3vl preprocessor slices based on a fixed window grid, so it does not support custom min/max image tokens hparams.warmup_image_size = hparams.image_size; } break; case PROJECTOR_TYPE_YOUTUVL: { hparams.n_merge = 2; hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; hparams.image_resize_pad = PAD_NONE; get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true); std::vector wa_layer_indexes_vec; get_arr_int(KEY_WIN_ATTN_LAYER_INDEXES, wa_layer_indexes_vec, true); for (auto & layer : wa_layer_indexes_vec) { hparams.wa_layer_indexes.insert(layer); } // support max_height * max_width = 8000 * 8000. 8000/16/2 = 250 image tokens hparams.set_limit_image_tokens(1, 62500); hparams.set_warmup_n_tokens(16*16); // avoid OOM on warmup } break; case PROJECTOR_TYPE_YASA2: { hparams.ffn_op = FFN_GELU_ERF; log_ffn_op = "gelu_erf"; hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; // reka model performs better when the image is stretched to fit // fixed square size (no padding) hparams.image_resize_pad = PAD_NONE; } break; case PROJECTOR_TYPE_GLM4V: { hparams.rope_theta = 10000.0f; hparams.n_merge = 2; // default value for GLM4-V hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); hparams.set_limit_image_tokens(8, 4096); hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup } break; case PROJECTOR_TYPE_GLM5V: { // glm4v tower with clamped SwiGLU, ceil-aligned resize and its own token budget hparams.rope_theta = 10000.0f; hparams.n_merge = 2; hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); get_f32(KEY_SWIGLU_CLAMP, hparams.swiglu_clamp, true); get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); hparams.set_limit_image_tokens(); hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup } break; case PROJECTOR_TYPE_LLAMA4: { hparams.rope_theta = 10000.0f; hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); set_llava_uhd_res_candidates(model, 3); } break; case PROJECTOR_TYPE_ULTRAVOX: case PROJECTOR_TYPE_QWEN2A: case PROJECTOR_TYPE_HIGGS_AUDIO: case PROJECTOR_TYPE_QWEN3A: case PROJECTOR_TYPE_GLMA: case PROJECTOR_TYPE_VOXTRAL: case PROJECTOR_TYPE_MERALION: case PROJECTOR_TYPE_MUSIC_FLAMINGO: { bool require_stack = model.proj_type == PROJECTOR_TYPE_ULTRAVOX || model.proj_type == PROJECTOR_TYPE_VOXTRAL || model.proj_type == PROJECTOR_TYPE_MERALION || model.proj_type == PROJECTOR_TYPE_GLMA; get_u32(KEY_A_PROJ_STACK_FACTOR, hparams.proj_stack_factor, require_stack); hparams.ffn_op = FFN_GELU_ERF; log_ffn_op = "gelu_erf"; // temporary solution for logging // audio preprocessing params hparams.audio_chunk_len = 30; // in seconds hparams.audio_sample_rate = 16000; hparams.audio_n_fft = 400; hparams.audio_window_len = 400; hparams.audio_hop_len = 160; if (model.proj_type == PROJECTOR_TYPE_HIGGS_AUDIO) { // each 4s chunk is encoded on its own hparams.audio_chunk_len = 4; hparams.warmup_audio_size = 400; } } break; case PROJECTOR_TYPE_VOXTRAL_RT: { get_u32(KEY_A_PROJ_STACK_FACTOR, hparams.proj_stack_factor); get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size); hparams.rope_theta = 1000000.0f; // params.json hparams.ffn_op = FFN_SILU; log_ffn_op = "silu"; // audio preprocessing params hparams.audio_sample_rate = 16000; hparams.audio_n_fft = 400; hparams.audio_window_len = 400; hparams.audio_hop_len = 160; // 2 context frames + 8 s of new frames, see mtmd_subcontext::enc_state_init() hparams.warmup_audio_size = 2 + 2*400; } break; case PROJECTOR_TYPE_MIMO_AUDIO: { get_u32(KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers, false); get_arr_int(KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size, false); if (hparams.rvq_num_quantizers <= 0) { throw std::runtime_error(mtmd_format("%s: mimo_audio: missing %s\n", __func__, KEY_A_RVQ_NUM_QUANTIZERS)); } if ((int) hparams.rvq_codebook_size.size() != hparams.rvq_num_quantizers) { throw std::runtime_error(mtmd_format( "%s: mimo_audio: %s length (%zu) must equal %s (%d)\n", __func__, KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size.size(), KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers)); } hparams.ffn_op = FFN_GELU_ERF; // PyTorch F.gelu default (approximate="none") hparams.rope_theta = 10000.0f; // audio preprocessing params (mel spectrogram) hparams.audio_sample_rate = 24000; hparams.audio_n_fft = 960; hparams.audio_window_len = 960; hparams.audio_hop_len = 240; get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size); std::vector wa_pattern; get_arr_int(KEY_A_WA_PATTERN_MODE, wa_pattern, true); if ((int) wa_pattern.size() != hparams.n_layer) { throw std::runtime_error(mtmd_format( "%s: mimo_audio: %s length (%zu) must equal n_layer (%d)\n", __func__, KEY_A_WA_PATTERN_MODE, wa_pattern.size(), hparams.n_layer)); } hparams.wa_pattern_mode.assign(wa_pattern.begin(), wa_pattern.end()); get_u32(KEY_A_LOCAL_BLOCK_COUNT, hparams.audio_local_n_layer); get_u32(KEY_A_LOCAL_GROUP_SIZE, hparams.audio_local_group_size); if (hparams.audio_local_group_size <= 0) { throw std::runtime_error(mtmd_format( "%s: mimo_audio: %s must be > 0\n", __func__, KEY_A_LOCAL_GROUP_SIZE)); } } break; case PROJECTOR_TYPE_ZONOS3_SPKENC: { // kaldi fbank front-end hparams.audio_sample_rate = 16000; hparams.audio_n_fft = 512; hparams.audio_window_len = 400; hparams.audio_hop_len = 160; hparams.warmup_audio_size = 1000; } break; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: { // ECAPA-TDNN speaker encoder, mel front-end uses the Slaney default (fmin=0, fmax=sr/2) hparams.audio_sample_rate = 24000; hparams.audio_n_fft = 1024; hparams.audio_window_len = 1024; hparams.audio_hop_len = 256; } break; case PROJECTOR_TYPE_CHATTERBOX_SPKENC: { // kaldi fbank front-end of CAMPPlus: 25 ms / 10 ms framing, 512-point spectrum hparams.audio_sample_rate = 16000; hparams.audio_n_fft = 512; hparams.audio_window_len = 400; hparams.audio_hop_len = 160; } break; case PROJECTOR_TYPE_LFM3A_GEN: { // TODO: CUDA has no flash attention kernel for the depthformer head dim 32, enable it once it does model.flash_attn_auto = false; // fixed by LiquidCodecConfig, head counts come from the tensors hparams.rope_theta = 1000000.0f; hparams.wav_tfm_n_layer = 8; hparams.wav_tfm_n_embd = 512; hparams.wav_tfm_eps = 1e-5f; hparams.wav_tfm_rope_theta = 1000000.0f; hparams.wav_tfm_swa = 30; // decoder_window hparams.wav_frame_stack = 6; // stft_frame_stack, STFT frames per codec frame // the ISTFT runs on the host, see mtmd_gen_audio_process() hparams.audio_sample_rate = 24000; hparams.audio_n_fft = 1280; hparams.audio_window_len = 1280; hparams.audio_hop_len = 320; // speech-code sampling, the converter writes the model card values; else the LFM3-Omni card hparams.gen_temp = 0.4f; hparams.gen_top_k = 0; hparams.gen_penalty_repeat = 1.2f; hparams.gen_penalty_last_n = 64; get_f32(KEY_GEN_AUDIO_TEMP, hparams.gen_temp, false); get_i32(KEY_GEN_AUDIO_TOP_K, hparams.gen_top_k, false); get_f32(KEY_GEN_AUDIO_PENALTY_REPEAT, hparams.gen_penalty_repeat, false); get_i32(KEY_GEN_AUDIO_PENALTY_LAST_N, hparams.gen_penalty_last_n, false); } break; case PROJECTOR_TYPE_QWEN3TTS_GEN: { // TODO: hardcoded for now, read from code_predictor_config instead hparams.rope_theta = 1000000.0f; // code2wav params hparams.wav_tfm_n_layer = 8; hparams.wav_tfm_n_embd = 512; hparams.wav_tfm_n_ff = 1024; hparams.wav_tfm_n_head = 16; hparams.wav_tfm_n_head_kv = 16; hparams.wav_tfm_eps = 1e-5f; hparams.wav_tfm_rope_theta = 10000.0f; hparams.wav_upsample_n_block = 2; hparams.wav_dac_n_block = 4; hparams.wav_dac_n_res = 3; // matches the reference decoder's sliding_window (speech_tokenizer/config.json) hparams.wav_tfm_swa = 72; } break; case PROJECTOR_TYPE_ZONOS3_GEN: { hparams.audio_sample_rate = 44100; } break; case PROJECTOR_TYPE_DRAMA3_GEN: { // Fish DAC decoder, codec.pth has no config hparams.audio_sample_rate = 44100; hparams.wav_tfm_n_layer = 8; hparams.wav_tfm_n_embd = 1024; hparams.wav_tfm_n_ff = 3072; hparams.wav_tfm_n_head = 16; hparams.wav_tfm_n_head_kv = 16; hparams.wav_tfm_eps = 1e-5f; hparams.wav_tfm_rope_theta = 10000.0f; hparams.wav_tfm_swa = 128; hparams.wav_upsample_n_block = 2; hparams.wav_dac_n_block = 4; hparams.wav_dac_n_res = 3; } break; case PROJECTOR_TYPE_POCKETTTS_SPKENC: case PROJECTOR_TYPE_POCKETTTS_GEN: { // mimi front-end takes the raw waveform, no mel hparams.audio_sample_rate = 24000; // seanet ratios are [6,5,4] in the config, the encoder reverses them hparams.seanet_ratios = { 4, 5, 6 }; hparams.seanet_n_stage = (int32_t) hparams.seanet_ratios.size(); hparams.mimi_downsample = 16; // matches the reference transformer's "context" hparams.mimi_tfm_context = 250; hparams.rope_theta = 10000.0f; // flow_lm defaults, see pocket_tts/default_parameters.py hparams.flow_n_step = 1; hparams.gen_eos_threshold = -4.0f; } break; case PROJECTOR_TYPE_CHATTERBOX: { // the classic cfm solver unrolls up to 10 steps x 2 cfg estimator evaluations in one graph model.max_nodes = 65536; // s3gen output rate hparams.audio_sample_rate = 24000; } break; case PROJECTOR_TYPE_SOPRANO: { // SopranoDecoder defaults, n_fft comes from the head tensor hparams.audio_sample_rate = 32000; hparams.audio_hop_len = 512; } break; case PROJECTOR_TYPE_PADDLEOCR: { hparams.n_merge = 2; hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); hparams.set_warmup_n_tokens(28*28); // avoid OOM on warmup } break; case PROJECTOR_TYPE_DEEPSEEKOCR: case PROJECTOR_TYPE_DEEPSEEKOCR2: { hparams.patch_size = 16; hparams.image_size = 1024; hparams.warmup_image_size = 1024; hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.image_pad_color = {127, 127, 127}; get_u32(KEY_SAM_N_BLOCK, hparams.sam_n_layer, true); get_u32(KEY_SAM_N_HEAD, hparams.sam_n_head, true); get_u32(KEY_SAM_N_EMBD, hparams.sam_n_embd, true); get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true); hparams.preproc_min_tiles = 2; if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR) { hparams.preproc_max_tiles = 9; hparams.preproc_tile_size = 640; // the CLIP/ViT body runs its layernorms at 1e-5 (the SAM stage uses 1e-6) hparams.eps = 1e-5f; } if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2) { hparams.preproc_max_tiles = 6; hparams.preproc_tile_size = 768; // qwen2 encoder is GQA, requires KEY_N_HEAD_KV get_u32(mtmd_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv); } // unlimited-ocr shares the v1 projector but tiles up to 32 get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false); get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false); GGML_ASSERT(hparams.preproc_min_tiles >= 0 && hparams.preproc_min_tiles <= hparams.preproc_max_tiles && hparams.preproc_max_tiles <= 256); } break; case PROJECTOR_TYPE_HUNYUANVL: { hparams.n_merge = 2; hparams.image_resize_algo = RESIZE_ALGO_LANCZOS; hparams.image_resize_pad = PAD_NONE; hparams.ffn_op = FFN_GELU; hparams.set_limit_image_tokens(256, 16384); get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels, false); get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels, false); hparams.set_warmup_n_tokens(32*32); } break; case PROJECTOR_TYPE_LFM2A: case PROJECTOR_TYPE_D1OMNI_A: { // audio preprocessing params hparams.audio_chunk_len = 1; // in seconds hparams.audio_sample_rate = 16000; hparams.audio_n_fft = 512; hparams.audio_window_len = 400; hparams.audio_hop_len = 160; } break; case PROJECTOR_TYPE_LFM3A: { get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size); if (hparams.attn_window_size <= 0) { throw std::runtime_error(mtmd_format("%s: invalid audio attention window %d\n", __func__, hparams.attn_window_size)); } hparams.rope_theta = 1000000.0f; // checked by the converter // audio preprocessing params, see mtmd_audio_preprocessor_lfm3a hparams.audio_chunk_len = 0; hparams.audio_sample_rate = 16000; hparams.audio_n_fft = 400; hparams.audio_window_len = 400; hparams.audio_hop_len = 160; } break; case PROJECTOR_TYPE_EXAONE4_5: { hparams.n_merge = 2; hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding hparams.image_round_half_even = true; // smart_resize uses Python round() get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, false); get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); hparams.set_warmup_n_tokens(46 * 46); if (hparams.rope_theta <= 0.0f) { hparams.rope_theta = 10000.0f; } get_u32(mtmd_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv); } break; case PROJECTOR_TYPE_GEMMA4A: { // Gemma4 feature_extraction_gemma4.py: // frame_length_ms=20 -> 320 samples, n_fft=512, hop=10ms -> 160 hparams.audio_chunk_len = 0; // no fixed-length padding hparams.audio_sample_rate = 16000; hparams.audio_n_fft = 512; hparams.audio_window_len = 320; // 20ms frame (NOT 25ms/400) hparams.audio_hop_len = 160; // due to a mistake in the original conversion code, rms_norm_eps is set to a wrong value // since all gemma4a models use 1e-6, we just hardcode it here to avoid re-conversion hparams.eps = 1e-6f; } break; case PROJECTOR_TYPE_GEMMA4UA: { // Encoder-free: raw 16 kHz waveform chunked into 640-sample frames. hparams.audio_chunk_len = 0; hparams.audio_sample_rate = 16000; hparams.eps = 1e-6f; hparams.n_mel_bins = 640; } break; case PROJECTOR_TYPE_GRANITE_SPEECH: { hparams.audio_chunk_len = 0; hparams.audio_sample_rate = 16000; hparams.audio_n_fft = 512; hparams.audio_window_len = 400; hparams.audio_hop_len = 160; get_u32(KEY_A_CHUNK_SIZE, hparams.audio_chunk_size); // context_size is squared for the attn_dists/mask buffers; cap to prevent int32 overflow // (legitimate values are small, e.g. 12-200; 8192^2 = 67M still fits int32) GGML_ASSERT(hparams.audio_chunk_size > 0 && hparams.audio_chunk_size <= 8192); get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size); get_u32(KEY_A_MAX_POS_EMB, hparams.audio_max_pos_emb); get_u32(KEY_A_PROJ_WINDOW_SIZE, hparams.audio_proj_window_size); get_u32(KEY_A_PROJ_DOWNSAMPLE_RATE, hparams.audio_proj_downsample_rate); get_u32(KEY_A_PROJ_HEAD_COUNT, hparams.audio_proj_head_count); // NOTE: feature layers loaded above in common path } break; case PROJECTOR_TYPE_JANUS_PRO: { hparams.image_pad_color = {127, 127, 127}; hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; } break; case PROJECTOR_TYPE_GRANITE4_VISION: { // SigLIP tower. hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.image_resize_pad = PAD_CEIL; // NOTE: feature_layers loaded in common path as optional get_arr_int(KEY_PROJ_SPATIAL_OFFSETS, hparams.proj_spatial_offsets); if (hparams.feature_layers.size() != hparams.proj_spatial_offsets.size()) { throw std::runtime_error(mtmd_format("%s: feature_layers.size() %d != proj_spatial_offsets.size() %d", hparams.feature_layers.size(), hparams.proj_spatial_offsets.size())); } get_u32(KEY_PROJ_SAMPLE_QUERY_SIDE, hparams.downsample_query_side); get_u32(KEY_PROJ_SAMPLE_WINDOW_SIDE, hparams.downsample_window_side); hparams.warmup_image_size = hparams.image_size; } break; default: throw std::runtime_error(mtmd_format("%s: unknown vision projector type %s\n", __func__, proj_type.c_str())); } // sanity check { if (hparams.image_size < 0) { // note: some models having hparams.image_size == 0, which means the image size is dynamic throw std::runtime_error(mtmd_format("%s: image_size (%d) cannot be negative\n", __func__, hparams.image_size)); } if (hparams.image_size > 8192) { // cap prevents int32 overflow in n_patches = (image_size/patch_size)^2 throw std::runtime_error(mtmd_format("%s: image_size (%d) is too large (max 8192)\n", __func__, hparams.image_size)); } if (hparams.patch_size <= 0 || hparams.patch_size >= 65536) { throw std::runtime_error(mtmd_format("%s: patch_size (%d) must be positive and less than 65536\n", __func__, hparams.patch_size)); } if (hparams.n_embd <= 0) { throw std::runtime_error(mtmd_format("%s: n_embd (%d) must be greater than 0\n", __func__, hparams.n_embd)); } if (hparams.image_max_pixels < hparams.image_min_pixels) { throw std::runtime_error(mtmd_format("%s: image_max_pixels (%d) is less than image_min_pixels (%d)\n", __func__, hparams.image_max_pixels, hparams.image_min_pixels)); } if (hparams.n_merge <= 0 || hparams.n_merge >= 65536) { throw std::runtime_error(mtmd_format("%s: n_merge (%d) must be greater than 0 and less than 65536\n", __func__, hparams.n_merge)); } if (hparams.attn_window_size > 4096) { throw std::runtime_error(mtmd_format("%s: attn_window_size (%d) is too large (max 4096)\n", __func__, hparams.attn_window_size)); } } LOG_INF("%s: projector: %s\n", __func__, proj_type.c_str()); LOG_INF("%s: n_embd: %d\n", __func__, hparams.n_embd); LOG_INF("%s: n_head: %d\n", __func__, hparams.n_head); LOG_INF("%s: n_ff: %d\n", __func__, hparams.n_ff); LOG_INF("%s: n_layer: %d\n", __func__, hparams.n_layer); LOG_INF("%s: ffn_op: %s\n", __func__, log_ffn_op.c_str()); LOG_INF("%s: projection_dim: %d\n", __func__, hparams.projection_dim); if (is_vision) { LOG_INF("\n--- vision hparams ---\n"); LOG_INF("%s: image_size: %d\n", __func__, hparams.image_size); LOG_INF("%s: patch_size: %d\n", __func__, hparams.patch_size); LOG_INF("%s: has_llava_proj: %d\n", __func__, hparams.has_llava_projector); LOG_INF("%s: minicpmv_version: %d\n", __func__, hparams.minicpmv_version); LOG_INF("%s: n_merge: %d\n", __func__, hparams.n_merge); LOG_INF("%s: n_wa_pattern: %d\n", __func__, hparams.n_wa_pattern); if (!hparams.wa_layer_indexes.empty()) { LOG_INF("%s: wa_layer_indexes: ", __func__); for (auto & layer : hparams.wa_layer_indexes) { LOG_INF("%d ", layer); } LOG_INF("\n"); } if (hparams.image_min_pixels > 0) { LOG_INF("%s: image_min_pixels: %d%s\n", __func__, hparams.image_min_pixels, hparams.custom_image_min_tokens > 0 ? " (custom value)" : ""); } if (hparams.image_max_pixels > 0) { LOG_INF("%s: image_max_pixels: %d%s\n", __func__, hparams.image_max_pixels, hparams.custom_image_max_tokens > 0 ? " (custom value)" : ""); } if (hparams.preproc_max_tiles > 0) { LOG_INF("%s: preproc_tiles: %d - %d\n", __func__, hparams.preproc_min_tiles, hparams.preproc_max_tiles); } } else if (is_audio) { LOG_INF("\n--- audio hparams ---\n"); LOG_INF("%s: n_mel_bins: %d\n", __func__, hparams.n_mel_bins); LOG_INF("%s: proj_stack_factor: %d\n", __func__, hparams.proj_stack_factor); LOG_INF("%s: audio_chunk_len: %d\n", __func__, hparams.audio_chunk_len); LOG_INF("%s: audio_sample_rate: %d\n", __func__, hparams.audio_sample_rate); LOG_INF("%s: audio_n_fft: %d\n", __func__, hparams.audio_n_fft); LOG_INF("%s: audio_window_len: %d\n", __func__, hparams.audio_window_len); LOG_INF("%s: audio_hop_len: %d\n", __func__, hparams.audio_hop_len); // GEMMA4UA is encoder-free: it uses n_mel_bins as a raw-waveform frame size (640) and has no FFT/filterbank, so the mel-range and FFT // checks below do not apply to it. // pocket-tts is encoder-free in the same sense: mimi convolves the raw waveform // Muse Spark is encoder-free in the same sense: its conv pyramid reads the raw waveform, one channel takes the place of the mel bins const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA && model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO && model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR && model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC; // Validate audio hparams loaded from GGUF metadata if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) { throw std::runtime_error(mtmd_format("%s: n_mel_bins (%d) must be in range [1, 256]\n", __func__, hparams.n_mel_bins)); } if (fft_based && (hparams.audio_sample_rate <= 0 || hparams.audio_n_fft <= 0 || hparams.audio_hop_len <= 0 || hparams.audio_window_len <= 0)) { throw std::runtime_error(mtmd_format("%s: audio hparams invalid: sample_rate=%d n_fft=%d window_len=%d hop_len=%d\n", __func__, hparams.audio_sample_rate, hparams.audio_n_fft, hparams.audio_window_len, hparams.audio_hop_len)); } } LOG_INF("\n"); LOG_INF("%s: model size: %.2f MiB\n", __func__, model_size / 1024.0 / 1024.0); LOG_INF("%s: metadata size: %.2f MiB\n", __func__, ggml_get_mem_size(ctx_meta.get()) / 1024.0 / 1024.0); } } void load_tensors(mtmd_model & model, ggml_backend_buffer_type_t buft, bool no_alloc) { auto & hparams = model.hparams; std::map tensor_offset; std::vector tensors_to_load; auto fin = open_ifstream_binary(fname); if (!fin) { throw std::runtime_error(mtmd_format("%s: failed to open %s\n", __func__, fname.c_str())); } // TODO @ngxson : support both audio and video in the future const char * prefix = model.modality == MTMD_MODALITY_AUDIO ? "a" : model.modality == MTMD_MODALITY_GEN_AUDIO ? "a.gen.code" : "v"; // get offsets for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { const char * name = gguf_get_tensor_name(ctx_gguf.get(), i); tensor_offset[name] = gguf_get_data_offset(ctx_gguf.get()) + gguf_get_tensor_offset(ctx_gguf.get(), i); } // create data context struct ggml_init_params params = { /*.mem_size =*/ static_cast(gguf_get_n_tensors(ctx_gguf.get()) + 1) * ggml_tensor_overhead(), /*.mem_buffer =*/ NULL, /*.no_alloc =*/ true, }; model.ctx_data.reset(ggml_init(params)); if (!model.ctx_data) { throw std::runtime_error(mtmd_format("%s: failed to init ggml context\n", __func__)); } // helper function std::unordered_set loaded_tensor_names; auto get_tensor = [&](const std::string & name, bool required = true) { // Each tensor should only be loaded once; duplicates indicate a bug if (loaded_tensor_names.count(name)) { throw std::runtime_error(mtmd_format("%s: tensor already loaded: %s\n", __func__, name.c_str())); } ggml_tensor * cur = ggml_get_tensor(ctx_meta.get(), name.c_str()); if (!cur && required) { throw std::runtime_error(mtmd_format("%s: unable to find tensor %s\n", __func__, name.c_str())); } if (cur) { tensors_to_load.push_back(cur); ggml_tensor * data_tensor = ggml_dup_tensor(model.ctx_data.get(), cur); ggml_set_name(data_tensor, cur->name); loaded_tensor_names.insert(name); cur = data_tensor; // add to weight memory counter model.mem_weights[ggml_backend_buft_get_device(buft)] += ggml_nbytes(cur); } return cur; }; // pocket-tts: the encoder and the decoder share the same layout, only the prefix differs auto load_seanet = [&](mtmd_seanet & seanet, bool is_decoder) { const char * conv_in = is_decoder ? TN_A_GEN_WAV_SEANET_CONV_IN : TN_A_SEANET_CONV_IN; const char * conv_out = is_decoder ? TN_A_GEN_WAV_SEANET_CONV_OUT : TN_A_SEANET_CONV_OUT; const char * res1 = is_decoder ? TN_A_GEN_WAV_SEANET_RES_CONV1 : TN_A_SEANET_RES_CONV1; const char * res2 = is_decoder ? TN_A_GEN_WAV_SEANET_RES_CONV2 : TN_A_SEANET_RES_CONV2; const char * scale = is_decoder ? TN_A_GEN_WAV_SEANET_SCALE_CONV : TN_A_SEANET_SCALE_CONV; seanet.conv_in_w = get_tensor(mtmd_format(conv_in, "weight")); seanet.conv_in_b = get_tensor(mtmd_format(conv_in, "bias")); seanet.conv_out_w = get_tensor(mtmd_format(conv_out, "weight")); seanet.conv_out_b = get_tensor(mtmd_format(conv_out, "bias")); seanet.stages.resize(hparams.seanet_n_stage); for (int i = 0; i < hparams.seanet_n_stage; i++) { auto & stage = seanet.stages[i]; stage.res_conv1_w = get_tensor(mtmd_format(res1, i, "weight")); stage.res_conv1_b = get_tensor(mtmd_format(res1, i, "bias")); stage.res_conv2_w = get_tensor(mtmd_format(res2, i, "weight")); stage.res_conv2_b = get_tensor(mtmd_format(res2, i, "bias")); stage.scale_conv_w = get_tensor(mtmd_format(scale, i, "weight")); stage.scale_conv_b = get_tensor(mtmd_format(scale, i, "bias")); } }; auto get_vector = [&](const std::string & name) { std::vector result; auto it = tensor_offset.find(name); if (it == tensor_offset.end()) { return result; } const int64_t idx = gguf_find_tensor(ctx_gguf.get(), name.c_str()); if (idx < 0) { throw std::runtime_error(mtmd_format("%s: failed to find tensor %s\n", __func__, name.c_str())); } if (const auto type = gguf_get_tensor_type(ctx_gguf.get(), idx); type != GGML_TYPE_F32) { throw std::runtime_error(mtmd_format("%s: %s must be %s, was %s\n", __func__, name.c_str(), ggml_type_name(GGML_TYPE_F32), ggml_type_name(type))); } const size_t n_bytes = gguf_get_tensor_size(ctx_gguf.get(), idx); if (n_bytes == 0) { throw std::runtime_error(mtmd_format("%s: tensor %s is empty\n", __func__, name.c_str())); } const size_t n_elems = n_bytes / sizeof(float); result.resize(n_elems); fin.seekg(it->second, std::ios::beg); fin.read(reinterpret_cast(result.data()), n_bytes); return result; }; auto get_scalar = [&](const std::string & name, float default_val) { auto v = get_vector(name); if (v.empty()) { return default_val; } if (v.size() != 1) { throw std::runtime_error(mtmd_format("%s: expected scalar tensor '%s' but got %d elements\n", __func__, name.c_str(), (int) v.size())); } return v[0]; }; model.class_embedding = get_tensor(TN_CLASS_EMBD, false); model.pre_ln_w = get_tensor(mtmd_format(TN_LN_PRE, prefix, "weight"), false); model.pre_ln_b = get_tensor(mtmd_format(TN_LN_PRE, prefix, "bias"), false); model.post_ln_w = get_tensor(mtmd_format(TN_LN_POST, prefix, "weight"), false); model.post_ln_b = get_tensor(mtmd_format(TN_LN_POST, prefix, "bias"), false); // every vision encoder starts with a patch embedding, except the ones with their own stem const bool has_patch_embd = ( model.modality == MTMD_MODALITY_VISION && model.proj_type != PROJECTOR_TYPE_GEMMA3NV && model.proj_type != PROJECTOR_TYPE_DEEPSEEKOCR && model.proj_type != PROJECTOR_TYPE_DEEPSEEKOCR2); model.patch_bias = get_tensor(TN_PATCH_BIAS, false); model.patch_embeddings_0 = get_tensor(TN_PATCH_EMBD, has_patch_embd); model.patch_embeddings_1 = get_tensor(TN_PATCH_EMBD_1, false); model.norm_embd_w = get_tensor(mtmd_format(TN_NORM_EMBD, "weight"), false); model.norm_embd_b = get_tensor(mtmd_format(TN_NORM_EMBD, "bias"), false); model.position_embeddings = get_tensor(mtmd_format(TN_POS_EMBD, prefix), false); // lfm3vl model.lfm3vl_embedding_norm_w = get_tensor(mtmd_format(TN_LFM3VL_EMBEDDING_NORM, prefix, "weight"), false); const bool has_standard_layers = ( model.proj_type != PROJECTOR_TYPE_GEMMA3NV && model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC && model.proj_type != PROJECTOR_TYPE_ZONOS3_SPKENC && model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN && model.proj_type != PROJECTOR_TYPE_SOPRANO && model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO && model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR && model.proj_type != PROJECTOR_TYPE_CHATTERBOX && model.proj_type != PROJECTOR_TYPE_CHATTERBOX_SPKENC); // layers const int n_layers_to_load = has_standard_layers ? hparams.n_layer : 0; model.layers.resize(n_layers_to_load); for (int il = 0; il < n_layers_to_load; ++il) { auto & layer = model.layers[il]; // lfm3vl / lfm3a conv layers have no attention tensors const bool o_w_required = model.proj_type != PROJECTOR_TYPE_LFM3VL && model.proj_type != PROJECTOR_TYPE_LFM3A; layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, prefix, il, "weight"), false); layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, prefix, il, "weight"), false); layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, prefix, il, "weight"), false); layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, prefix, il, "weight"), o_w_required); layer.qkv_w = get_tensor(mtmd_format(TN_ATTN_QKV, prefix, il, "weight"), false); layer.k_norm = get_tensor(mtmd_format(TN_ATTN_K_NORM, prefix, il, "weight"), false); layer.q_norm = get_tensor(mtmd_format(TN_ATTN_Q_NORM, prefix, il, "weight"), false); layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, prefix, il, "weight"), false); layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, prefix, il, "weight"), false); layer.ls_1_w = get_tensor(mtmd_format(TN_LS_1, prefix, il, "weight"), false); // no bias layer.ls_2_w = get_tensor(mtmd_format(TN_LS_2, prefix, il, "weight"), false); // no bias layer.ls_out_w = get_tensor(mtmd_format(TN_LS_OUT, prefix, il, "weight"), false); // no bias layer.attn_post_norm_w = get_tensor(mtmd_format(TN_ATTN_POST_NORM, prefix, il, "weight"), false); // no bias layer.ff_post_norm_w = get_tensor(mtmd_format(TN_FFN_POST_NORM, prefix, il, "weight"), false); // no bias layer.k_b = get_tensor(mtmd_format(TN_ATTN_K, prefix, il, "bias"), false); layer.q_b = get_tensor(mtmd_format(TN_ATTN_Q, prefix, il, "bias"), false); layer.v_b = get_tensor(mtmd_format(TN_ATTN_V, prefix, il, "bias"), false); layer.o_b = get_tensor(mtmd_format(TN_ATTN_OUTPUT, prefix, il, "bias"), false); layer.qkv_b = get_tensor(mtmd_format(TN_ATTN_QKV, prefix, il, "bias"), false); layer.ln_1_b = get_tensor(mtmd_format(TN_LN_1, prefix, il, "bias"), false); layer.ln_2_b = get_tensor(mtmd_format(TN_LN_2, prefix, il, "bias"), false); // MoE ffn (dots3note vision pyramid blocks); replaces the dense ffn when present layer.ff_gate_inp_w = get_tensor(mtmd_format(TN_FFN_GATE_INP, prefix, il, "weight"), false); layer.ff_gate_exps_w = get_tensor(mtmd_format(TN_FFN_GATE_EXPS, prefix, il, "weight"), false); layer.ff_up_exps_w = get_tensor(mtmd_format(TN_FFN_UP_EXPS, prefix, il, "weight"), false); layer.ff_down_exps_w = get_tensor(mtmd_format(TN_FFN_DOWN_EXPS, prefix, il, "weight"), false); layer.ff_exp_probs_b = get_tensor(mtmd_format(TN_FFN_EXP_PROBS_B, prefix, il, "weight"), false); const bool is_moe = layer.ff_gate_exps_w != nullptr; // ffn layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, prefix, il, "weight"), !is_moe); layer.ff_up_b = get_tensor(mtmd_format(TN_FFN_UP, prefix, il, "bias"), false); layer.ff_gate_w = get_tensor(mtmd_format(TN_FFN_GATE, prefix, il, "weight"), false); layer.ff_gate_b = get_tensor(mtmd_format(TN_FFN_GATE, prefix, il, "bias"), false); layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, prefix, il, "weight"), !is_moe); layer.ff_down_b = get_tensor(mtmd_format(TN_FFN_DOWN, prefix, il, "bias"), false); // mimovl per-head attention sink bias layer.attn_sinks = get_tensor(mtmd_format(TN_ATTN_SINKS, prefix, il), false); // qwen3vl deepstack layer layer.deepstack_norm_w = get_tensor(mtmd_format(TN_DEEPSTACK_NORM, il, "weight"), false); layer.deepstack_norm_b = get_tensor(mtmd_format(TN_DEEPSTACK_NORM, il, "bias"), false); layer.deepstack_fc1_w = get_tensor(mtmd_format(TN_DEEPSTACK_FC1, il, "weight"), false); layer.deepstack_fc1_b = get_tensor(mtmd_format(TN_DEEPSTACK_FC1, il, "bias"), false); layer.deepstack_fc2_w = get_tensor(mtmd_format(TN_DEEPSTACK_FC2, il, "weight"), false); layer.deepstack_fc2_b = get_tensor(mtmd_format(TN_DEEPSTACK_FC2, il, "bias"), false); if (layer.has_deepstack()) { model.n_deepstack_layers++; } // lfm3vl conv layers layer.conv2d_in_proj_w = get_tensor(mtmd_format(TN_LFM3VL_CONV2D_IN_PROJ, prefix, il, "weight"), false); layer.conv2d_spatial_w = get_tensor(mtmd_format(TN_LFM3VL_CONV2D_SPATIAL, prefix, il, "weight"), false); layer.conv2d_out_proj_w = get_tensor(mtmd_format(TN_LFM3VL_CONV2D_OUT_PROJ, prefix, il, "weight"), false); // some models already exported with legacy (incorrect) naming which is quite messy, let's fix it here // note: Qwen model converted from the old surgery script has n_ff = 0, so we cannot use n_ff to check! bool is_ffn_swapped = ( // only old models need this fix model.proj_type == PROJECTOR_TYPE_MLP || model.proj_type == PROJECTOR_TYPE_MLP_NORM || model.proj_type == PROJECTOR_TYPE_LDP || model.proj_type == PROJECTOR_TYPE_LDPV2 || model.proj_type == PROJECTOR_TYPE_QWEN2VL || model.proj_type == PROJECTOR_TYPE_QWEN25VL || model.proj_type == PROJECTOR_TYPE_EXAONE4_5 || model.proj_type == PROJECTOR_TYPE_GLM_EDGE || model.proj_type == PROJECTOR_TYPE_GEMMA3 || model.proj_type == PROJECTOR_TYPE_IDEFICS3 || model.proj_type == PROJECTOR_TYPE_MINICPMV || model.proj_type == PROJECTOR_TYPE_MINICPMV4_6 || model.proj_type == PROJECTOR_TYPE_MINICPMV4_7 || model.proj_type == PROJECTOR_TYPE_MINICPM_OCR_1_0 ) && layer.ff_up_w && layer.ff_down_w && layer.ff_down_w->ne[0] == hparams.n_embd; if (is_ffn_swapped) { // swap up and down weights ggml_tensor * tmp = layer.ff_up_w; layer.ff_up_w = layer.ff_down_w; layer.ff_down_w = tmp; // swap up and down biases tmp = layer.ff_up_b; layer.ff_up_b = layer.ff_down_b; layer.ff_down_b = tmp; if (il == 0) { LOG_WRN("%s: ffn up/down are swapped\n", __func__); } } } // code2wav: RVQ codes -> raw PCM, drama3 has no quant/pre_conv/tfm projections auto load_code2wav = [&](bool with_proj) { auto & c2w = model.c2w; c2w.quant_first_in_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_IN, "weight"), with_proj); c2w.quant_first_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight"), with_proj); c2w.quant_first_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight")); c2w.quant_rest_in_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_IN, "weight"), with_proj); c2w.quant_rest_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_OUT, "weight"), with_proj); c2w.quant_rest_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_CB, "weight")); c2w.pre_conv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_PRE_CONV, "weight"), with_proj); c2w.pre_conv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_PRE_CONV, "bias"), with_proj); c2w.tfm_in_proj_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_IN_PROJ, "weight"), with_proj); c2w.tfm_in_proj_b = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_IN_PROJ, "bias"), with_proj); c2w.tfm_out_proj_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "weight"), with_proj); c2w.tfm_out_proj_b = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias"), with_proj); c2w.tfm_output_norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight")); // loaded manually, the generic model.layers loop is taken by code_predictor c2w.tfm_layers.resize(hparams.wav_tfm_n_layer); for (int il = 0; il < hparams.wav_tfm_n_layer; il++) { auto & layer = c2w.tfm_layers[il]; const char * p = "a.gen.wav.tfm"; layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, p, il, "weight")); layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, p, il, "weight")); layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, p, il, "weight")); layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight")); layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, p, il, "weight")); layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, p, il, "weight")); layer.ls_1_w = get_tensor(mtmd_format(TN_LS_1, p, il, "weight")); layer.ls_2_w = get_tensor(mtmd_format(TN_LS_2, p, il, "weight")); layer.ff_gate_w = get_tensor(mtmd_format(TN_FFN_GATE, p, il, "weight")); layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, p, il, "weight")); layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, p, il, "weight")); } // upsample: 2x (causal ConvTranspose1d + ConvNeXt block) c2w.upsample.resize(hparams.wav_upsample_n_block); for (int il = 0; il < hparams.wav_upsample_n_block; il++) { auto & up = c2w.upsample[il]; up.conv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_CONV, il, "weight")); up.conv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_CONV, il, "bias")); up.dwconv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight")); up.dwconv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias")); up.norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_NORM, il, "weight")); up.norm_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_NORM, il, "bias")); up.pw1_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW1, il, "weight")); up.pw1_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW1, il, "bias")); up.pw2_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW2, il, "weight")); up.pw2_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW2, il, "bias")); up.gamma = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_GAMMA, il)); } // DAC decoder: conv_pre + n upsample blocks (each with n_res residual units) + conv_post c2w.dac_entry_w = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_ENTRY, "weight")); c2w.dac_entry_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_ENTRY, "bias")); c2w.dac.resize(hparams.wav_dac_n_block); for (int il = 0; il < hparams.wav_dac_n_block; il++) { auto & blk = c2w.dac[il]; blk.snake_alpha = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_SNAKE, il, "alpha")); blk.snake_beta = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_SNAKE, il, "beta")); blk.conv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_CONV, il, "weight")); blk.conv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_CONV, il, "bias")); blk.res.resize(hparams.wav_dac_n_res); for (int ir = 0; ir < hparams.wav_dac_n_res; ir++) { auto & res = blk.res[ir]; res.act1_alpha = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "alpha")); res.act1_beta = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "beta")); res.conv1_w = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "weight")); res.conv1_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "bias")); res.act2_alpha = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "alpha")); res.act2_beta = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "beta")); res.conv2_w = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "weight")); res.conv2_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "bias")); } } c2w.dac_post_snake_alpha = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "alpha")); c2w.dac_post_snake_beta = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "beta")); c2w.dac_post_conv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_POST_CONV, "weight")); c2w.dac_post_conv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_POST_CONV, "bias")); }; switch (model.proj_type) { case PROJECTOR_TYPE_MLP: case PROJECTOR_TYPE_MLP_NORM: { // LLaVA projection: mm.0 -> gelu -> mm.2 // Yi-type llava (MLP_NORM): mm.0 -> norm mm.1 -> gelu -> mm.3 -> norm mm.4 model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); model.mm_3_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 3, "weight"), model.proj_type == PROJECTOR_TYPE_MLP_NORM); if (model.mm_3_w) { // TODO: this is a hack to support Yi-type llava model.proj_type = PROJECTOR_TYPE_MLP_NORM; model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); model.mm_3_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 3, "bias")); model.mm_4_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 4, "weight")); model.mm_4_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 4, "bias")); } else { model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); } model.image_newline = get_tensor(TN_IMAGE_NEWLINE, false); } break; case PROJECTOR_TYPE_LDP: { // MobileVLM projection model.mm_model_mlp_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "weight")); model.mm_model_mlp_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "bias")); model.mm_model_mlp_3_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "weight")); model.mm_model_mlp_3_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "bias")); model.mm_model_block_1_block_0_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 0, "0.weight")); model.mm_model_block_1_block_0_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.weight")); model.mm_model_block_1_block_0_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.bias")); model.mm_model_block_1_block_1_fc1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.weight")); model.mm_model_block_1_block_1_fc1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.bias")); model.mm_model_block_1_block_1_fc2_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.weight")); model.mm_model_block_1_block_1_fc2_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.bias")); model.mm_model_block_1_block_2_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 2, "0.weight")); model.mm_model_block_1_block_2_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.weight")); model.mm_model_block_1_block_2_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.bias")); model.mm_model_block_2_block_0_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 0, "0.weight")); model.mm_model_block_2_block_0_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.weight")); model.mm_model_block_2_block_0_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.bias")); model.mm_model_block_2_block_1_fc1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.weight")); model.mm_model_block_2_block_1_fc1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.bias")); model.mm_model_block_2_block_1_fc2_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.weight")); model.mm_model_block_2_block_1_fc2_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.bias")); model.mm_model_block_2_block_2_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 2, "0.weight")); model.mm_model_block_2_block_2_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.weight")); model.mm_model_block_2_block_2_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.bias")); } break; case PROJECTOR_TYPE_LDPV2: { // MobilVLM_V2 projection model.mm_model_mlp_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "weight")); model.mm_model_mlp_0_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "bias")); model.mm_model_mlp_2_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 2, "weight")); model.mm_model_mlp_2_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 2, "bias")); model.mm_model_peg_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_PEG, 0, "weight")); model.mm_model_peg_0_b = get_tensor(mtmd_format(TN_MVLM_PROJ_PEG, 0, "bias")); } break; case PROJECTOR_TYPE_MINICPMV: { // model.mm_model_pos_embed = get_tensor(new_clip->ctx_data, TN_MINICPMV_POS_EMBD); model.mm_model_pos_embed_k = get_tensor(TN_MINICPMV_POS_EMBD_K); model.mm_model_query = get_tensor(TN_MINICPMV_QUERY); model.mm_model_proj = get_tensor(TN_MINICPMV_PROJ); model.mm_model_kv_proj = get_tensor(TN_MINICPMV_KV_PROJ); model.mm_model_attn_q_w = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "q", "weight")); model.mm_model_attn_k_w = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "k", "weight")); model.mm_model_attn_v_w = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "v", "weight")); model.mm_model_attn_q_b = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "q", "bias")); model.mm_model_attn_k_b = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "k", "bias")); model.mm_model_attn_v_b = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "v", "bias")); model.mm_model_attn_o_w = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "out", "weight")); model.mm_model_attn_o_b = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "out", "bias")); model.mm_model_ln_q_w = get_tensor(mtmd_format(TN_MINICPMV_LN, "q", "weight")); model.mm_model_ln_q_b = get_tensor(mtmd_format(TN_MINICPMV_LN, "q", "bias")); model.mm_model_ln_kv_w = get_tensor(mtmd_format(TN_MINICPMV_LN, "kv", "weight")); model.mm_model_ln_kv_b = get_tensor(mtmd_format(TN_MINICPMV_LN, "kv", "bias")); model.mm_model_ln_post_w = get_tensor(mtmd_format(TN_MINICPMV_LN, "post", "weight")); model.mm_model_ln_post_b = get_tensor(mtmd_format(TN_MINICPMV_LN, "post", "bias")); } break; case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: case PROJECTOR_TYPE_MINICPM_OCR_1_0: { const bool merger_required = hparams.n_merge == 4; auto get_merger_tensor = [&](const std::string & name, bool required = true) { return get_tensor(name, merger_required && required); }; // ViT merger: window self-attention model.vit_merger_ln1_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_LN1, "weight")); model.vit_merger_ln1_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_LN1, "bias")); model.vit_merger_attn_q_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_Q, "weight")); model.vit_merger_attn_q_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_Q, "bias"), false); model.vit_merger_attn_k_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_K, "weight")); model.vit_merger_attn_k_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_K, "bias"), false); model.vit_merger_attn_v_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_V, "weight")); model.vit_merger_attn_v_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_V, "bias"), false); model.vit_merger_attn_o_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_O, "weight")); model.vit_merger_attn_o_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_O, "bias"), false); // ViT merger: MLP downsample model.vit_merger_ds_ln_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_LN, "weight")); model.vit_merger_ds_ln_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_LN, "bias")); model.vit_merger_ds_up_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_UP, "weight")); model.vit_merger_ds_up_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_UP, "bias"), false); model.vit_merger_ds_down_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_DOWN, "weight")); model.vit_merger_ds_down_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_DOWN, "bias"), false); // Final Merger (DownsampleMLP) model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B, false); model.mm_ffn_up_w = get_tensor(mtmd_format(TN_MM_UP, "weight")); model.mm_ffn_up_b = get_tensor(mtmd_format(TN_MM_UP, "bias"), false); model.mm_ffn_down_w = get_tensor(mtmd_format(TN_MM_DOWN, "weight")); model.mm_ffn_down_b = get_tensor(mtmd_format(TN_MM_DOWN, "bias"), false); model.tok_embd_sep = get_tensor(TN_TOK_EMBD_SEP, model.proj_type == PROJECTOR_TYPE_MINICPMV4_7); } break; case PROJECTOR_TYPE_GLM_EDGE: { model.mm_model_adapter_conv_w = get_tensor(mtmd_format(TN_GLM_ADAPER_CONV, "weight")); model.mm_model_adapter_conv_b = get_tensor(mtmd_format(TN_GLM_ADAPER_CONV, "bias")); model.mm_model_mlp_0_w = get_tensor(mtmd_format(TN_GLM_ADAPTER_LINEAR, "weight")); model.mm_model_ln_q_w = get_tensor(mtmd_format(TN_GLM_ADAPTER_NORM_1, "weight")); model.mm_model_ln_q_b = get_tensor(mtmd_format(TN_GLM_ADAPTER_NORM_1, "bias")); model.mm_model_mlp_1_w = get_tensor(mtmd_format(TN_GLM_ADAPTER_D_H_2_4H, "weight")); model.mm_model_mlp_2_w = get_tensor(mtmd_format(TN_GLM_ADAPTER_GATE, "weight")); model.mm_model_mlp_3_w = get_tensor(mtmd_format(TN_GLM_ADAPTER_D_4H_2_H, "weight")); model.mm_boi = get_tensor(mtmd_format(TN_TOK_GLM_BOI)); model.mm_eoi = get_tensor(mtmd_format(TN_TOK_GLM_EOI)); } break; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_EXAONE4_5: { model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); } break; case PROJECTOR_TYPE_MAGEVL: { // d_head rope factors: the first half drives the even outputs, the second half the odd ones model.rope_freqs = get_tensor(TN_ROPE_FREQS); const int d_head = hparams.n_head > 0 ? hparams.n_embd / hparams.n_head : 0; if (d_head <= 0 || d_head * hparams.n_head != hparams.n_embd || d_head % 32 != 0 || model.rope_freqs->type != GGML_TYPE_F32 || ggml_nelements(model.rope_freqs) != d_head) { throw std::runtime_error(mtmd_format("%s: hparams do not match the tensor shapes\n", __func__)); } model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); } break; case PROJECTOR_TYPE_QWEN3VL: { model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); } break; case PROJECTOR_TYPE_LFM3VL: { model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); // each layer must be either a conv block or a complete attention block for (size_t il = 0; il < model.layers.size(); ++il) { const auto & layer = model.layers[il]; const bool is_conv = layer.conv2d_in_proj_w != nullptr; const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w; if (is_conv == is_attn) { throw std::runtime_error(mtmd_format("%s: lfm3vl layer %zu is neither a complete conv nor a complete attention block\n", __func__, il)); } } } break; case PROJECTOR_TYPE_LING3VL: { model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); // merger.norm model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); // merger.norm model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); // linear_proj.0 model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); // linear_proj.2 model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); } break; case PROJECTOR_TYPE_MIMOVL: { model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"), false); model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false); } break; case PROJECTOR_TYPE_MINIMAX_M3: { // per-patch MLP: mm.1 -> gelu -> mm.2 model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); // 2x2 merge MLP: mm.merge.fc1 -> gelu -> mm.merge.fc2 model.mm_merger_fc1_w = get_tensor(mtmd_format(TN_MM_MERGER_FC1, "weight")); model.mm_merger_fc1_b = get_tensor(mtmd_format(TN_MM_MERGER_FC1, "bias")); model.mm_merger_fc2_w = get_tensor(mtmd_format(TN_MM_MERGER_FC2, "weight")); model.mm_merger_fc2_b = get_tensor(mtmd_format(TN_MM_MERGER_FC2, "bias")); } break; case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: { model.layers.resize(hparams.n_layer); model.a_patches.resize(hparams.audio_patch_sizes.size()); for (size_t i = 0; i < model.a_patches.size(); ++i) { auto & pe = model.a_patches[i]; pe.proj_w = get_tensor(mtmd_format(TN_A_PATCH, (int) i, "weight")); pe.proj_b = get_tensor(mtmd_format(TN_A_PATCH, (int) i, "bias")); pe.norm_w = get_tensor(mtmd_format(TN_A_PATCH_NORM, (int) i, "weight")); if (pe.proj_w->ne[1] != hparams.audio_hidden_sizes[i]) { throw std::runtime_error(mtmd_format("%s: audio stage %zu projection has width %" PRId64 ", expected %d\n", __func__, i, pe.proj_w->ne[1], hparams.audio_hidden_sizes[i])); } } for (int il = 0; il < hparams.n_layer; ++il) { auto & layer = model.layers[il]; layer.conv_w = get_tensor(mtmd_format(TN_CONV1D, il, "weight")); layer.conv_b = get_tensor(mtmd_format(TN_CONV1D, il, "bias")); layer.conv_norm_w = get_tensor(mtmd_format(TN_A_CONV1D_NORM, il, "weight")); layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, "a", il, "weight")); layer.ff_up_b = get_tensor(mtmd_format(TN_FFN_UP, "a", il, "bias")); layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, "a", il, "weight")); layer.ff_down_b = get_tensor(mtmd_format(TN_FFN_DOWN, "a", il, "bias")); } // pre_ln and post_ln come from the generic path above, where they are optional if (!model.pre_ln_w || !model.post_ln_w) { throw std::runtime_error(mtmd_format("%s: missing pre_ln or post_ln weight\n", __func__)); } model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight")); model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias")); } break; case PROJECTOR_TYPE_VIBEVOICE_ASR: { const int n_stage = (int) hparams.audio_patch_sizes.size(); for (int e = 0; e < 2; ++e) { auto & enc = model.cnx_enc[e]; enc.conv_w.resize(n_stage + 1); enc.conv_b.resize(n_stage + 1); for (int i = 0; i <= n_stage; ++i) { enc.conv_w[i] = get_tensor(mtmd_format(TN_A_CNX_CONV, e, i, "weight")); enc.conv_b[i] = get_tensor(mtmd_format(TN_A_CNX_CONV, e, i, "bias")); // the receptive field in load_hparams assumes these kernel sizes const int64_t k_expected = (i == 0 || i == n_stage) ? hparams.audio_conv_kernel_size : 2 * hparams.audio_patch_sizes[i]; if (enc.conv_w[i]->ne[0] != k_expected) { throw std::runtime_error(mtmd_format("%s: %s has kernel %" PRId64 ", expected %" PRId64 "\n", __func__, ggml_get_name(enc.conv_w[i]), enc.conv_w[i]->ne[0], k_expected)); } } enc.blocks.resize(hparams.n_layer); for (int il = 0; il < hparams.n_layer; ++il) { auto & layer = enc.blocks[il]; layer.ln_1_w = get_tensor(mtmd_format(TN_A_CNX_NORM, e, il, "weight")); layer.conv_w = get_tensor(mtmd_format(TN_A_CNX_DW, e, il, "weight")); layer.conv_b = get_tensor(mtmd_format(TN_A_CNX_DW, e, il, "bias")); layer.ls_1_w = get_tensor(mtmd_format(TN_A_CNX_LS1, e, il, "weight")); layer.ln_2_w = get_tensor(mtmd_format(TN_A_CNX_FFN_NORM, e, il, "weight")); layer.ff_up_w = get_tensor(mtmd_format(TN_A_CNX_FFN_UP, e, il, "weight")); layer.ff_up_b = get_tensor(mtmd_format(TN_A_CNX_FFN_UP, e, il, "bias")); layer.ff_down_w = get_tensor(mtmd_format(TN_A_CNX_FFN_DOWN, e, il, "weight")); layer.ff_down_b = get_tensor(mtmd_format(TN_A_CNX_FFN_DOWN, e, il, "bias")); layer.ls_2_w = get_tensor(mtmd_format(TN_A_CNX_LS2, e, il, "weight")); if (layer.conv_w->ne[0] != hparams.audio_conv_kernel_size || layer.conv_w->type != GGML_TYPE_F32) { throw std::runtime_error(mtmd_format("%s: %s must be F32 with kernel %d\n", __func__, ggml_get_name(layer.conv_w), hparams.audio_conv_kernel_size)); } } } // acoustic connector: mlp 0 -> norm -> mlp 1, semantic connector: mlp 2 -> norm -> mlp 3 model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias")); model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP_NORM, 0, "weight")); model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias")); model.mm_norm_mid_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP_NORM, 2, "weight")); model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight")); model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias")); } break; case PROJECTOR_TYPE_INKLING2V: { // these come from the generic path above, where they are optional if (!model.class_embedding || !model.position_embeddings || !model.pre_ln_w || !model.pre_ln_b) { throw std::runtime_error(mtmd_format("%s: missing class_embd, position_embd or pre_ln\n", __func__)); } model.class_position_embedding = get_tensor(TN_CLASS_POS_EMBD); model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"), false); model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false); } break; case PROJECTOR_TYPE_MUSE_GLIMMER: { // 3-linear MLP: fc -> erf-GELU -> proj -> erf-GELU -> vision_proj (into LLM residual dim) model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); } break; case PROJECTOR_TYPE_STEP3VL: { model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"), false); model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"), false); model.mm_model_proj = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight")); } break; case PROJECTOR_TYPE_YOUTUVL: { model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); // merger.ln_q (RMS norm) model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); // merger.mlp.0 model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); // merger.mlp.2 model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); } break; case PROJECTOR_TYPE_YASA2: { // reuse tensors already loaded by the common section // (TN_PATCH_EMBD and TN_PATCH_BIAS have the same tensor names) model.yasa_patch_w = model.patch_embeddings_0; model.yasa_patch_b = model.patch_bias; model.yasa_patch_ln_w = get_tensor(TN_YASA_PATCH_LN_W, false); model.yasa_patch_ln_b = get_tensor(TN_YASA_PATCH_LN_B, false); model.yasa_backbone_ln_w = get_tensor(TN_YASA_BACKBONE_LN_W, false); model.yasa_backbone_ln_b = get_tensor(TN_YASA_BACKBONE_LN_B, false); model.yasa_vision_pos_embed = get_tensor(TN_YASA_POS_EMBD, false); model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"), false); model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false); model.yasa_stages.clear(); for (int s = 0; ; ++s) { yasa2_stage stage; stage.down_ln_w = get_tensor(mtmd_format(TN_YASA_STAGE_DOWN_LN, s, "weight"), false); stage.down_ln_b = get_tensor(mtmd_format(TN_YASA_STAGE_DOWN_LN, s, "bias"), false); stage.down_conv_w = get_tensor(mtmd_format(TN_YASA_STAGE_DOWN_CONV, s, "weight"), false); stage.down_conv_b = get_tensor(mtmd_format(TN_YASA_STAGE_DOWN_CONV, s, "bias"), false); for (int bi = 0; ; ++bi) { yasa2_block blk; blk.dw_w = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "dw", "weight"), false); if (!blk.dw_w) { break; } blk.dw_b = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "dw", "bias"), false); blk.ln_w = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "ln", "weight"), false); blk.ln_b = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "ln", "bias"), false); blk.pw1_w = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "pw1", "weight"), false); blk.pw1_b = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "pw1", "bias"), false); blk.grn_w = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "grn", "weight"), false); blk.grn_b = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "grn", "bias"), false); blk.pw2_w = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "pw2", "weight"), false); blk.pw2_b = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "pw2", "bias"), false); stage.blocks.push_back(blk); } if (!stage.down_conv_w && stage.blocks.empty()) { break; } model.yasa_stages.push_back(std::move(stage)); } } break; case PROJECTOR_TYPE_GLM4V: case PROJECTOR_TYPE_GLM5V: { model.mm_fc_w = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight")); model.mm_ffn_up_w = get_tensor(mtmd_format(TN_MM_UP, "weight")); model.mm_ffn_up_b = get_tensor(mtmd_format(TN_MM_UP, "bias"), false); model.mm_ffn_gate_w = get_tensor(mtmd_format(TN_MM_GATE, "weight")); model.mm_ffn_gate_b = get_tensor(mtmd_format(TN_MM_GATE, "bias"), false); model.mm_ffn_down_w = get_tensor(mtmd_format(TN_MM_DOWN, "weight")); model.mm_ffn_down_b = get_tensor(mtmd_format(TN_MM_DOWN, "bias"), false); model.mm_post_norm_w = get_tensor(mtmd_format(TN_MM_POST_NORM, "weight")); model.mm_post_norm_b = get_tensor(mtmd_format(TN_MM_POST_NORM, "bias"), false); model.mm_patch_merger_w = get_tensor(mtmd_format(TN_MM_PATCH_MERGER, "weight")); model.mm_patch_merger_b = get_tensor(mtmd_format(TN_MM_PATCH_MERGER, "bias")); } break; case PROJECTOR_TYPE_GEMMA3: { model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ); model.mm_soft_emb_norm_w = get_tensor(TN_MM_SOFT_EMB_N); } break; case PROJECTOR_TYPE_GEMMA4V: { model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ); model.std_bias = get_tensor(TN_STD_BIAS, false); model.std_scale = get_tensor(TN_STD_SCALE, false); // load scalar for Gemma4ClippableLinear for (auto * tensor : tensors_to_load) { std::string name = tensor->name; if (string_ends_with(name, ".weight")) { std::string name_inp_max = name; std::string name_inp_min = name; std::string name_out_max = name; std::string name_out_min = name; string_replace_all(name_inp_max, ".weight", ".input_max"); string_replace_all(name_inp_min, ".weight", ".input_min"); string_replace_all(name_out_max, ".weight", ".output_max"); string_replace_all(name_out_min, ".weight", ".output_min"); model.clamp_info_map[name] = { get_scalar(name_inp_max, FLT_MAX), get_scalar(name_inp_min, -FLT_MAX), get_scalar(name_out_max, FLT_MAX), get_scalar(name_out_min, -FLT_MAX) }; } } } break; case PROJECTOR_TYPE_GEMMA4UV: { model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ); model.patch_norm_1_w = get_tensor(mtmd_format(TN_PATCH_NORM, 1, "weight")); model.patch_norm_1_b = get_tensor(mtmd_format(TN_PATCH_NORM, 1, "bias")); model.patch_norm_2_w = get_tensor(mtmd_format(TN_PATCH_NORM, 2, "weight")); model.patch_norm_2_b = get_tensor(mtmd_format(TN_PATCH_NORM, 2, "bias")); model.patch_norm_3_w = get_tensor(mtmd_format(TN_PATCH_NORM, 3, "weight")); // pos_norm model.patch_norm_3_b = get_tensor(mtmd_format(TN_PATCH_NORM, 3, "bias")); // pos_norm } break; case PROJECTOR_TYPE_GEMMA3NV: { model.mobilenet_stem_conv_w = get_tensor(TN_MNV5_STEM_CONV, false); model.mobilenet_stem_conv_b = get_tensor(TN_MNV5_STEM_BIAS, false); model.mobilenet_stem_norm_w = get_tensor(TN_MNV5_STEM_BN, false); model.msfa_ffn_expand_w = get_tensor(TN_MNV5_MSFA_FFN_EXP_W, false); model.msfa_ffn_expand_bn = get_tensor(TN_MNV5_MSFA_FFN_EXP_BN, false); // Consume BN if present but likely folded model.msfa_ffn_project_w = get_tensor(TN_MNV5_MSFA_FFN_PROJ_W, false); model.msfa_ffn_project_bn = get_tensor(TN_MNV5_MSFA_FFN_PROJ_BN, false); model.msfa_concat_norm_w = get_tensor(TN_MNV5_MSFA_NORM, false); // Dynamically load blocks stage by stage for (int stage = 0; stage < 4; ++stage) { int blocks_found_in_stage = 0; for (int blk_idx = 0; ; ++blk_idx) { bool found_block = false; mobilenetv5_block block; // 1. Check for Edge Residual (S0) block.s0_conv_exp_w = get_tensor(mtmd_format(TN_MNV5_BLK_S0_EXP_W, stage, blk_idx), false); if (block.s0_conv_exp_w) { found_block = true; block.s0_bn1_w = get_tensor(mtmd_format(TN_MNV5_BLK_S0_BN1_W, stage, blk_idx), false); block.s0_conv_pwl_w = get_tensor(mtmd_format(TN_MNV5_BLK_S0_PWL_W, stage, blk_idx), false); block.s0_bn2_w = get_tensor(mtmd_format(TN_MNV5_BLK_S0_BN2_W, stage, blk_idx), false); } // 2. Check for UIR (Universal Inverted Residual) else { // Check for dw_start OR pw_exp (some UIR blocks skip dw_start) block.dw_start_w = get_tensor(mtmd_format(TN_MNV5_BLK_DW_START_W, stage, blk_idx), false); block.pw_exp_w = get_tensor(mtmd_format(TN_MNV5_BLK_PW_EXP_W, stage, blk_idx), false); if (block.dw_start_w || block.pw_exp_w) { found_block = true; if (block.dw_start_w) { block.dw_start_bn_w = get_tensor(mtmd_format(TN_MNV5_BLK_DW_START_BN, stage, blk_idx), false); } if (block.pw_exp_w) { block.pw_exp_bn_w = get_tensor(mtmd_format(TN_MNV5_BLK_PW_EXP_BN, stage, blk_idx), false); } block.dw_mid_w = get_tensor(mtmd_format(TN_MNV5_BLK_DW_MID_W, stage, blk_idx), false); if (block.dw_mid_w) { block.dw_mid_bn_w = get_tensor(mtmd_format(TN_MNV5_BLK_DW_MID_BN, stage, blk_idx), false); } block.pw_proj_w = get_tensor(mtmd_format(TN_MNV5_BLK_PW_PROJ_W, stage, blk_idx), false); if (block.pw_proj_w) { block.pw_proj_bn_w = get_tensor(mtmd_format(TN_MNV5_BLK_PW_PROJ_BN, stage, blk_idx), false); } block.layer_scale_w = get_tensor(mtmd_format(TN_MNV5_BLK_LAYER_SCALE, stage, blk_idx), false); } } // 3. Check for Attention (MQA) // Even if UIR/Edge check failed, this might be a pure attention block ggml_tensor* attn_q_check = get_tensor(mtmd_format(TN_MNV5_ATTN_Q_W, stage, blk_idx), false); if (attn_q_check) { found_block = true; block.attn_q_w = attn_q_check; block.attn_k_w = get_tensor(mtmd_format(TN_MNV5_ATTN_K_W, stage, blk_idx), false); block.attn_v_w = get_tensor(mtmd_format(TN_MNV5_ATTN_V_W, stage, blk_idx), false); block.attn_o_w = get_tensor(mtmd_format(TN_MNV5_ATTN_O_W, stage, blk_idx), false); block.attn_k_dw_w = get_tensor(mtmd_format(TN_MNV5_ATTN_K_DW, stage, blk_idx), false); block.attn_k_norm_w = get_tensor(mtmd_format(TN_MNV5_ATTN_K_NORM, stage, blk_idx), false); block.attn_v_dw_w = get_tensor(mtmd_format(TN_MNV5_ATTN_V_DW, stage, blk_idx), false); block.attn_v_norm_w = get_tensor(mtmd_format(TN_MNV5_ATTN_V_NORM, stage, blk_idx), false); block.attn_norm_w = get_tensor(mtmd_format(TN_MNV5_ATTN_NORM, stage, blk_idx), false); // Note: Attention blocks also have layer_scale, load it if not already loaded by UIR check if (!block.layer_scale_w) { block.layer_scale_w = get_tensor(mtmd_format(TN_MNV5_BLK_LAYER_SCALE, stage, blk_idx), false); } } if (found_block) { model.mobilenet_blocks.push_back(block); blocks_found_in_stage++; } else { // End of blocks for this stage break; } } // Track where this stage ends in the flat vector if (blocks_found_in_stage > 0) { model.mobilenet_stage_ends.push_back(model.mobilenet_blocks.size() - 1); LOG_INF("%s: Stage %d ended at global block index %zu\n", __func__, stage, model.mobilenet_blocks.size() - 1); } } model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ); model.mm_soft_emb_norm_w = get_tensor(TN_MM_SOFT_EMB_N); } break; case PROJECTOR_TYPE_IDEFICS3: { model.mm_fc_w = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight")); } break; case PROJECTOR_TYPE_COHERE2V: { model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); } break; case PROJECTOR_TYPE_LFM2: case PROJECTOR_TYPE_D1OMNI_V: { model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM, false); model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B, false); model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); } break; case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_KIMIK25: { model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); } break; case PROJECTOR_TYPE_KIMIK31: { model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_post_norm_w = get_tensor(mtmd_format(TN_MM_POST_NORM, "weight")); } break; case PROJECTOR_TYPE_DEEPSEEK4V: { model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); // sentinel token embeddings written into the output block model.image_newline = get_tensor(TN_IMAGE_NEWLINE); model.token_embd_img_start = get_tensor(TN_TOK_IMG_START); model.token_embd_img_end = get_tensor(TN_TOK_IMG_END); model.token_embd_img_pad = get_tensor(TN_TOK_IMG_PAD); } break; case PROJECTOR_TYPE_DEEPSEEK41V: { model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); // image_start / image_end / image_newline, written into the output block model.image_newline = get_tensor(TN_IMAGE_NEWLINE); model.token_embd_img_start = get_tensor(TN_TOK_IMG_START); model.token_embd_img_end = get_tensor(TN_TOK_IMG_END); } break; case PROJECTOR_TYPE_PIXTRAL: { model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"), false); model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false); // [IMG_BREAK] token embedding model.token_embd_img_break = get_tensor(TN_TOK_IMG_BREAK); // for mistral small 3.1 model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM, false); model.mm_patch_merger_w = get_tensor(mtmd_format(TN_MM_PATCH_MERGER, "weight"), false); } break; case PROJECTOR_TYPE_LIGHTONOCR: { model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"), false); model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false); model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM, false); model.mm_patch_merger_w = get_tensor(mtmd_format(TN_MM_PATCH_MERGER, "weight"), false); } break; case PROJECTOR_TYPE_DOTS_OCR: case PROJECTOR_TYPE_DOTS3NOTE_V: { model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); // post_trunk_norm: applied after all ViT blocks, before the merger model.post_ln_w = get_tensor(mtmd_format(TN_MM_POST_NORM, "weight")); } break; case PROJECTOR_TYPE_DOTS3NOTE_A: { model.conv2d_1_w = get_tensor(mtmd_format(TN_CONV2D, 1, "weight")); model.conv2d_1_b = get_tensor(mtmd_format(TN_CONV2D, 1, "bias")); model.conv2d_2_w = get_tensor(mtmd_format(TN_CONV2D, 2, "weight")); model.conv2d_2_b = get_tensor(mtmd_format(TN_CONV2D, 2, "bias")); model.conv2d_3_w = get_tensor(mtmd_format(TN_CONV2D, 3, "weight")); model.conv2d_3_b = get_tensor(mtmd_format(TN_CONV2D, 3, "bias")); model.conv_out_w = get_tensor(mtmd_format(TN_CONV_OUT, "weight")); // no bias // adapter: LayerNorm -> Linear -> GELU -> Linear model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_NORM_PRE, "weight")); model.mm_norm_pre_b = get_tensor(mtmd_format(TN_MM_NORM_PRE, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias")); } break; case PROJECTOR_TYPE_ULTRAVOX: { model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_NORM_PRE, "weight")); model.mm_norm_mid_w = get_tensor(mtmd_format(TN_MM_NORM_MID, "weight")); } break; case PROJECTOR_TYPE_MERALION: { // Whisper encoder conv layers model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); // MERaLiON adaptor: 4 linear layers + ln_pre // linear_0 = frame compression (19200->6400) + SiLU // linear_1 = gate_proj (6400->6400) for GLU // linear_2 = pool_proj (6400->6400) for GLU // linear_3 = out_proj (6400->3584) model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias")); model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight")); model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias")); // ln_speech (LayerNorm before adaptor) model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_NORM_PRE, "weight")); model.mm_norm_pre_b = get_tensor(mtmd_format(TN_MM_NORM_PRE, "bias")); } break; case PROJECTOR_TYPE_QWEN2A: { model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight")); model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias")); } break; case PROJECTOR_TYPE_HIGGS_AUDIO: { model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); model.downsample_conv_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight")); model.downsample_conv_b = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias")); } break; case PROJECTOR_TYPE_QWEN3A: { model.conv2d_1_w = get_tensor(mtmd_format(TN_CONV2D, 1, "weight")); model.conv2d_1_b = get_tensor(mtmd_format(TN_CONV2D, 1, "bias")); model.conv2d_2_w = get_tensor(mtmd_format(TN_CONV2D, 2, "weight")); model.conv2d_2_b = get_tensor(mtmd_format(TN_CONV2D, 2, "bias")); model.conv2d_3_w = get_tensor(mtmd_format(TN_CONV2D, 3, "weight")); model.conv2d_3_b = get_tensor(mtmd_format(TN_CONV2D, 3, "bias")); model.conv_out_w = get_tensor(mtmd_format(TN_CONV_OUT, "weight")); // no bias model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias")); } break; case PROJECTOR_TYPE_MIMO_AUDIO: { model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); model.downsample_conv_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight")); model.downsample_norm_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_NORM, "weight")); model.downsample_norm_b = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_NORM, "bias")); model.rvq_codebook = get_tensor(mtmd_format(TN_A_RVQ_CODEBOOK, "weight"), false); model.mm_a_code_embd = get_tensor(mtmd_format(TN_MM_A_CODE_EMBD, "weight"), false); if (!model.rvq_codebook || !model.mm_a_code_embd) { throw std::runtime_error(mtmd_format("%s: mimo_audio: missing %s or %s\n", __func__, TN_A_RVQ_CODEBOOK, TN_MM_A_CODE_EMBD)); } // hparams.rvq_codebook_size comes from GGUF metadata and is independent of the // tensors' actual shapes - bound it so codebook/code_embd views built from it // (mimo-audio.cpp) can never read past either tensor's allocated bins/vocab. for (int32_t bins : hparams.rvq_codebook_size) { if (bins <= 0 || bins > model.rvq_codebook->ne[1] || bins > model.mm_a_code_embd->ne[1]) { throw std::runtime_error(mtmd_format( "%s: mimo_audio: %s entry (%d) out of range for codebook/code_embd tensors\n", __func__, KEY_A_RVQ_CODEBOOK_SIZE, bins)); } } // LLM-side connector: input_local_transformer + projection model.mm_a_local_layers.resize(hparams.audio_local_n_layer); for (int il = 0; il < hparams.audio_local_n_layer; il++) { auto & layer = model.mm_a_local_layers[il]; layer.q_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_Q, il, "weight")); layer.q_b = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_Q, il, "bias")); layer.k_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_K, il, "weight")); layer.k_b = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_K, il, "bias")); layer.v_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_V, il, "weight")); layer.v_b = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_V, il, "bias")); layer.o_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_OUT, il, "weight")); layer.ff_gate_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_FFN_GATE, il, "weight")); layer.ff_up_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_FFN_UP, il, "weight")); layer.ff_down_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_FFN_DOWN, il, "weight")); layer.ln_1_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_LN1, il, "weight")); layer.ln_2_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_LN2, il, "weight")); } model.mm_a_local_norm_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_NORM, "weight")); model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); } break; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: { // stem TDNN (block 0) model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 0, "weight")); model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 0, "bias")); // SE-Res2Net blocks (GGUF bid 1..3, one per hparams.n_layer) model.layers.resize(hparams.n_layer); for (int il = 0; il < hparams.n_layer; il++) { auto & layer = model.layers[il]; int bid = il + 1; layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, bid, "weight")); layer.conv_pw1_b = get_tensor(mtmd_format(TN_CONV_PW1, prefix, bid, "bias")); layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, bid, "weight")); layer.conv_pw2_b = get_tensor(mtmd_format(TN_CONV_PW2, prefix, bid, "bias")); layer.se_conv1_w = get_tensor(mtmd_format(TN_A_SE_CONV1, bid, "weight")); layer.se_conv1_b = get_tensor(mtmd_format(TN_A_SE_CONV1, bid, "bias")); layer.se_conv2_w = get_tensor(mtmd_format(TN_A_SE_CONV2, bid, "weight")); layer.se_conv2_b = get_tensor(mtmd_format(TN_A_SE_CONV2, bid, "bias")); layer.res2_conv_w.resize(7); layer.res2_conv_b.resize(7); for (int xid = 0; xid < 7; xid++) { layer.res2_conv_w[xid] = get_tensor(mtmd_format(TN_A_CONV_RES2, bid, xid, "weight")); layer.res2_conv_b[xid] = get_tensor(mtmd_format(TN_A_CONV_RES2, bid, xid, "bias")); } } // multi-layer feature aggregation model.conv_out_w = get_tensor(mtmd_format(TN_CONV_OUT, "weight")); model.conv_out_b = get_tensor(mtmd_format(TN_CONV_OUT, "bias")); // attentive statistics pooling model.spk_asp_attn_w = get_tensor(mtmd_format(TN_A_ASP_ATTN, "weight")); model.spk_asp_attn_b = get_tensor(mtmd_format(TN_A_ASP_ATTN, "bias")); model.spk_asp_tdnn_w = get_tensor(mtmd_format(TN_A_ASP_TDNN, "weight")); model.spk_asp_tdnn_b = get_tensor(mtmd_format(TN_A_ASP_TDNN, "bias")); // final speaker embedding projection model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight")); model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias")); } break; case PROJECTOR_TYPE_POCKETTTS_SPKENC: { load_seanet(model.seanet, false); model.downsample_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight")); model.spk_proj_w = get_tensor(mtmd_format(TN_A_SPEAKER_PROJ, "weight")); } break; case PROJECTOR_TYPE_POCKETTTS_GEN: { auto & flow = model.flow; flow.input_proj_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_INPUT_PROJ, "weight")); flow.input_proj_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_INPUT_PROJ, "bias")); flow.cond_embd_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_COND_EMBD, "weight")); flow.cond_embd_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_COND_EMBD, "bias")); flow.final_ada_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_FINAL_ADA, "weight")); flow.final_ada_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_FINAL_ADA, "bias")); flow.final_proj_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_FINAL_PROJ, "weight")); flow.final_proj_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_FINAL_PROJ, "bias")); flow.time.resize(2); for (size_t i = 0; i < flow.time.size(); i++) { auto & t = flow.time[i]; t.freqs = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_FREQS, (int) i)); t.up_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_UP, (int) i, "weight")); t.up_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_UP, (int) i, "bias")); t.down_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_DOWN, (int) i, "weight")); t.down_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_DOWN, (int) i, "bias")); t.norm = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_NORM, (int) i)); } // one AdaLN block per flow depth, the count is only known from the tensors for (int il = 0; ; il++) { ggml_tensor * probe = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_NORM, il, "weight"), false); if (probe == nullptr) { break; } mtmd_flow_net::block blk; blk.norm_w = probe; blk.norm_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_NORM, il, "bias")); blk.up_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_UP, il, "weight")); blk.up_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_UP, il, "bias")); blk.down_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_DOWN, il, "weight")); blk.down_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_DOWN, il, "bias")); blk.ada_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_ADA, il, "weight")); blk.ada_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_ADA, il, "bias")); flow.blocks.push_back(blk); } model.gen_out_eos_w = get_tensor(mtmd_format(TN_A_GEN_OUT_EOS, "weight")); model.gen_out_eos_b = get_tensor(mtmd_format(TN_A_GEN_OUT_EOS, "bias")); model.gen_input_lin_w = get_tensor(mtmd_format(TN_A_GEN_INPUT_LINEAR, "weight")); model.gen_emb_mean = get_tensor(TN_A_GEN_EMB_MEAN); model.gen_emb_std = get_tensor(TN_A_GEN_EMB_STD); // mimi decoder model.gen_quant_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_OUT, "weight")); model.gen_upsample_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UPSAMPLE, "weight")); load_seanet(model.seanet, true); model.gen_tfm_layers.resize(hparams.n_layer); for (int il = 0; il < hparams.n_layer; il++) { auto & layer = model.gen_tfm_layers[il]; const char * p = "a.gen.wav.tfm"; layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, p, il, "weight")); layer.ln_1_b = get_tensor(mtmd_format(TN_LN_1, p, il, "bias")); layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, p, il, "weight")); layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, p, il, "weight")); layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, p, il, "weight")); layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight")); layer.ls_1_w = get_tensor(mtmd_format(TN_LS_1, p, il, "weight")); layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, p, il, "weight")); layer.ln_2_b = get_tensor(mtmd_format(TN_LN_2, p, il, "bias")); layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, p, il, "weight")); layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, p, il, "weight")); layer.ls_2_w = get_tensor(mtmd_format(TN_LS_2, p, il, "weight")); } for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { const char * name = gguf_get_tensor_name(ctx_gguf.get(), i); if (strncmp(name, "voice.", 6) != 0) { continue; } const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name); if (t->type != GGML_TYPE_F32 || ggml_n_dims(t) > 2 || t->ne[0] != model.gen_input_lin_w->ne[1]) { throw std::runtime_error(mtmd_format("%s: invalid voice tensor %s\n", __func__, name)); } auto & embd = model.gen_voices[name + 6].rows; embd.resize(ggml_nelements(t)); fin.seekg(tensor_offset.at(name), std::ios::beg); if (!fin.read(reinterpret_cast(embd.data()), ggml_nbytes(t))) { throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name)); } } if (!model.gen_voices.empty()) { LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size()); } } break; case PROJECTOR_TYPE_LFM3A_GEN: { // must match the "tts" chat template, see conversion/lfm3.py // note for contributor: this is one-off fix, do not repeat this for future models model.gen_voices = { {"female", {}}, {"male", {}} }; // depthformer, the generic loop above loaded the a.gen.code.blk.* tensors for (int il = 0; il < hparams.n_layer; ++il) { model.layers[il].attn_gate_w = get_tensor(mtmd_format(TN_ATTN_GATE, prefix, il, "weight")); } model.gen_code_proj_in_w = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "weight")); model.gen_code_proj_in_b = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "bias")); model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight")); // tied head model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); model.gen_code_norm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_NORM, "weight")); model.gen_code_head_norm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD_NORM, "weight")); model.gen_code_mask_embd = get_tensor(TN_A_GEN_CODE_MASK_EMBD); // codec decoder: FSQ level table, one projection over all codebooks, LFM3 decoder -> polar STFT frames auto & c2w = model.c2w; c2w.quant_first_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight")); c2w.quant_first_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight")); c2w.quant_first_out_b = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "bias")); c2w.tfm_in_proj_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_IN_PROJ, "weight")); c2w.tfm_in_proj_b = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_IN_PROJ, "bias")); c2w.tfm_out_proj_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "weight")); c2w.tfm_out_proj_b = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias")); c2w.tfm_output_norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight")); // each layer is a short conv block or a gated attention block, both with sandwich norms c2w.tfm_layers.resize(hparams.wav_tfm_n_layer); for (int il = 0; il < hparams.wav_tfm_n_layer; il++) { auto & layer = c2w.tfm_layers[il]; const char * p = "a.gen.wav.tfm"; layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, p, il, "weight")); layer.attn_post_norm_w = get_tensor(mtmd_format(TN_ATTN_POST_NORM, p, il, "weight")); layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, p, il, "weight")); layer.ff_post_norm_w = get_tensor(mtmd_format(TN_FFN_POST_NORM, p, il, "weight")); layer.ff_gate_w = get_tensor(mtmd_format(TN_FFN_GATE, p, il, "weight")); layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, p, il, "weight")); layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, p, il, "weight")); layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, p, il, "weight"), false); layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, p, il, "weight"), false); layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, p, il, "weight"), false); layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, p, il, "weight"), false); layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, p, il, "weight"), false); layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, p, il, "weight"), false); layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight"), false); layer.q_norm = get_tensor(mtmd_format(TN_ATTN_Q_NORM, p, il, "weight"), false); layer.k_norm = get_tensor(mtmd_format(TN_ATTN_K_NORM, p, il, "weight"), false); layer.attn_gate_w = get_tensor(mtmd_format(TN_ATTN_GATE, p, il, "weight"), false); const bool is_conv = layer.conv_pw1_w && layer.conv_dw_w && layer.conv_pw2_w; const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w && layer.q_norm && layer.k_norm && layer.attn_gate_w; if (is_conv == is_attn) { throw std::runtime_error(mtmd_format("%s: codec layer %d is neither a conv nor an attention block\n", __func__, il)); } if (is_attn) { const int d_head = layer.q_norm->ne[0]; hparams.wav_tfm_n_head = layer.q_w->ne[1] / d_head; hparams.wav_tfm_n_head_kv = layer.k_w->ne[1] / d_head; } } } break; case PROJECTOR_TYPE_QWEN3TTS_GEN: { // code_predictor, proj_in is absent when the talker and the predictor share the hidden size model.gen_code_proj_in_w = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "weight"), false); model.gen_code_proj_in_b = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "bias"), false); model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight")); model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight")); model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); model.gen_code_norm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_NORM, "weight")); // code2wav: RVQ codes -> raw PCM, lives in the same ctx as code_predictor load_code2wav(true); } break; case PROJECTOR_TYPE_ZONOS3_GEN: { model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight")); model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); if (model.gen_code_out_embd_w->ne[1] % mtmd_graph_zonos3_gen::n_codebooks != 0 || model.gen_code_head_w->ne[1] != model.gen_code_out_embd_w->ne[1]) { throw std::runtime_error(mtmd_format("%s: zonos3: unexpected codebook table shape\n", __func__)); } // voice.: the voice rows of the prompt for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { const char * name = gguf_get_tensor_name(ctx_gguf.get(), i); if (strncmp(name, "voice.", 6) != 0) { continue; } const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name); if (t->type != GGML_TYPE_F32 || ggml_n_dims(t) > 2 || t->ne[0] != model.gen_code_out_embd_w->ne[0]) { throw std::runtime_error(mtmd_format("%s: invalid voice tensor %s\n", __func__, name)); } auto & embd = model.gen_voices[name + 6].rows; embd.resize(ggml_nelements(t)); fin.seekg(tensor_offset.at(name), std::ios::beg); if (!fin.read(reinterpret_cast(embd.data()), ggml_nbytes(t))) { throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name)); } } LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size()); // ZAC codec decoder auto & z = model.zonos3; const std::string p = "a.gen.wav"; auto has = [&](const std::string & name) { return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0; }; auto load_convnext = [&](const std::string & q) { mtmd_zonos3_wav::convnext b; b.dw_w = get_tensor(q + ".dw.weight"); b.dw_b = get_tensor(q + ".dw.bias"); b.norm_w = get_tensor(q + ".norm.weight"); b.norm_b = get_tensor(q + ".norm.bias"); b.pw1_w = get_tensor(q + ".pw1.weight"); b.pw1_b = get_tensor(q + ".pw1.bias"); b.pw2_w = get_tensor(q + ".pw2.weight"); b.pw2_b = get_tensor(q + ".pw2.bias"); return b; }; z.sem_table = get_tensor(p + ".sem.table"); z.sem_bias = get_tensor(p + ".sem.bias"); for (int i = 0; has(mtmd_format("%s.sem.%d.dw.weight", p.c_str(), i)); i++) { z.sem.push_back(load_convnext(p + ".sem." + std::to_string(i))); } z.sem_out_w = get_tensor(p + ".sem.out.weight"); z.sem_out_b = get_tensor(p + ".sem.out.bias"); z.res_table = get_tensor(p + ".res.table"); z.res_bias = get_tensor(p + ".res.bias"); for (int s = 0; has(mtmd_format("%s.stage.%d.in.weight", p.c_str(), s)); s++) { const std::string q = p + ".stage." + std::to_string(s); mtmd_zonos3_wav::stage st; st.in_w = get_tensor(q + ".in.weight"); st.out_w = get_tensor(q + ".out.weight", false); for (int il = 0; has(mtmd_format("%s.blk.%d.qkv.weight", q.c_str(), il)); il++) { const std::string b = q + ".blk." + std::to_string(il); mtmd_zonos3_wav::layer l; l.norm1_w = get_tensor(b + ".norm1.weight"); l.norm1_b = get_tensor(b + ".norm1.bias"); l.qkv_w = get_tensor(b + ".qkv.weight"); l.wo_w = get_tensor(b + ".wo.weight"); l.attn_scale = get_tensor(b + ".attn_scale"); l.norm2_w = get_tensor(b + ".norm2.weight"); l.norm2_b = get_tensor(b + ".norm2.bias"); l.ffn1_w = get_tensor(b + ".ffn1.weight"); l.ffn2_w = get_tensor(b + ".ffn2.weight"); l.ffn_scale = get_tensor(b + ".ffn_scale"); st.layers.push_back(l); } z.stages.push_back(std::move(st)); } for (int i = 0; has(mtmd_format("%s.head.%d.dw.weight", p.c_str(), i)); i++) { z.head.push_back(load_convnext(p + ".head." + std::to_string(i))); } z.head_norm_w = get_tensor(p + ".head.norm.weight"); z.head_norm_b = get_tensor(p + ".head.norm.bias"); z.head_out_w = get_tensor(p + ".head.out.weight"); z.head_out_b = get_tensor(p + ".head.out.bias"); if (z.stages.size() != mtmd_graph_zonos3_gen::n_stages || z.sem.empty() || z.head.empty() || z.head_out_w->ne[1] != mtmd_graph_zonos3_gen::n_fft[0] + mtmd_graph_zonos3_gen::n_fft[1] + 4) { throw std::runtime_error(mtmd_format("%s: zonos3: unexpected codec decoder layout\n", __func__)); } // read on the host auto read_host = [&](const std::string & name, std::vector & dst) -> const ggml_tensor * { const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name.c_str()); if (!t || t->type != GGML_TYPE_F32 || ggml_n_dims(t) != 2) { throw std::runtime_error(mtmd_format("%s: invalid or missing tensor %s\n", __func__, name.c_str())); } dst.resize(ggml_nelements(t)); fin.seekg(tensor_offset.at(name), std::ios::beg); if (!fin.read(reinterpret_cast(dst.data()), ggml_nbytes(t))) { throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name.c_str())); } return t; }; const ggml_tensor * mixer = read_host(p + ".mixer.weight", z.mixer); const ggml_tensor * rs = read_host(p + ".resample.weight", z.resample); if (mixer->ne[1] != 2 || rs->ne[1] != mtmd_graph_zonos3_gen::rs_new || rs->ne[0] < mtmd_graph_zonos3_gen::rs_orig || (rs->ne[0] - mtmd_graph_zonos3_gen::rs_orig) % 2 != 0) { throw std::runtime_error(mtmd_format("%s: zonos3: unexpected mixer or resampler shape\n", __func__)); } z.fir_kernel = mixer->ne[0]; z.rs_new = rs->ne[1]; z.rs_kernel_len = rs->ne[0]; } break; case PROJECTOR_TYPE_DRAMA3_GEN: { // generic a.gen.code.blk.* tensors are loaded by the loop above model.drama3_layers.resize(hparams.n_layer); for (int il = 0; il < hparams.n_layer; ++il) { auto & tl = model.drama3_layers[il]; model.layers[il].attn_gate_w = get_tensor(mtmd_format(TN_ATTN_GATE, prefix, il, "weight")); tl.enorm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_ENORM, il, "weight")); tl.hnorm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HNORM, il, "weight")); tl.eh_proj_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EH_PROJ, il, "weight")); tl.embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_BLK_EMBD, il, "weight")); tl.out_norm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_BLK_OUT_NORM, il, "weight")); } model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight")); // merged 3D, one table per codebook model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight")); // merged 3D, codebooks 1.. model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); // backbone rows of the semantic band load_code2wav(false); } break; case PROJECTOR_TYPE_CHATTERBOX: { auto & c = model.cbx; auto has = [&](const std::string & name) { return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0; }; auto load_enc_layer = [&](const std::string & p, mtmd_chatterbox::enc_layer & l) { l.norm_mha_w = get_tensor(p + ".norm_mha.weight"); l.norm_mha_b = get_tensor(p + ".norm_mha.bias"); l.attn_q_w = get_tensor(p + ".self_attn.linear_q.weight"); l.attn_q_b = get_tensor(p + ".self_attn.linear_q.bias"); l.attn_k_w = get_tensor(p + ".self_attn.linear_k.weight"); l.attn_k_b = get_tensor(p + ".self_attn.linear_k.bias"); l.attn_v_w = get_tensor(p + ".self_attn.linear_v.weight"); l.attn_v_b = get_tensor(p + ".self_attn.linear_v.bias"); l.attn_pos_w = get_tensor(p + ".self_attn.linear_pos.weight"); l.attn_pos_bias_u = get_tensor(p + ".self_attn.pos_bias_u"); l.attn_pos_bias_v = get_tensor(p + ".self_attn.pos_bias_v"); l.attn_out_w = get_tensor(p + ".self_attn.linear_out.weight"); l.attn_out_b = get_tensor(p + ".self_attn.linear_out.bias"); l.norm_ff_w = get_tensor(p + ".norm_ff.weight"); l.norm_ff_b = get_tensor(p + ".norm_ff.bias"); l.ffn_1_w = get_tensor(p + ".feed_forward.w_1.weight"); l.ffn_1_b = get_tensor(p + ".feed_forward.w_1.bias"); l.ffn_2_w = get_tensor(p + ".feed_forward.w_2.weight"); l.ffn_2_b = get_tensor(p + ".feed_forward.w_2.bias"); }; auto load_causal = [&](const std::string & p, mtmd_chatterbox::causal_block & b) { b.conv_w = get_tensor(p + ".block.0.weight"); b.conv_b = get_tensor(p + ".block.0.bias"); b.norm_w = get_tensor(p + ".block.2.weight"); b.norm_b = get_tensor(p + ".block.2.bias"); }; auto load_resnet = [&](const std::string & p, mtmd_chatterbox::resnet & r) { load_causal(p + ".block1", r.block1); load_causal(p + ".block2", r.block2); r.mlp_w = get_tensor(p + ".mlp.1.weight"); r.mlp_b = get_tensor(p + ".mlp.1.bias"); r.res_conv_w = get_tensor(p + ".res_conv.weight"); r.res_conv_b = get_tensor(p + ".res_conv.bias"); }; auto load_tfm = [&](const std::string & p, mtmd_chatterbox::tfm_block & b) { b.norm1_w = get_tensor(p + ".norm1.weight"); b.norm1_b = get_tensor(p + ".norm1.bias"); b.attn_q_w = get_tensor(p + ".attn1.to_q.weight"); b.attn_k_w = get_tensor(p + ".attn1.to_k.weight"); b.attn_v_w = get_tensor(p + ".attn1.to_v.weight"); b.attn_out_w = get_tensor(p + ".attn1.to_out.0.weight"); b.attn_out_b = get_tensor(p + ".attn1.to_out.0.bias"); b.norm3_w = get_tensor(p + ".norm3.weight"); b.norm3_b = get_tensor(p + ".norm3.bias"); b.ff_in_w = get_tensor(p + ".ff.net.0.proj.weight"); b.ff_in_b = get_tensor(p + ".ff.net.0.proj.bias"); b.ff_out_w = get_tensor(p + ".ff.net.2.weight"); b.ff_out_b = get_tensor(p + ".ff.net.2.bias"); }; auto load_stage = [&](const std::string & p, mtmd_chatterbox::est_stage & s, bool boundary) { load_resnet(p + ".0", s.res); for (int j = 0; has(p + ".1." + std::to_string(j) + ".norm1.weight"); j++) { mtmd_chatterbox::tfm_block b; load_tfm(p + ".1." + std::to_string(j), b); s.tfm.push_back(b); } if (boundary) { s.conv_w = get_tensor(p + ".2.weight"); s.conv_b = get_tensor(p + ".2.bias"); } }; auto load_hres = [&](const std::string & p, mtmd_chatterbox::hift_res & r) { for (int j = 0; has(p + ".convs1." + std::to_string(j) + ".weight"); j++) { const std::string js = std::to_string(j); mtmd_chatterbox::hift_res_unit u; u.act1_alpha = get_tensor(p + ".activations1." + js + ".alpha"); u.act2_alpha = get_tensor(p + ".activations2." + js + ".alpha"); u.conv1_w = get_tensor(p + ".convs1." + js + ".weight"); u.conv1_b = get_tensor(p + ".convs1." + js + ".bias"); u.conv2_w = get_tensor(p + ".convs2." + js + ".weight"); u.conv2_b = get_tensor(p + ".convs2." + js + ".bias"); r.units.push_back(u); } }; // flow encoder c.input_embedding_w = get_tensor("a.gen.flow.input_embedding.weight"); c.spk_affine_w = get_tensor("a.gen.flow.spk_embed_affine_layer.weight"); c.spk_affine_b = get_tensor("a.gen.flow.spk_embed_affine_layer.bias"); c.embed_linear_w = get_tensor("a.gen.fenc.embed.out.0.weight"); c.embed_linear_b = get_tensor("a.gen.fenc.embed.out.0.bias"); c.embed_norm_w = get_tensor("a.gen.fenc.embed.out.1.weight"); c.embed_norm_b = get_tensor("a.gen.fenc.embed.out.1.bias"); c.pre_conv1_w = get_tensor("a.gen.fenc.pre_lookahead_layer.conv1.weight"); c.pre_conv1_b = get_tensor("a.gen.fenc.pre_lookahead_layer.conv1.bias"); c.pre_conv2_w = get_tensor("a.gen.fenc.pre_lookahead_layer.conv2.weight"); c.pre_conv2_b = get_tensor("a.gen.fenc.pre_lookahead_layer.conv2.bias"); for (int i = 0; has("a.gen.fenc.encoders." + std::to_string(i) + ".norm_mha.weight"); i++) { mtmd_chatterbox::enc_layer l; load_enc_layer("a.gen.fenc.encoders." + std::to_string(i), l); c.enc.push_back(l); } c.up_conv_w = get_tensor("a.gen.fenc.up_layer.conv.weight"); c.up_conv_b = get_tensor("a.gen.fenc.up_layer.conv.bias"); c.up_embed_linear_w = get_tensor("a.gen.fenc.up_embed.out.0.weight"); c.up_embed_linear_b = get_tensor("a.gen.fenc.up_embed.out.0.bias"); c.up_embed_norm_w = get_tensor("a.gen.fenc.up_embed.out.1.weight"); c.up_embed_norm_b = get_tensor("a.gen.fenc.up_embed.out.1.bias"); for (int i = 0; has("a.gen.fenc.up_encoders." + std::to_string(i) + ".norm_mha.weight"); i++) { mtmd_chatterbox::enc_layer l; load_enc_layer("a.gen.fenc.up_encoders." + std::to_string(i), l); c.up_enc.push_back(l); } c.after_norm_w = get_tensor("a.gen.fenc.after_norm.weight"); c.after_norm_b = get_tensor("a.gen.fenc.after_norm.bias"); c.encoder_proj_w = get_tensor("a.gen.flow.encoder_proj.weight"); c.encoder_proj_b = get_tensor("a.gen.flow.encoder_proj.bias"); // cfm estimator c.time_mlp_1_w = get_tensor("a.gen.est.time_mlp.linear_1.weight"); c.time_mlp_1_b = get_tensor("a.gen.est.time_mlp.linear_1.bias"); c.time_mlp_2_w = get_tensor("a.gen.est.time_mlp.linear_2.weight"); c.time_mlp_2_b = get_tensor("a.gen.est.time_mlp.linear_2.bias"); c.time_embed_mixer_w = get_tensor("a.gen.est.time_embed_mixer.weight", false); load_stage("a.gen.est.down_blocks.0", c.est_down, true); for (int i = 0; has("a.gen.est.mid_blocks." + std::to_string(i) + ".0.block1.block.0.weight"); i++) { mtmd_chatterbox::est_stage s; load_stage("a.gen.est.mid_blocks." + std::to_string(i), s, false); c.est_mid.push_back(std::move(s)); } load_stage("a.gen.est.up_blocks.0", c.est_up, true); load_causal("a.gen.est.final_block", c.est_final_block); c.est_final_proj_w = get_tensor("a.gen.est.final_proj.weight"); c.est_final_proj_b = get_tensor("a.gen.est.final_proj.bias"); // hift vocoder c.hift_pre_w = get_tensor("a.gen.hift.conv_pre.weight"); c.hift_pre_b = get_tensor("a.gen.hift.conv_pre.bias"); c.hift_post_w = get_tensor("a.gen.hift.conv_post.weight"); c.hift_post_b = get_tensor("a.gen.hift.conv_post.bias"); for (int i = 0; has("a.gen.hift.ups." + std::to_string(i) + ".weight"); i++) { const std::string is = std::to_string(i); mtmd_chatterbox::hift_up up; up.up_w = get_tensor("a.gen.hift.ups." + is + ".weight"); up.up_b = get_tensor("a.gen.hift.ups." + is + ".bias"); up.source_down_w = get_tensor("a.gen.hift.source_downs." + is + ".weight"); up.source_down_b = get_tensor("a.gen.hift.source_downs." + is + ".bias"); load_hres("a.gen.hift.source_resblocks." + is, up.source_res); load_hres("a.gen.hift.resblocks." + std::to_string(3 * i), up.res_0); load_hres("a.gen.hift.resblocks." + std::to_string(3 * i + 1), up.res_1); load_hres("a.gen.hift.resblocks." + std::to_string(3 * i + 2), up.res_2); c.hift_ups.push_back(std::move(up)); } for (int i = 0; has("a.gen.hift.f0_predictor.condnet." + std::to_string(i) + ".weight"); i += 2) { const std::string is = std::to_string(i); mtmd_chatterbox::f0_conv fc; fc.w = get_tensor("a.gen.hift.f0_predictor.condnet." + is + ".weight"); fc.b = get_tensor("a.gen.hift.f0_predictor.condnet." + is + ".bias"); c.f0_condnet.push_back(fc); } c.f0_classifier_w = get_tensor("a.gen.hift.f0_predictor.classifier.weight"); c.f0_classifier_b = get_tensor("a.gen.hift.f0_predictor.classifier.bias"); // s3 tokenizer c.s3tok_conv1_w = get_tensor("a.s3tok.encoder.conv1.weight"); c.s3tok_conv1_b = get_tensor("a.s3tok.encoder.conv1.bias"); c.s3tok_conv2_w = get_tensor("a.s3tok.encoder.conv2.weight"); c.s3tok_conv2_b = get_tensor("a.s3tok.encoder.conv2.bias"); for (int i = 0; has("a.s3tok.encoder.blocks." + std::to_string(i) + ".attn_ln.weight"); i++) { const std::string p = "a.s3tok.encoder.blocks." + std::to_string(i); mtmd_chatterbox::s3tok_block b; b.attn_ln_w = get_tensor(p + ".attn_ln.weight"); b.attn_ln_b = get_tensor(p + ".attn_ln.bias"); b.attn_q_w = get_tensor(p + ".attn.query.weight"); b.attn_q_b = get_tensor(p + ".attn.query.bias"); b.attn_k_w = get_tensor(p + ".attn.key.weight"); b.attn_v_w = get_tensor(p + ".attn.value.weight"); b.attn_v_b = get_tensor(p + ".attn.value.bias"); b.fsmn_w = get_tensor(p + ".attn.fsmn_block.weight"); b.attn_out_w = get_tensor(p + ".attn.out.weight"); b.attn_out_b = get_tensor(p + ".attn.out.bias"); b.mlp_ln_w = get_tensor(p + ".mlp_ln.weight"); b.mlp_ln_b = get_tensor(p + ".mlp_ln.bias"); b.mlp_in_w = get_tensor(p + ".mlp.0.weight"); b.mlp_in_b = get_tensor(p + ".mlp.0.bias"); b.mlp_out_w = get_tensor(p + ".mlp.2.weight"); b.mlp_out_b = get_tensor(p + ".mlp.2.bias"); c.s3tok_blocks.push_back(b); } c.s3tok_down_w = get_tensor("a.s3tok.quantizer._codebook.project_down.weight"); c.s3tok_down_b = get_tensor("a.s3tok.quantizer._codebook.project_down.bias"); // read on the host: embedding tables, source module for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta.get()); t; t = ggml_get_next_tensor(ctx_meta.get(), t)) { const std::string name = t->name; if (name.rfind("a.gen.code.", 0) == 0 || name.rfind("a.gen.t3.", 0) == 0 || name.rfind("a.gen.hift.m_source.", 0) == 0) { model.cbx_tensors[name] = get_tensor(name); } } // voice. holds the talker rows, voice..ref_* the flow decoder reference const int64_t n_embd_talker = model.cbx_tensors.at("a.gen.code.out_embd.weight")->ne[0]; for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { const std::string name = gguf_get_tensor_name(ctx_gguf.get(), i); if (name.rfind("voice.", 0) != 0) { continue; } const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name.c_str()); std::string voice = name.substr(6); std::vector ids; // the converter stores 1D tensors as F32, ids are exact void * dst = nullptr; bool ok = false; auto take = [&](const char * suffix) { const size_t n = strlen(suffix); if (voice.size() > n && voice.compare(voice.size() - n, n, suffix) == 0) { voice.resize(voice.size() - n); return true; } return false; }; if (take(".ref_tokens")) { ids.resize(ggml_nelements(t)); dst = ids.data(); ok = t->type == GGML_TYPE_F32; } else if (take(".ref_feat")) { auto & v = model.gen_voices[voice].ref.feat; v.resize(ggml_nelements(t)); dst = v.data(); ok = t->type == GGML_TYPE_F32 && t->ne[0] == 80; } else if (take(".ref_spk")) { auto & v = model.gen_voices[voice].ref.spk; v.resize(ggml_nelements(t)); dst = v.data(); ok = t->type == GGML_TYPE_F32 && t->ne[0] == 192; } else { auto & v = model.gen_voices[voice].rows; v.resize(ggml_nelements(t)); dst = v.data(); ok = t->type == GGML_TYPE_F32 && ggml_n_dims(t) <= 2 && t->ne[0] == n_embd_talker; } if (!ok) { throw std::runtime_error(mtmd_format("%s: invalid voice tensor %s\n", __func__, name.c_str())); } fin.seekg(tensor_offset.at(name), std::ios::beg); if (!fin.read(reinterpret_cast(dst), ggml_nbytes(t))) { throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name.c_str())); } if (!ids.empty()) { model.gen_voices[voice].ref.tokens.assign(ids.begin(), ids.end()); } } for (const auto & [voice, v] : model.gen_voices) { if (v.rows.empty() || v.ref.tokens.empty() || v.ref.feat.size() != 2 * 80 * v.ref.tokens.size() || v.ref.spk.empty()) { throw std::runtime_error(mtmd_format("%s: incomplete voice %s\n", __func__, voice.c_str())); } } } break; case PROJECTOR_TYPE_ZONOS3_SPKENC: { // the graph reads the tensors by name for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { const char * name = gguf_get_tensor_name(ctx_gguf.get(), i); if (strncmp(name, "a.spk.", 6) == 0) { model.zonos3_spk[name] = get_tensor(name); } } if (!model.zonos3_spk.count("a.spk.proj.weight") || !model.zonos3_spk.count("a.spk.linear.weight")) { throw std::runtime_error(mtmd_format("%s: zonos3: missing speaker encoder tensors\n", __func__)); } } break; case PROJECTOR_TYPE_CHATTERBOX_SPKENC: { auto & c = model.cbx; auto has = [&](const std::string & name) { return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0; }; auto load_bn = [&](const std::string & p, mtmd_chatterbox::bn & n) { n.mean = get_tensor(p + ".running_mean"); n.var = get_tensor(p + ".running_var"); n.w = get_tensor(p + ".weight", false); n.b = get_tensor(p + ".bias", false); }; auto load_res2d = [&](const std::string & p, mtmd_chatterbox::spk_res2d & r) { r.conv1_w = get_tensor(p + ".conv1.weight"); load_bn(p + ".bn1", r.bn1); r.conv2_w = get_tensor(p + ".conv2.weight"); load_bn(p + ".bn2", r.bn2); r.shortcut_w = get_tensor(p + ".shortcut.0.weight", false); if (r.shortcut_w) { load_bn(p + ".shortcut.1", r.shortcut_bn); } }; auto load_cam_block = [&](const std::string & bp, const std::string & tp, mtmd_chatterbox::spk_cam_block & blk) { for (int li = 1; has(bp + ".tdnnd" + std::to_string(li) + ".linear1.weight"); li++) { const std::string p = bp + ".tdnnd" + std::to_string(li); mtmd_chatterbox::spk_cam_layer l; load_bn(p + ".nonlinear1.batchnorm", l.nl1_bn); l.linear1_w = get_tensor(p + ".linear1.weight"); load_bn(p + ".nonlinear2.batchnorm", l.nl2_bn); l.local_w = get_tensor(p + ".cam_layer.linear_local.weight"); l.ctx1_w = get_tensor(p + ".cam_layer.linear1.weight"); l.ctx1_b = get_tensor(p + ".cam_layer.linear1.bias"); l.ctx2_w = get_tensor(p + ".cam_layer.linear2.weight"); l.ctx2_b = get_tensor(p + ".cam_layer.linear2.bias"); blk.layers.push_back(l); } load_bn(tp + ".nonlinear.batchnorm", blk.transit_bn); blk.transit_w = get_tensor(tp + ".linear.weight"); }; c.spk_conv1_w = get_tensor("a.spk.head.conv1.weight"); load_bn("a.spk.head.bn1", c.spk_bn1); load_res2d("a.spk.head.layer1.0", c.spk_layer1_0); load_res2d("a.spk.head.layer1.1", c.spk_layer1_1); load_res2d("a.spk.head.layer2.0", c.spk_layer2_0); load_res2d("a.spk.head.layer2.1", c.spk_layer2_1); c.spk_conv2_w = get_tensor("a.spk.head.conv2.weight"); load_bn("a.spk.head.bn2", c.spk_bn2); c.spk_tdnn_w = get_tensor("a.spk.xvector.tdnn.linear.weight"); load_bn("a.spk.xvector.tdnn.nonlinear.batchnorm", c.spk_tdnn_bn); load_cam_block("a.spk.xvector.block1", "a.spk.xvector.transit1", c.spk_block1); load_cam_block("a.spk.xvector.block2", "a.spk.xvector.transit2", c.spk_block2); load_cam_block("a.spk.xvector.block3", "a.spk.xvector.transit3", c.spk_block3); load_bn("a.spk.xvector.out_nonlinear.batchnorm", c.spk_out_bn); c.spk_dense_w = get_tensor("a.spk.xvector.dense.linear.weight"); load_bn("a.spk.xvector.dense.nonlinear.batchnorm", c.spk_dense_bn); // read on the host: voice encoder lstm, conditioning encoder for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta.get()); t; t = ggml_get_next_tensor(ctx_meta.get(), t)) { const std::string name = t->name; if (name.rfind("a.ve.", 0) == 0 || name.rfind("a.cenc.", 0) == 0) { model.cbx_tensors[name] = get_tensor(name); } } } break; case PROJECTOR_TYPE_SOPRANO: { auto & v = model.vocos; v.input_w = get_tensor(mtmd_format(TN_A_GEN_WAV_INPUT, "weight")); v.input_b = get_tensor(mtmd_format(TN_A_GEN_WAV_INPUT, "bias")); v.norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_NORM, "weight")); v.norm_b = get_tensor(mtmd_format(TN_A_GEN_WAV_NORM, "bias")); v.output_norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_OUTPUT_NORM, "weight")); v.output_norm_b = get_tensor(mtmd_format(TN_A_GEN_WAV_OUTPUT_NORM, "bias")); v.output_w = get_tensor(mtmd_format(TN_A_GEN_WAV_OUTPUT, "weight")); v.output_b = get_tensor(mtmd_format(TN_A_GEN_WAV_OUTPUT, "bias")); v.blocks.resize(hparams.n_layer); for (int il = 0; il < hparams.n_layer; ++il) { auto & b = v.blocks[il]; b.dwconv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight")); b.dwconv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias")); b.norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_NORM, il, "weight")); b.norm_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_NORM, il, "bias")); b.pw1_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW1, il, "weight")); b.pw1_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW1, il, "bias")); b.pw2_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW2, il, "weight")); b.pw2_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW2, il, "bias")); b.gamma = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_GAMMA, il)); } } break; case PROJECTOR_TYPE_VOXTRAL: case PROJECTOR_TYPE_VOXTRAL_RT: { model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); } break; case PROJECTOR_TYPE_MUSIC_FLAMINGO: { model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias")); } break; case PROJECTOR_TYPE_INTERNVL: { model.mm_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "bias")); model.mm_3_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "weight")); model.mm_3_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "bias")); } break; case PROJECTOR_TYPE_NEMOTRON_V2_VL: case PROJECTOR_TYPE_NEMOTRON_OMNI: { model.mm_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "weight")); model.mm_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "weight")); model.mm_3_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "weight")); } break; case PROJECTOR_TYPE_GLMA: { model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias")); model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_NORM_PRE, "weight")); model.mm_norm_pre_b = get_tensor(mtmd_format(TN_MM_NORM_PRE, "bias")); model.mm_boi = get_tensor(mtmd_format(TN_TOK_BOI)); model.mm_eoi = get_tensor(mtmd_format(TN_TOK_EOI)); } break; case PROJECTOR_TYPE_LLAMA4: { model.mm_model_proj = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight")); model.mm_model_mlp_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "weight")); model.mm_model_mlp_2_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 2, "weight")); } break; case PROJECTOR_TYPE_COGVLM: { model.mm_model_proj = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight")); model.mm_post_fc_norm_w = get_tensor(mtmd_format(TN_MM_POST_FC_NORM, "weight")); model.mm_post_fc_norm_b = get_tensor(mtmd_format(TN_MM_POST_FC_NORM, "bias")); model.mm_h_to_4h_w = get_tensor(mtmd_format(TN_MM_H_TO_4H, "weight")); model.mm_gate_w = get_tensor(mtmd_format(TN_MM_GATE, "weight")); model.mm_4h_to_h_w = get_tensor(mtmd_format(TN_MM_4H_TO_H, "weight")); model.mm_boi = get_tensor(TN_TOK_BOI); model.mm_eoi = get_tensor(TN_TOK_EOI); } break; case PROJECTOR_TYPE_HUNYUANVL: { // proj.0 -> mm.0 (conv1), proj.2 -> mm.2 (conv2), mlp -> mm.model.fc (linear) model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); model.mm_model_proj = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight")); model.mm_model_proj_b = get_tensor(mtmd_format(TN_MM_PROJECTOR, "bias")); model.mm_pre_norm_w = get_tensor(mtmd_format(TN_MM_PRE_NORM, "weight")); model.mm_post_norm_w = get_tensor(mtmd_format(TN_MM_POST_NORM, "weight")); model.mm_img_begin = get_tensor(TN_MM_IMG_BEGIN); model.mm_img_end = get_tensor(TN_MM_IMG_END); model.image_newline = get_tensor(TN_IMAGE_NEWLINE); model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR, false); // the position embedding is resampled on the host as F32 if (!model.position_embeddings || model.position_embeddings->type != GGML_TYPE_F32) { throw std::runtime_error(mtmd_format("%s: HunyuanVL requires an F32 %s\n", __func__, mtmd_format(TN_POS_EMBD, prefix).c_str())); } } break; case PROJECTOR_TYPE_JANUS_PRO: { model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); } break; case PROJECTOR_TYPE_PHI4: { model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); } break; case PROJECTOR_TYPE_DEEPSEEKOCR: case PROJECTOR_TYPE_DEEPSEEKOCR2: { model.pos_embed = get_tensor(mtmd_format(TN_SAM_POS_EMBD, "weight")); model.patch_embed_proj_w = get_tensor(mtmd_format(TN_SAM_PATCH_EMBD, "weight")); model.patch_embed_proj_b = get_tensor(mtmd_format(TN_SAM_PATCH_EMBD, "bias")); model.n_sam_layers = hparams.sam_n_layer; model.sam_layers.resize(model.n_sam_layers); for (int il = 0; il < model.n_sam_layers; ++il) { auto & layer = model.sam_layers[il]; layer.qkv_w = get_tensor(mtmd_format(TN_SAM_ATTN_QKV, il, "weight")); layer.qkv_b = get_tensor(mtmd_format(TN_SAM_ATTN_QKV, il, "bias")); layer.o_w = get_tensor(mtmd_format(TN_SAM_ATTN_OUT, il, "weight")); layer.o_b = get_tensor(mtmd_format(TN_SAM_ATTN_OUT, il, "bias")); layer.ln_1_w = get_tensor(mtmd_format(TN_SAM_PRE_NORM, il, "weight")); layer.ln_1_b = get_tensor(mtmd_format(TN_SAM_PRE_NORM, il, "bias")); layer.ln_2_w = get_tensor(mtmd_format(TN_SAM_POST_NORM, il, "weight")); layer.ln_2_b = get_tensor(mtmd_format(TN_SAM_POST_NORM, il, "bias")); layer.rel_pos_h = get_tensor(mtmd_format(TN_SAM_ATTN_POS_H, il, "weight")); layer.rel_pos_w = get_tensor(mtmd_format(TN_SAM_ATTN_POS_W, il, "weight")); layer.ff_up_w = get_tensor(mtmd_format(TN_SAM_FFN_UP, il, "weight")); layer.ff_up_b = get_tensor(mtmd_format(TN_SAM_FFN_UP, il, "bias")); layer.ff_down_w = get_tensor(mtmd_format(TN_SAM_FFN_DOWN, il, "weight")); layer.ff_down_b = get_tensor(mtmd_format(TN_SAM_FFN_DOWN, il, "bias")); } model.neck_0_w = get_tensor(mtmd_format(TN_SAM_NECK, 0, "weight")); model.neck_1_b = get_tensor(mtmd_format(TN_SAM_NECK, 1, "bias")); model.neck_1_w = get_tensor(mtmd_format(TN_SAM_NECK, 1, "weight")); model.neck_2_w = get_tensor(mtmd_format(TN_SAM_NECK, 2, "weight")); model.neck_3_b = get_tensor(mtmd_format(TN_SAM_NECK, 3, "bias")); model.neck_3_w = get_tensor(mtmd_format(TN_SAM_NECK, 3, "weight")); model.net_2 = get_tensor(mtmd_format(TN_SAM_NET, 2, "weight")); model.net_3 = get_tensor(mtmd_format(TN_SAM_NET, 3, "weight")); model.image_newline = get_tensor(TN_IMAGE_NEWLINE, false); model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR); model.mm_fc_w = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight")); model.mm_fc_b = get_tensor(mtmd_format(TN_MM_PROJECTOR, "bias")); model.resample_query_768 = get_tensor(mtmd_format(TN_RESMPL_QUERY, 768, "weight"), false); model.resample_query_1024 = get_tensor(mtmd_format(TN_RESMPL_QUERY, 1024, "weight"), false); } break; case PROJECTOR_TYPE_GEMMA4A: { for (int i = 0; i < 2; i++) { model.sscp_conv_w[i] = get_tensor(mtmd_format(TN_A_CONV1D, i, "weight")); model.sscp_conv_b[i] = get_tensor(mtmd_format(TN_A_CONV1D, i, "bias"), false); model.sscp_norm_w[i] = get_tensor(mtmd_format(TN_A_CONV1D_NORM, i, "weight"), false); } model.sscp_inp_proj_w = get_tensor(mtmd_format(TN_A_INP_PROJ, "weight")); model.sscp_inp_proj_b = get_tensor(mtmd_format(TN_A_INP_PROJ, "bias"), false); model.audio_out_proj_w = get_tensor(mtmd_format(TN_A_OUT_PROJ, "weight"), false); model.audio_out_proj_b = get_tensor(mtmd_format(TN_A_OUT_PROJ, "bias"), false); // audio multimodal embedder (mm.a.* namespace, not mm.*) model.mm_soft_emb_norm_w = get_tensor(mtmd_format(TN_A_MM_SOFT_EMB_N, "weight"), false); model.mm_input_proj_w = get_tensor(mtmd_format(TN_A_MM_INP_PROJ, "weight"), false); // Per-layer tensors NOT loaded by the generic loop above for (int il = 0; il < hparams.n_layer; ++il) { auto & layer = model.layers[il]; // Gemma4 audio conformer-specific tensors layer.ff_norm_w = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "weight")); layer.attn_pre_norm_w = get_tensor(mtmd_format(TN_A_ATTN_PRE_NORM, prefix, il, "weight"), false); layer.per_dim_scale_w = get_tensor(mtmd_format(TN_A_PER_DIM_SCALE, prefix, il, "weight"), false); layer.per_dim_k_scale_w = get_tensor(mtmd_format(TN_A_PER_DIM_K_SCALE, prefix, il, "weight"), false); layer.attn_k_rel_w = get_tensor(mtmd_format(TN_A_ATTN_K_REL, prefix, il, "weight"), false); // Convolution module // Note: conv_norm / norm_conv are swapped in GGUF due to // upstream tensor_mapping.py, so we load them in reverse order layer.norm_conv_w = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "weight"), false); layer.norm_conv_b = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "bias"), false); layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "weight")); layer.conv_pw1_b = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "bias"), false); layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "weight")); layer.conv_dw_b = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "bias"), false); layer.conv_norm_w = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "weight"), false); layer.conv_norm_b = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "bias"), false); layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "weight")); layer.conv_pw2_b = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "bias"), false); // FFN2 (second half-step) layer.ff_norm_1_w = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "weight")); layer.ff_up_1_w = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "weight")); layer.ff_up_1_b = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "bias"), false); layer.ff_down_1_w = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "weight")); layer.ff_down_1_b = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "bias"), false); layer.ff_post_norm_1_w = get_tensor(mtmd_format(TN_A_FFN_POST_NORM_1, prefix, il, "weight"), false); } // Load clamp info for ClippableLinear AFTER all tensors are loaded for (auto * tensor : tensors_to_load) { std::string name = tensor->name; if (string_ends_with(name, ".weight")) { std::string name_inp_max = name; std::string name_inp_min = name; std::string name_out_max = name; std::string name_out_min = name; string_replace_all(name_inp_max, ".weight", ".input_max"); string_replace_all(name_inp_min, ".weight", ".input_min"); string_replace_all(name_out_max, ".weight", ".output_max"); string_replace_all(name_out_min, ".weight", ".output_min"); model.clamp_info_map[name] = { get_scalar(name_inp_max, FLT_MAX), get_scalar(name_inp_min, -FLT_MAX), get_scalar(name_out_max, FLT_MAX), get_scalar(name_out_min, -FLT_MAX) }; } } } break; case PROJECTOR_TYPE_GEMMA4UA: { model.mm_input_proj_w = get_tensor(mtmd_format(TN_A_MM_INP_PROJ, "weight")); } break; case PROJECTOR_TYPE_LFM2A: case PROJECTOR_TYPE_D1OMNI_A: { for (int i : {0, 2, 3, 5, 6}) { model.pre_encode_conv_X_w[i] = get_tensor(mtmd_format(TN_CONV1D, i, "weight")); model.pre_encode_conv_X_b[i] = get_tensor(mtmd_format(TN_CONV1D, i, "bias")); } model.pre_encode_out_w = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "weight")); model.pre_encode_out_b = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "bias")); model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight")); model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias")); // residual block after the projector: norm, down, up if (model.proj_type == PROJECTOR_TYPE_D1OMNI_A) { model.mm_4_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 4, "weight")); model.mm_4_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 4, "bias")); model.mm_5_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 5, "weight")); model.mm_5_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 5, "bias")); model.mm_6_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 6, "weight")); model.mm_6_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 6, "bias")); } for (int il = 0; il < hparams.n_layer; ++il) { auto & layer = model.layers[il]; layer.ff_norm_w = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "weight")); layer.ff_norm_b = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "bias")); layer.ff_norm_1_w = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "weight")); layer.ff_norm_1_b = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "bias")); layer.ff_up_1_w = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "weight")); layer.ff_up_1_b = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "bias")); layer.ff_down_1_w = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "weight")); layer.ff_down_1_b = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "bias")); layer.pos_bias_u = get_tensor(mtmd_format(TN_POS_BIAS_U, prefix, il)); layer.pos_bias_v = get_tensor(mtmd_format(TN_POS_BIAS_V, prefix, il)); layer.norm_conv_w = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "weight")); layer.norm_conv_b = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "bias")); layer.linear_pos_w = get_tensor(mtmd_format(TN_LINEAR_POS, prefix, il, "weight")); layer.conv_norm_w = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "weight")); layer.conv_norm_b = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "bias")); layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "weight")); layer.conv_dw_b = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "bias")); layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "weight")); layer.conv_pw1_b = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "bias")); layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "weight")); layer.conv_pw2_b = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "bias")); } } break; case PROJECTOR_TYPE_LFM3A: { for (int i : {0, 2, 3, 5, 6}) { model.pre_encode_conv_X_w[i] = get_tensor(mtmd_format(TN_CONV1D, i, "weight")); model.pre_encode_conv_X_b[i] = get_tensor(mtmd_format(TN_CONV1D, i, "bias")); } model.pre_encode_out_w = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "weight")); model.pre_encode_out_b = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "bias")); model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias")); model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight")); model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias")); if (!model.post_ln_w) { throw std::runtime_error(mtmd_format("%s: missing tensor %s\n", __func__, mtmd_format(TN_LN_POST, prefix, "weight").c_str())); } // each layer is either a short conv block or a complete attention block for (int il = 0; il < hparams.n_layer; ++il) { auto & layer = model.layers[il]; layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "weight"), false); layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "weight"), false); layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "weight"), false); const bool is_conv = layer.conv_pw1_w && layer.conv_dw_w && layer.conv_pw2_w; const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w && layer.q_norm && layer.k_norm; if (is_conv == is_attn) { throw std::runtime_error(mtmd_format("%s: lfm3a layer %d is neither a complete conv nor a complete attention block\n", __func__, il)); } } } break; case PROJECTOR_TYPE_PARAKEET: { hparams.mel_filters = get_vector(TN_MEL_FILTERS); hparams.window = get_vector(TN_WINDOW); // Subsampling layers (conv1d) for (int i : {0, 2, 3, 5, 6}) { model.pre_encode_conv_X_w[i] = get_tensor(mtmd_format(TN_CONV1D, i, "weight")); model.pre_encode_conv_X_b[i] = get_tensor(mtmd_format(TN_CONV1D, i, "bias")); } model.pre_encode_out_w = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "weight")); model.pre_encode_out_b = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "bias")); // Projection layers model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_NORM_PRE, "weight"), false); model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"), false); model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"), false); // Encoder layers for (int il = 0; il < hparams.n_layer; ++il) { auto & layer = model.layers[il]; // Attention (from shared above) // Relative position encoding layer.linear_pos_w = get_tensor(mtmd_format(TN_LINEAR_POS, prefix, il, "weight")); layer.pos_bias_u = get_tensor(mtmd_format(TN_POS_BIAS_U, prefix, il)); layer.pos_bias_v = get_tensor(mtmd_format(TN_POS_BIAS_V, prefix, il)); // Convolution module layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "weight")); layer.conv_pw1_b = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "bias"), false); layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "weight")); layer.conv_dw_b = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "bias"), false); layer.conv_norm_w = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "weight")); layer.conv_norm_b = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "bias")); layer.conv_norm_mean = get_tensor(mtmd_format(TN_CONV_NORM_MEAN, prefix, il)); layer.conv_norm_var = get_tensor(mtmd_format(TN_CONV_NORM_VAR, prefix, il)); layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "weight")); layer.conv_pw2_b = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "bias"), false); // Feed-forward networks layer.ff_norm_w = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "weight")); layer.ff_norm_b = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "bias")); layer.ff_norm_1_w = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "weight")); layer.ff_norm_1_b = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "bias")); layer.ff_up_1_w = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "weight")); layer.ff_up_1_b = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "bias"), false); layer.ff_down_1_w = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "weight")); layer.ff_down_1_b = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "bias"), false); // Layer norms layer.norm_conv_w = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "weight")); layer.norm_conv_b = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "bias")); } model.mm_model_mlp_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "weight")); model.mm_model_mlp_2_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "weight")); model.mm_model_mlp_3_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "weight")); } break; case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: { hparams.mel_filters = get_vector(TN_MEL_FILTERS); hparams.window = get_vector(TN_WINDOW); // ASR expert: blocks, pre_ln and post_ln come from the generic loader model.inp_proj_w = get_tensor(mtmd_format(TN_INP_PROJ, "weight")); for (int il = 0; il < hparams.n_layer; ++il) { auto & layer = model.layers[il]; layer.q_norm_b = get_tensor(mtmd_format(TN_ATTN_Q_NORM, prefix, il, "bias")); layer.k_norm_b = get_tensor(mtmd_format(TN_ATTN_K_NORM, prefix, il, "bias")); } // speaker expert const char * sp = "a.spk"; model.spk_inp_proj_w = get_tensor(mtmd_format(TN_A_SPK_INP_PROJ, "weight")); model.spk_pre_ln_w = get_tensor(mtmd_format(TN_LN_PRE, sp, "weight")); model.spk_pre_ln_b = get_tensor(mtmd_format(TN_LN_PRE, sp, "bias")); model.spk_post_ln_w = get_tensor(mtmd_format(TN_LN_POST, sp, "weight")); model.spk_post_ln_b = get_tensor(mtmd_format(TN_LN_POST, sp, "bias")); model.spk_layers.resize(hparams.spk_n_layer); for (int il = 0; il < hparams.spk_n_layer; ++il) { auto & layer = model.spk_layers[il]; layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, sp, il, "weight")); layer.ln_1_b = get_tensor(mtmd_format(TN_LN_1, sp, il, "bias")); layer.qkv_w = get_tensor(mtmd_format(TN_ATTN_QKV, sp, il, "weight")); layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, sp, il, "weight")); layer.o_b = get_tensor(mtmd_format(TN_ATTN_OUTPUT, sp, il, "bias")); layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, sp, il, "weight")); layer.ln_2_b = get_tensor(mtmd_format(TN_LN_2, sp, il, "bias")); layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, sp, il, "weight")); layer.ff_up_b = get_tensor(mtmd_format(TN_FFN_UP, sp, il, "bias")); layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, sp, il, "weight")); layer.ff_down_b = get_tensor(mtmd_format(TN_FFN_DOWN, sp, il, "bias")); } // Sortformer head, fusion model.spk_head_proj_w = get_tensor(mtmd_format(TN_A_SPK_HEAD, "proj", "weight")); model.spk_head_proj_b = get_tensor(mtmd_format(TN_A_SPK_HEAD, "proj", "bias")); model.spk_head_up_w = get_tensor(mtmd_format(TN_A_SPK_HEAD, "upsample", "weight")); model.spk_head_up_b = get_tensor(mtmd_format(TN_A_SPK_HEAD, "upsample", "bias")); model.spk_head_fc1_w = get_tensor(mtmd_format(TN_A_SPK_HEAD, "fc1", "weight")); model.spk_head_fc1_b = get_tensor(mtmd_format(TN_A_SPK_HEAD, "fc1", "bias")); model.spk_head_fc2_w = get_tensor(mtmd_format(TN_A_SPK_HEAD, "fc2", "weight")); model.spk_head_fc2_b = get_tensor(mtmd_format(TN_A_SPK_HEAD, "fc2", "bias")); model.fusion_asr_norm_w = get_tensor(mtmd_format(TN_A_FUSION_NORM, "asr_norm", "weight")); model.fusion_asr_norm_b = get_tensor(mtmd_format(TN_A_FUSION_NORM, "asr_norm", "bias")); model.fusion_spk_norm_w = get_tensor(mtmd_format(TN_A_FUSION_NORM, "spk_norm", "weight")); model.fusion_spk_norm_b = get_tensor(mtmd_format(TN_A_FUSION_NORM, "spk_norm", "bias")); model.fusion_spk_kernel = get_tensor(TN_A_FUSION_KERNEL); model.spk_sil_emb = get_vector(TN_A_SPK_SIL_EMB); // the head layouts and the widths the graph and the speaker state use must match the tensors hparams.spk_n_embd = model.spk_inp_proj_w->ne[1]; const int64_t d_head = hparams.n_head > 0 ? hparams.n_embd / hparams.n_head : 0; const int64_t d_head_spk = hparams.spk_n_head > 0 ? hparams.spk_n_embd / hparams.spk_n_head : 0; const int64_t n_spk = model.spk_head_fc2_w->ne[1]; // the graph splits each fused QKV into three views of the layer width bool qkv_ok = true; for (const auto & layer : model.layers) { qkv_ok = qkv_ok && layer.qkv_w && layer.qkv_w->ne[0] == hparams.n_embd && layer.qkv_w->ne[1] == 3 * hparams.n_embd; } for (const auto & layer : model.spk_layers) { qkv_ok = qkv_ok && layer.qkv_w->ne[0] == hparams.spk_n_embd && layer.qkv_w->ne[1] == 3 * hparams.spk_n_embd; } if (!qkv_ok || model.inp_proj_w->ne[1] != hparams.n_embd || d_head <= 0 || d_head * hparams.n_head != hparams.n_embd || (hparams.n_embd_head > 0 && hparams.n_embd_head != d_head) || hparams.audio_rope_dims <= 0 || hparams.audio_rope_dims % 2 != 0 || hparams.audio_rope_dims > d_head || d_head_spk <= 0 || d_head_spk % 2 != 0 || d_head_spk * hparams.spk_n_head != hparams.spk_n_embd || (int64_t) model.spk_sil_emb.size() != hparams.spk_n_embd || n_spk <= 0 || model.fusion_spk_kernel->ne[0] != n_spk || model.fusion_spk_kernel->ne[1] != hparams.n_embd || hparams.mel_filters.size() != (size_t) hparams.n_mel_bins * (hparams.audio_n_fft / 2 + 1) || hparams.window.size() != (size_t) hparams.audio_window_len) { throw std::runtime_error(mtmd_format("%s: hparams do not match the tensor shapes\n", __func__)); } // projection to the text model model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight")); model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias")); } break; case PROJECTOR_TYPE_GRANITE_SPEECH: { model.inp_proj_w = get_tensor(mtmd_format(TN_INP_PROJ, "weight")); model.inp_proj_b = get_tensor(mtmd_format(TN_INP_PROJ, "bias")); model.ctc_out_w = get_tensor(mtmd_format(TN_CTC_OUT, "weight")); model.ctc_out_b = get_tensor(mtmd_format(TN_CTC_OUT, "bias")); model.ctc_out_mid_w = get_tensor(mtmd_format(TN_CTC_OUT_MID, "weight")); model.ctc_out_mid_b = get_tensor(mtmd_format(TN_CTC_OUT_MID, "bias")); // per-layer tensors not loaded by the generic loop above for (int il = 0; il < hparams.n_layer; ++il) { auto & layer = model.layers[il]; layer.attn_rel_pos_emb = get_tensor(mtmd_format(TN_ATTN_REL_POS_EMB, prefix, il)); layer.ff_norm_w = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "weight")); layer.ff_norm_b = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "bias")); layer.ff_norm_1_w = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "weight")); layer.ff_norm_1_b = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "bias")); layer.ff_up_1_w = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "weight")); layer.ff_up_1_b = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "bias")); layer.ff_down_1_w = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "weight")); layer.ff_down_1_b = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "bias")); layer.norm_conv_w = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "weight")); layer.norm_conv_b = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "bias")); layer.conv_norm_w = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "weight")); layer.conv_norm_b = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "bias")); layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "weight")); layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "weight")); layer.conv_pw1_b = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "bias")); layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "weight")); layer.conv_pw2_b = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "bias")); } model.qf_proj_blocks.resize(1); auto & qf = model.qf_proj_blocks[0]; qf.qf_proj_query = get_tensor(mtmd_format(TN_QF_PROJ_QUERY, prefix)); qf.qf_proj_norm_w = get_tensor(mtmd_format(TN_QF_PROJ_NORM, prefix, "weight")); qf.qf_proj_norm_b = get_tensor(mtmd_format(TN_QF_PROJ_NORM, prefix, "bias")); qf.qf_proj_linear_w = get_tensor(mtmd_format(TN_QF_PROJ_LINEAR, prefix, "weight")); qf.qf_proj_linear_b = get_tensor(mtmd_format(TN_QF_PROJ_LINEAR, prefix, "bias")); const int n_proj_layers = 2; qf.qf_proj_layers.resize(n_proj_layers); for (int il = 0; il < n_proj_layers; ++il) { auto & pl = qf.qf_proj_layers[il]; pl.q_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_Q, prefix, il, "weight")); pl.q_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_Q, prefix, il, "bias")); pl.k_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_K, prefix, il, "weight")); pl.k_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_K, prefix, il, "bias")); pl.v_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_V, prefix, il, "weight")); pl.v_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_V, prefix, il, "bias")); pl.o_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_O, prefix, il, "weight")); pl.o_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_O, prefix, il, "bias")); pl.ln_1_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_N, prefix, il, "weight")); pl.ln_1_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_N, prefix, il, "bias")); pl.cross_attn_q_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_Q, prefix, il, "weight")); pl.cross_attn_q_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_Q, prefix, il, "bias")); pl.cross_attn_k_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_K, prefix, il, "weight")); pl.cross_attn_k_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_K, prefix, il, "bias")); pl.cross_attn_v_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_V, prefix, il, "weight")); pl.cross_attn_v_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_V, prefix, il, "bias")); pl.cross_attn_o_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_O, prefix, il, "weight")); pl.cross_attn_o_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_O, prefix, il, "bias")); pl.cross_attn_norm_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_N, prefix, il, "weight")); pl.cross_attn_norm_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_N, prefix, il, "bias")); pl.ff_up_w = get_tensor(mtmd_format(TN_QF_FFN_UP, prefix, il, "weight")); pl.ff_up_b = get_tensor(mtmd_format(TN_QF_FFN_UP, prefix, il, "bias")); pl.ff_down_w = get_tensor(mtmd_format(TN_QF_FFN_DOWN, prefix, il, "weight")); pl.ff_down_b = get_tensor(mtmd_format(TN_QF_FFN_DOWN, prefix, il, "bias")); pl.ln_2_w = get_tensor(mtmd_format(TN_QF_FFN_NORM, prefix, il, "weight")); pl.ln_2_b = get_tensor(mtmd_format(TN_QF_FFN_NORM, prefix, il, "bias")); } } break; case PROJECTOR_TYPE_GRANITE4_VISION: { // image_newline lives at the top-level. model.image_newline = get_tensor(TN_IMAGE_NEWLINE); // Load separate layerwise and spatial projector tensors const auto projector_count = hparams.feature_layers.size(); model.qf_proj_blocks.resize(projector_count); for (size_t bid = 0; bid < projector_count; ++bid) { auto & b = model.qf_proj_blocks[bid]; // non-layerwise tensors b.qf_proj_img_pos = get_tensor(mtmd_format(TN_MULTI_PROJ_IMG_POS, bid)); b.qf_proj_query = get_tensor(mtmd_format(TN_MULTI_PROJ_QUERY, prefix, bid)); b.qf_proj_linear_w = get_tensor(mtmd_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "weight")); b.qf_proj_linear_b = get_tensor(mtmd_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "bias")); b.qf_proj_norm_w = get_tensor(mtmd_format(TN_MULTI_PROJ_NORM, prefix, bid, "weight")); b.qf_proj_norm_b = get_tensor(mtmd_format(TN_MULTI_PROJ_NORM, prefix, bid, "bias")); b.qf_proj_post_norm_w = get_tensor(mtmd_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "weight")); b.qf_proj_post_norm_b = get_tensor(mtmd_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "bias")); // laywerwise tensors // NOTE: If any model uses multi-layer qformers, this will need to change b.qf_proj_layers.resize(1); auto & pl = b.qf_proj_layers[0]; pl.q_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_Q, prefix, bid, "weight")); pl.q_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_Q, prefix, bid, "bias")); pl.k_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_K, prefix, bid, "weight")); pl.k_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_K, prefix, bid, "bias")); pl.v_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_V, prefix, bid, "weight")); pl.v_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_V, prefix, bid, "bias")); pl.o_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_O, prefix, bid, "weight")); pl.o_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_O, prefix, bid, "bias")); pl.ln_1_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_N, prefix, bid, "weight")); pl.ln_1_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_N, prefix, bid, "bias")); pl.cross_attn_q_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "weight")); pl.cross_attn_q_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "bias")); pl.cross_attn_k_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_K, prefix, bid, "weight")); pl.cross_attn_k_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_K, prefix, bid, "bias")); pl.cross_attn_v_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_V, prefix, bid, "weight")); pl.cross_attn_v_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_V, prefix, bid, "bias")); pl.cross_attn_o_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_O, prefix, bid, "weight")); pl.cross_attn_o_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_O, prefix, bid, "bias")); pl.cross_attn_norm_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_N, prefix, bid, "weight")); pl.cross_attn_norm_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_N, prefix, bid, "bias")); pl.ff_up_w = get_tensor(mtmd_format(TN_QF_FFN_UP, prefix, bid, "weight")); pl.ff_up_b = get_tensor(mtmd_format(TN_QF_FFN_UP, prefix, bid, "bias")); pl.ff_down_w = get_tensor(mtmd_format(TN_QF_FFN_DOWN, prefix, bid, "weight")); pl.ff_down_b = get_tensor(mtmd_format(TN_QF_FFN_DOWN, prefix, bid, "bias")); pl.ln_2_w = get_tensor(mtmd_format(TN_QF_FFN_NORM, prefix, bid, "weight")); pl.ln_2_b = get_tensor(mtmd_format(TN_QF_FFN_NORM, prefix, bid, "bias")); } } break; case PROJECTOR_TYPE_SENSENOVA_U1: { model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); } break; default: GGML_ASSERT(false && "unknown projector type"); } // load data { std::vector read_buf; // start loading event if (progress_callback){ progress_callback(0.0, progress_callback_user_data); } // compute total tensor data size for progress reporting size_t total_data_size = 0; for (auto & t : tensors_to_load) { total_data_size += ggml_nbytes(t); } // alloc memory and offload data model.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(model.ctx_data.get(), buft)); ggml_backend_buffer_set_usage(model.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); // read the weight from file if (!no_alloc) { size_t data_loaded = 0; for (auto & t : tensors_to_load) { ggml_tensor * cur = ggml_get_tensor(model.ctx_data.get(), t->name); GGML_ASSERT(cur && "tensor not found in ctx_data"); auto it_off = tensor_offset.find(t->name); GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor"); const size_t offset = it_off->second; fin.seekg(offset, std::ios::beg); if (!fin) { throw std::runtime_error(mtmd_format("%s: failed to seek for tensor %s\n", __func__, t->name)); } size_t num_bytes = ggml_nbytes(cur); if (ggml_backend_buft_is_host(buft)) { // for the CPU and Metal backend, we can read directly into the tensor fin.read(reinterpret_cast(cur->data), num_bytes); } else { // read into a temporary buffer first, then copy to device memory read_buf.resize(num_bytes); fin.read(reinterpret_cast(read_buf.data()), num_bytes); ggml_backend_tensor_set(cur, read_buf.data(), 0, num_bytes); } data_loaded += num_bytes; if (progress_callback && total_data_size > 0) { const float progress = (float)data_loaded / (float)total_data_size; if (!progress_callback(progress, progress_callback_user_data)) { throw std::runtime_error(mtmd_format("%s: model loading cancelled by progress_callback\n", __func__)); } } } LOG_DBG("%s: loaded %zu tensors from %s\n", __func__, tensors_to_load.size(), fname.c_str()); } else { LOG_DBG("%s: no_alloc is set, skipping tensor data loading (%zu tensors)\n", __func__, tensors_to_load.size()); } fin.close(); } } void get_bool(const std::string & key, bool & output, bool required = true) const { const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); if (i < 0) { if (required) { throw std::runtime_error("Key not found: " + key); } return; } output = gguf_get_val_bool(ctx_gguf.get(), i); } void get_i32(const std::string & key, int & output, bool required = true) const { const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); if (i < 0) { if (required) { throw std::runtime_error("Key not found: " + key); } return; } output = gguf_get_val_i32(ctx_gguf.get(), i); } void get_u32(const std::string & key, int & output, bool required = true) const { const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); if (i < 0) { if (required) { throw std::runtime_error("Key not found: " + key); } return; } const uint32_t val = gguf_get_val_u32(ctx_gguf.get(), i); // sanity check if (val > (uint32_t) INT32_MAX) { throw std::runtime_error(mtmd_format("%s: value %u for key '%s' exceeds INT32_MAX\n", __func__, val, key.c_str())); } output = (int) val; } void get_f32(const std::string & key, float & output, bool required = true) const { const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); if (i < 0) { if (required) { throw std::runtime_error("Key not found: " + key); } return; } output = gguf_get_val_f32(ctx_gguf.get(), i); } void get_arr_f32(const std::string & key, std::vector & output, bool required = true) const { const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); if (i < 0) { if (required) { throw std::runtime_error("Key not found: " + key); } return; } if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) { throw std::runtime_error(mtmd_format("%s: key '%s' is not an array\n", __func__, key.c_str())); } const auto type = gguf_get_arr_type(ctx_gguf.get(), i); if (type != GGUF_TYPE_FLOAT32) { throw std::runtime_error(mtmd_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_FLOAT32)\n", __func__, key.c_str(), type, GGUF_TYPE_FLOAT32)); } const size_t n = gguf_get_arr_n(ctx_gguf.get(), i); if (n > (size_t) std::numeric_limits::max()) { throw std::runtime_error(mtmd_format("%s: array '%s' is too large (%zu elements)\n", __func__, key.c_str(), n)); } output.resize(n); const float * values = (const float *)gguf_get_arr_data(ctx_gguf.get(), i); for (size_t j = 0; j < n; ++j) { output[j] = values[j]; } } void get_string(const std::string & key, std::string & output, bool required = true) const { const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); if (i < 0) { if (required) { throw std::runtime_error("Key not found: " + key); } return; } output = std::string(gguf_get_val_str(ctx_gguf.get(), i)); } void get_arr_int(const std::string & key, std::vector & output, bool required = true) const { const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); if (i < 0) { if (required) { throw std::runtime_error("Key not found: " + key); } return; } if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) { throw std::runtime_error(mtmd_format("%s: key '%s' is not an array\n", __func__, key.c_str())); } const auto type = gguf_get_arr_type(ctx_gguf.get(), i); if (type != GGUF_TYPE_INT32) { throw std::runtime_error(mtmd_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_INT32)\n", __func__, key.c_str(), type, GGUF_TYPE_INT32)); } const size_t n = gguf_get_arr_n(ctx_gguf.get(), i); if (n > (size_t) std::numeric_limits::max()) { throw std::runtime_error(mtmd_format("%s: array '%s' is too large (%zu elements)\n", __func__, key.c_str(), n)); } output.resize(n); const int32_t * values = (const int32_t *)gguf_get_arr_data(ctx_gguf.get(), i); for (size_t j = 0; j < n; ++j) { output[j] = values[j]; } } static void set_llava_uhd_res_candidates(mtmd_model & model, const int max_patches_per_side) { auto & hparams = model.hparams; for (int x = 1; x <= max_patches_per_side; x++) { for (int y = 1; y <= max_patches_per_side; y++) { if (x == 1 && y == 1) { continue; // skip the first point } hparams.image_res_candidates.push_back(mtmd_image_size{ x*hparams.image_size, y*hparams.image_size, }); } } } static void set_internvl_dhr_res_candidates(mtmd_model & model) { auto & hparams = model.hparams; int min_num = hparams.preproc_min_tiles; int max_num = hparams.preproc_max_tiles; if (min_num < 1) { return; // avoid divide by 0 } for (int a = min_num; a <= max_num; ++a) { int b_lo = (min_num + a - 1) / a; int b_hi = max_num / a; b_lo = std::max(b_lo, min_num); b_hi = std::min(b_hi, max_num); for (int b = b_lo; b <= b_hi; ++b) { hparams.image_res_candidates.push_back(mtmd_image_size { a*hparams.image_size, b*hparams.image_size, }); } } } }; // // loading // mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params, ggml_backend_buffer_type_t buft) { mtmd_model_loader loader(fname, /* skip_tensors */ false, params.progress_callback, params.progress_callback_user_data); auto load = [&](mtmd_modality modality) { auto model = std::make_unique(); model->hparams.custom_image_min_tokens = params.image_min_tokens; model->hparams.custom_image_max_tokens = params.image_max_tokens; loader.load_hparams(*model, modality); loader.load_tensors(*model, buft, params.no_alloc); const std::string & def = model->hparams.gen_default_voice; if (!def.empty() && model->gen_voices.find(def) == model->gen_voices.end()) { throw std::runtime_error(mtmd_format("%s: default voice \"%s\" is not a predefined voice\n", __func__, def.c_str())); } return model; }; mtmd_models res; if (loader.has_vision) { res.v = load(MTMD_MODALITY_VISION); } // TODO: we don't support audio for Gemma 3N, but GGUF contains audio tensors // we can remove this check when we implement audio support for Gemma 3N const bool skip_audio = res.v && res.v->proj_type == PROJECTOR_TYPE_GEMMA3NV; if (loader.has_audio && !skip_audio) { res.a = load(MTMD_MODALITY_AUDIO); } if (loader.has_gen_audio) { res.gen_a = load(MTMD_MODALITY_GEN_AUDIO); } return res; } mtmd_model_caps mtmd_model_get_caps(const char * fname) { mtmd_model_caps res; mtmd_model_loader loader(fname, /* skip_tensors= */ true); res.has_vision = loader.has_vision; res.has_audio = loader.has_audio; res.has_gen_audio = loader.has_gen_audio; return res; } // // mtmd_model // int mtmd_model::image_max_tokens() const { if (proj_type == PROJECTOR_TYPE_DEEPSEEK4V || proj_type == PROJECTOR_TYPE_DEEPSEEK41V) { return hparams.dsv4_max_n_token; } if (hparams.image_max_pixels <= 0) { return -1; } const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; return hparams.image_max_pixels / patch_area; } int mtmd_model::n_output_tokens_x(const mtmd_image_f32 * img) const { const auto & params = hparams; const int n_total = n_output_tokens(img); const auto & proj = proj_type; switch (proj) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: case PROJECTOR_TYPE_MAGEVL: case PROJECTOR_TYPE_SENSENOVA_U1: case PROJECTOR_TYPE_LING3VL: case PROJECTOR_TYPE_EXAONE4_5: case PROJECTOR_TYPE_MIMOVL: case PROJECTOR_TYPE_GLM4V: case PROJECTOR_TYPE_GLM5V: case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_HUNYUANVL: case PROJECTOR_TYPE_YOUTUVL: case PROJECTOR_TYPE_MUSE_GLIMMER: case PROJECTOR_TYPE_LFM3VL: return (img->nx() / params.patch_size) / 2; case PROJECTOR_TYPE_STEP3VL: case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: case PROJECTOR_TYPE_MINICPM_OCR_1_0: case PROJECTOR_TYPE_INKLING2V: return img->nx() / (params.patch_size * params.n_merge); case PROJECTOR_TYPE_DEEPSEEKOCR: case PROJECTOR_TYPE_DEEPSEEKOCR2: return (img->nx() / params.patch_size) / 4; default: break; } return n_total; } int mtmd_model::n_output_tokens_y(const mtmd_image_f32 * img) const { const auto & params = hparams; const auto & proj = proj_type; switch (proj) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: case PROJECTOR_TYPE_MAGEVL: case PROJECTOR_TYPE_SENSENOVA_U1: case PROJECTOR_TYPE_LING3VL: case PROJECTOR_TYPE_EXAONE4_5: case PROJECTOR_TYPE_MIMOVL: case PROJECTOR_TYPE_GLM4V: case PROJECTOR_TYPE_GLM5V: case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_HUNYUANVL: case PROJECTOR_TYPE_YOUTUVL: case PROJECTOR_TYPE_MUSE_GLIMMER: case PROJECTOR_TYPE_LFM3VL: return (img->ny() / params.patch_size) / 2; case PROJECTOR_TYPE_STEP3VL: case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: case PROJECTOR_TYPE_MINICPM_OCR_1_0: case PROJECTOR_TYPE_INKLING2V: return img->ny() / (params.patch_size * params.n_merge); default: break; } return 1; } int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const { const auto & params = hparams; // for models with fixed size image, the input image is already pre-processed and resized to square int patch_size = params.patch_size; int n_patches = (img->nx() / patch_size) * (img->ny() / patch_size); projector_type proj = proj_type; switch (proj) { case PROJECTOR_TYPE_MLP: case PROJECTOR_TYPE_MLP_NORM: case PROJECTOR_TYPE_JANUS_PRO: case PROJECTOR_TYPE_PHI4: { // do nothing } break; case PROJECTOR_TYPE_YASA2: { n_patches = 64; // adaptive average pooling to 8x8 tokens } break; case PROJECTOR_TYPE_LDP: case PROJECTOR_TYPE_LDPV2: case PROJECTOR_TYPE_GLM_EDGE: { n_patches /= 4; if (mm_boi) { n_patches += 2; // for BOI and EOI token embeddings } } break; case PROJECTOR_TYPE_MINICPMV: { // Use actual config value if available, otherwise fall back to hardcoded values if (params.minicpmv_query_num > 0) { n_patches = params.minicpmv_query_num; } else { // Fallback to hardcoded values for legacy models if (params.minicpmv_version == 2) { n_patches = 96; } else if (params.minicpmv_version == 3) { n_patches = 64; } else if (params.minicpmv_version == 4) { n_patches = 64; } else if (params.minicpmv_version == 5) { // MiniCPM-V 4.0 n_patches = 64; } else if (params.minicpmv_version == 6) { // MiniCPM-V 4.5 n_patches = 64; } else if (params.minicpmv_version == 100045) { // MiniCPM-o 4.5 n_patches = 64; } else { GGML_ABORT("Unknown minicpmv version"); } } } break; case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: case PROJECTOR_TYPE_MINICPM_OCR_1_0: { n_patches /= params.n_merge * params.n_merge; } break; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: case PROJECTOR_TYPE_MAGEVL: case PROJECTOR_TYPE_SENSENOVA_U1: case PROJECTOR_TYPE_LING3VL: case PROJECTOR_TYPE_EXAONE4_5: case PROJECTOR_TYPE_MIMOVL: case PROJECTOR_TYPE_MINIMAX_M3: case PROJECTOR_TYPE_GLM4V: case PROJECTOR_TYPE_GLM5V: case PROJECTOR_TYPE_YOUTUVL: case PROJECTOR_TYPE_MUSE_GLIMMER: case PROJECTOR_TYPE_LFM3VL: { // dynamic size (2 conv, so double patch size) int x_patch = img->nx() / (params.patch_size * 2); int y_patch = img->ny() / (params.patch_size * 2); n_patches = x_patch * y_patch; } break; case PROJECTOR_TYPE_STEP3VL: case PROJECTOR_TYPE_INKLING2V: { int x_patch = img->nx() / (params.patch_size * params.n_merge); int y_patch = img->ny() / (params.patch_size * params.n_merge); n_patches = x_patch * y_patch; } break; case PROJECTOR_TYPE_GEMMA3: case PROJECTOR_TYPE_GEMMA4V: case PROJECTOR_TYPE_GEMMA4UV: case PROJECTOR_TYPE_IDEFICS3: case PROJECTOR_TYPE_COHERE2V: case PROJECTOR_TYPE_INTERNVL: case PROJECTOR_TYPE_NEMOTRON_V2_VL: case PROJECTOR_TYPE_NEMOTRON_OMNI: case PROJECTOR_TYPE_LLAMA4: { // both X and Y are downscaled by the scale factor int scale_factor = hparams.n_merge; n_patches /= (scale_factor * scale_factor); } break; case PROJECTOR_TYPE_GEMMA3NV: { // MobileNetV5 MSFA adapter always outputs fixed 16x16 resolution // regardless of input size (see architecture description) n_patches = hparams.image_size / hparams.patch_size; } break; case PROJECTOR_TYPE_LFM2: case PROJECTOR_TYPE_D1OMNI_V: case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_KIMIK25: case PROJECTOR_TYPE_KIMIK31: { // dynamic size int out_patch_size = params.patch_size * hparams.n_merge; int x_patch = MTMD_ALIGN(img->nx(), out_patch_size) / out_patch_size; int y_patch = MTMD_ALIGN(img->ny(), out_patch_size) / out_patch_size; n_patches = x_patch * y_patch; } break; case PROJECTOR_TYPE_DEEPSEEK4V: { const int out_patch_size = params.patch_size * params.n_merge; const int n_llm_w = MTMD_ALIGN(img->nx(), out_patch_size) / out_patch_size; const int n_llm_h = MTMD_ALIGN(img->ny(), out_patch_size) / out_patch_size; n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out; } break; case PROJECTOR_TYPE_DEEPSEEK41V: { const int out_patch_size = params.patch_size * params.n_merge; const int n_llm_w = MTMD_ALIGN(img->nx(), out_patch_size) / out_patch_size; const int n_llm_h = MTMD_ALIGN(img->ny(), out_patch_size) / out_patch_size; n_patches = dsv41_n_block_tokens(n_llm_w, n_llm_h); } break; case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_DOTS_OCR: case PROJECTOR_TYPE_DOTS3NOTE_V: { // dynamic size int n_merge = hparams.n_merge; int stride = n_merge * n_merge; n_patches = MTMD_ALIGN(n_patches, stride) / stride; } break; case PROJECTOR_TYPE_DOTS3NOTE_A: { // 3x stride-2 conv2d over mel frames n_patches = (img->nx() + 7) / 8; } break; case PROJECTOR_TYPE_PIXTRAL: case PROJECTOR_TYPE_LIGHTONOCR: { // dynamic size int n_merge = hparams.n_merge; int n_patches_x = img->nx() / patch_size / n_merge; int n_patches_y = img->ny() / patch_size / n_merge; if (token_embd_img_break) { n_patches = n_patches_y * n_patches_x + n_patches_y - 1; // + one [IMG_BREAK] per row, except the last row } else { n_patches = n_patches_y * n_patches_x; } } break; case PROJECTOR_TYPE_VOXTRAL_RT: { // 2 context frames, conv stride 2, then stacked frames n_patches = (img->nx() - 2) / 2 / hparams.proj_stack_factor; } break; case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: { const int64_t fold = hparams.audio_samples_per_token(); n_patches = (int) ((img->nx() + fold - 1) / fold); } break; case PROJECTOR_TYPE_VIBEVOICE_ASR: { n_patches = (img->nx() - img->audio_n_ctx) / hparams.audio_samples_per_token(); } break; case PROJECTOR_TYPE_VOXTRAL: case PROJECTOR_TYPE_ULTRAVOX: case PROJECTOR_TYPE_QWEN2A: case PROJECTOR_TYPE_MERALION: case PROJECTOR_TYPE_MUSIC_FLAMINGO: { n_patches = img->nx(); const int proj_stack_factor = hparams.proj_stack_factor; if (audio_has_stack_frames()) { GGML_ASSERT(proj_stack_factor > 0); const int n_len = MTMD_ALIGN(n_patches, proj_stack_factor); n_patches = n_len / proj_stack_factor; } // whisper downscales input token by half after conv1d n_patches /= 2; if (audio_has_avgpool()) { // divide by 2 because of nn.AvgPool1d(2, stride=2) n_patches /= 2; } } break; case PROJECTOR_TYPE_HIGGS_AUDIO: { // stride-2 conv1d, avgpool(2), stride-2 depthwise conv1d n_patches = (img->nx() + 1) / 2; n_patches = n_patches / 2; n_patches = (n_patches + 1) / 2; } break; case PROJECTOR_TYPE_QWEN3A: { // chunk_size=100 frames --> 3x stride-2 conv2d --> 13 tokens per chunk const int chunk_size = 100; const int tokens_per_chunk = 13; n_patches = (img->nx() / chunk_size) * tokens_per_chunk; } break; case PROJECTOR_TYPE_GLMA: { n_patches = img->nx(); // whisper downscales input token by half after conv1d n_patches /= 2; // reshape by merge_factor n_patches /= hparams.proj_stack_factor; // for BOI and EOI token embeddings n_patches += 2; } break; case PROJECTOR_TYPE_COGVLM: { n_patches += 2; // for BOI and EOI token embeddings } break; case PROJECTOR_TYPE_DEEPSEEKOCR: { // SAM encoder applies two stride-2 convolutions (net_2 and net_3) // that reduce spatial dimensions by 4x in each direction (16x total) // E.g., 64x64 -> 16x16 patches n_patches /= 16; if (img->add_viewsep) { // global view: one image-newline per token-row + trailing view separator const int h = static_cast(std::sqrt(static_cast(n_patches))); n_patches = h * (h + 1) + 1; } else if (img->ny() >= img->nx() && img->ny() % img->nx() == 0) { // tile row: one image-newline per token-row const int grid_w = img->ny() / img->nx(); const int tile_patches = img->nx() / (patch_size * 4); // patches per tile side (SAM divides by 4) const int h = tile_patches; n_patches = (tile_patches * grid_w + 1) * h; } } break; case PROJECTOR_TYPE_HUNYUANVL: { int merge = hparams.n_merge; int ow = (img->nx() / patch_size) / merge; int oh = (img->ny() / patch_size) / merge; n_patches = (ow + 1) * oh + 2; } break; case PROJECTOR_TYPE_DEEPSEEKOCR2: { // 1024 global view -> 256 query tokens + 1 view separator = 257; // 768 local tile -> 144 query tokens, no separator. n_patches /= 16; if (img->add_viewsep) { n_patches += 1; // view separator, appended only after the global view } } break; case PROJECTOR_TYPE_LFM2A: case PROJECTOR_TYPE_D1OMNI_A: case PROJECTOR_TYPE_LFM3A: { n_patches = ((((img->nx() + 1) / 2) + 1) / 2 + 1) / 2; } break; case PROJECTOR_TYPE_GEMMA4A: { // Two Conv2D stride-2: O = floor((I + 2p - k) / s) + 1, p=1, k=3, s=2 // O = floor((I - 1) / 2) + 1 int n = img->nx(); for (int i = 0; i < 2; i++) { n = (n - 1) / 2 + 1; } n_patches = n; } break; case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: case PROJECTOR_TYPE_PARAKEET: { n_patches = (img->nx() + (params.subsampling_factor - 1)) / params.subsampling_factor; } break; case PROJECTOR_TYPE_GEMMA4UA: { n_patches = img->nx(); // no downsampling: one token per raw waveform frame } break; case PROJECTOR_TYPE_MIMO_AUDIO: { // conv1(s=1) + conv2(s=2) -> RVQ-encoder downsample conv(k=2,s=2) int n = img->nx(); n = (n - 1) / 2 + 1; // conv1 + conv2 n = (n - 2) / 2 + 1; // downsample conv const int group_size = params.audio_local_group_size; n_patches = (n + group_size - 1) / group_size; } break; case PROJECTOR_TYPE_GRANITE_SPEECH: { const int ws = hparams.audio_proj_window_size; const int ds = hparams.audio_proj_downsample_rate; n_patches = ((img->nx() + ws - 1) / ws) * (ws / ds); } break; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: case PROJECTOR_TYPE_ZONOS3_SPKENC: { // pooling gives one speaker embedding, whatever the clip length is n_patches = 1; } break; case PROJECTOR_TYPE_CHATTERBOX_SPKENC: { // statistics pooling gives one speaker embedding, whatever the clip length is n_patches = 1; } break; case PROJECTOR_TYPE_QWEN3TTS_GEN: case PROJECTOR_TYPE_LFM3A_GEN: case PROJECTOR_TYPE_DRAMA3_GEN: case PROJECTOR_TYPE_ZONOS3_GEN: { // one hidden-state vector fed back to the talker per call n_patches = 1; } break; case PROJECTOR_TYPE_POCKETTTS_SPKENC: { // one conditioning row per 12.5Hz frame const int hop = hparams.mimi_downsample * 120; n_patches = img->nx() / hop; } break; case PROJECTOR_TYPE_POCKETTTS_GEN: { // one latent per call for GEN_CODE, GEN_WAV sizes its input from the caller n_patches = 1; } break; case PROJECTOR_TYPE_GRANITE4_VISION: { // Per-tile output token count: each projector block outputs // query_side^2 tokens per window x n^2 windows. // For 384x384 input: n = 24/8 = 3, query_side = 4 -> 144. const int window_side = hparams.downsample_window_side; const int query_side = hparams.downsample_query_side; const int side = img->nx() / params.patch_size; const int n = side / window_side; const int out_side = query_side * n; n_patches = out_side * out_side; if (img->anyres.is_tiled()) { // overview tile, then the unpadded tile grid with one newline per row int off_x, off_y, w, h; mtmd_anyres_unpad(img->anyres.grid_x * out_side, img->anyres.grid_y * out_side, img->anyres.orig_nx, img->anyres.orig_ny, off_x, off_y, w, h); n_patches += h * (w + 1); } } break; default: GGML_ABORT("unsupported projector type"); } n_patches += (int) mtmd_suffix_rows(img->suffix_type).size(); return n_patches; } size_t mtmd_model::cbx_read_tensor(const char * name, float * out, size_t n_max) const { const auto & tensors = cbx_tensors; auto it = tensors.find(name); if (it == tensors.end() || !it->second) { return 0; } ggml_tensor * t = it->second; const size_t n = (size_t) ggml_nelements(t); if (!out) { return n; } if (n_max < n) { return 0; } if (t->type == GGML_TYPE_F32) { ggml_backend_tensor_get(t, out, 0, n * sizeof(float)); } else if (t->type == GGML_TYPE_F16) { std::vector tmp(n); ggml_backend_tensor_get(t, tmp.data(), 0, n * sizeof(ggml_fp16_t)); for (size_t i = 0; i < n; i++) out[i] = ggml_fp16_to_fp32(tmp[i]); } else { return 0; } return n; } int mtmd_model::n_mmproj_embd() const { switch (proj_type) { case PROJECTOR_TYPE_LDP: return mm_model_block_1_block_2_1_b->ne[0]; case PROJECTOR_TYPE_LDPV2: return mm_model_peg_0_b->ne[0]; case PROJECTOR_TYPE_MLP: case PROJECTOR_TYPE_PHI4: case PROJECTOR_TYPE_PIXTRAL: case PROJECTOR_TYPE_LIGHTONOCR: case PROJECTOR_TYPE_DOTS_OCR: case PROJECTOR_TYPE_DOTS3NOTE_V: case PROJECTOR_TYPE_DOTS3NOTE_A: return mm_2_w->ne[1]; case PROJECTOR_TYPE_MLP_NORM: return mm_3_b->ne[0]; case PROJECTOR_TYPE_MINICPMV: return mm_model_proj->ne[1]; case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: case PROJECTOR_TYPE_MINICPM_OCR_1_0: return mm_ffn_down_w->ne[1]; case PROJECTOR_TYPE_GLM_EDGE: return mm_model_mlp_3_w->ne[1]; case PROJECTOR_TYPE_MINIMAX_M3: return mm_merger_fc2_b->ne[0]; case PROJECTOR_TYPE_MUSE_GLIMMER: case PROJECTOR_TYPE_INKLING2V: return mm_2_w->ne[1]; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_MAGEVL: case PROJECTOR_TYPE_LFM3VL: case PROJECTOR_TYPE_EXAONE4_5: case PROJECTOR_TYPE_JANUS_PRO: case PROJECTOR_TYPE_YOUTUVL: return mm_1_b->ne[0]; case PROJECTOR_TYPE_SENSENOVA_U1: return hparams.projection_dim; case PROJECTOR_TYPE_QWEN3VL: // main path + deepstack paths return mm_1_b->ne[0] * (1 + n_deepstack_layers); case PROJECTOR_TYPE_LING3VL: return mm_1_b->ne[0]; case PROJECTOR_TYPE_MIMOVL: return mm_1_w->ne[1]; case PROJECTOR_TYPE_STEP3VL: return mm_model_proj->ne[1]; case PROJECTOR_TYPE_GEMMA3: case PROJECTOR_TYPE_GEMMA3NV: return mm_input_proj_w->ne[0]; case PROJECTOR_TYPE_GEMMA4V: case PROJECTOR_TYPE_GEMMA4UV: case PROJECTOR_TYPE_GEMMA4A: case PROJECTOR_TYPE_GEMMA4UA: return mm_input_proj_w->ne[1]; case PROJECTOR_TYPE_IDEFICS3: case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: return mm_fc_w->ne[1]; case PROJECTOR_TYPE_VIBEVOICE_ASR: return mm_1_w->ne[1]; case PROJECTOR_TYPE_ULTRAVOX: case PROJECTOR_TYPE_VOXTRAL: case PROJECTOR_TYPE_VOXTRAL_RT: case PROJECTOR_TYPE_MUSIC_FLAMINGO: return mm_2_w->ne[1]; case PROJECTOR_TYPE_MERALION: return mm_3_w->ne[1]; // out_proj output dim case PROJECTOR_TYPE_INTERNVL: case PROJECTOR_TYPE_NEMOTRON_V2_VL: case PROJECTOR_TYPE_NEMOTRON_OMNI: return mm_3_w->ne[1]; case PROJECTOR_TYPE_LLAMA4: return mm_model_proj->ne[1]; case PROJECTOR_TYPE_QWEN2A: case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: return mm_fc_w->ne[1]; case PROJECTOR_TYPE_QWEN3A: case PROJECTOR_TYPE_HIGGS_AUDIO: return mm_2_w->ne[1]; case PROJECTOR_TYPE_GLMA: case PROJECTOR_TYPE_LFM2: case PROJECTOR_TYPE_D1OMNI_V: case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_KIMIK25: case PROJECTOR_TYPE_KIMIK31: case PROJECTOR_TYPE_YASA2: case PROJECTOR_TYPE_DEEPSEEK4V: case PROJECTOR_TYPE_DEEPSEEK41V: case PROJECTOR_TYPE_COHERE2V: return mm_2_w->ne[1]; case PROJECTOR_TYPE_HUNYUANVL: return mm_model_proj->ne[1]; case PROJECTOR_TYPE_COGVLM: return mm_4h_to_h_w->ne[1]; case PROJECTOR_TYPE_DEEPSEEKOCR: case PROJECTOR_TYPE_DEEPSEEKOCR2: return mm_fc_w->ne[1]; case PROJECTOR_TYPE_LFM2A: return position_embeddings->ne[0]; case PROJECTOR_TYPE_D1OMNI_A: case PROJECTOR_TYPE_LFM3A: return mm_3_w->ne[1]; case PROJECTOR_TYPE_GRANITE_SPEECH: return qf_proj_blocks[0].qf_proj_linear_w->ne[1]; case PROJECTOR_TYPE_GRANITE4_VISION: return qf_proj_blocks.size() * hparams.projection_dim; case PROJECTOR_TYPE_GLM4V: case PROJECTOR_TYPE_GLM5V: return mm_ffn_down_w->ne[1]; case PROJECTOR_TYPE_MIMO_AUDIO: return mm_2_w->ne[1]; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: return mm_fc_w->ne[2]; case PROJECTOR_TYPE_ZONOS3_SPKENC: return zonos3_spk.at("a.spk.proj.weight")->ne[1]; case PROJECTOR_TYPE_QWEN3TTS_GEN: case PROJECTOR_TYPE_LFM3A_GEN: case PROJECTOR_TYPE_DRAMA3_GEN: case PROJECTOR_TYPE_ZONOS3_GEN: return gen_code_out_embd_w->ne[0]; case PROJECTOR_TYPE_POCKETTTS_SPKENC: return spk_proj_w->ne[1]; case PROJECTOR_TYPE_POCKETTTS_GEN: return gen_input_lin_w->ne[1]; case PROJECTOR_TYPE_CHATTERBOX: // talker width, carried by the speech table return cbx_tensors.at("a.gen.code.out_embd.weight")->ne[0]; case PROJECTOR_TYPE_CHATTERBOX_SPKENC: // talker conditioning rows, the speaker projection carries their width return cbx_tensors.at("a.cenc.spkr_enc.weight")->ne[1]; case PROJECTOR_TYPE_SOPRANO: return hparams.projection_dim; case PROJECTOR_TYPE_PARAKEET: return mm_1_w->ne[1]; default: GGML_ABORT("Unknown projector type"); } } // TODO @ngxson : this is no longer correct with mtmd_batch API // this was only meant to be used by qwen-vl-based models, to fuse 2 input images into one (qwen-vl video support) // this logic should be refactored in near future to distinctly handle "merge frames" and "batching" int mtmd_model::n_temporal_merge() const { switch (proj_type) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: case PROJECTOR_TYPE_LFM3VL: case PROJECTOR_TYPE_NEMOTRON_OMNI: case PROJECTOR_TYPE_LING3VL: return 2; default: return 1; } }