#include "mtmd-subcontext.h" #include "mtmd-graph.h" #include "models/models.h" #include "mtmd-audio.h" #include "ggml.h" #include "ggml-cpp.h" #include "ggml-alloc.h" #include "ggml-backend.h" #include "ggml-cpu.h" #include #include #include #include #include #include #include #include #include #include // ggml_backend_cpu_set_n_threads(backend_cpu, n_threads); static void mtmd_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) { ggml_backend_dev_t dev = ggml_backend_get_device(backend_cpu); ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; if (reg) { auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); if (ggml_backend_set_n_threads_fn) { ggml_backend_set_n_threads_fn(backend_cpu, n_threads); } } } static std::shared_ptr mtmd_threadpool_init(int n_threads) { ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; if (!reg || n_threads < 2) { return nullptr; } auto * new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new"); auto * free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free"); if (!new_fn || !free_fn) { return nullptr; } ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads); tpp.poll = 0; // the pool is idle most of the time, don't take CPU away from the text model ggml_threadpool_t threadpool = new_fn(&tpp); if (!threadpool) { throw std::runtime_error("failed to create threadpool"); } return std::shared_ptr(threadpool, free_fn); } ggml_backend_dev_t mtmd_select_device(bool use_gpu, ggml_backend_dev_t device) { if (!use_gpu) { return nullptr; } if (device != nullptr) { return device; } ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU); return dev ? dev : ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU); } mtmd_backends::mtmd_backends(ggml_backend_dev_t device, int n_threads, int max_nodes, ggml_backend_sched_eval_callback cb_eval, void * cb_eval_user_data) { threadpool = mtmd_threadpool_init(n_threads); backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); if (!backend_cpu) { throw std::runtime_error("failed to initialize CPU backend"); } if (threadpool) { ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); if (set_threadpool_fn) { set_threadpool_fn(backend_cpu, threadpool.get()); } } if (device != nullptr) { backend = ggml_backend_dev_init(device, nullptr); if (!backend) { ggml_backend_free(backend_cpu); throw std::runtime_error(mtmd_format("%s: failed to initialize \"%s\" backend\n", __func__, ggml_backend_dev_name(device))); } } if (backend) { LOG_INF("%s: using %s backend\n", __func__, ggml_backend_name(backend)); ptrs.push_back(backend); bufts.push_back(ggml_backend_get_default_buffer_type(backend)); } else { backend = backend_cpu; LOG_INF("%s: using CPU backend\n", __func__); } ptrs.push_back(backend_cpu); bufts.push_back(ggml_backend_get_default_buffer_type(backend_cpu)); sched.reset(ggml_backend_sched_new(ptrs.data(), bufts.data(), ptrs.size(), max_nodes, false, true)); if (cb_eval != nullptr) { ggml_backend_sched_set_eval_callback(sched.get(), cb_eval, cb_eval_user_data); } } mtmd_backends::~mtmd_backends() { // before the backends it uses sched.reset(); ggml_backend_free(backend); if (backend != backend_cpu) { ggml_backend_free(backend_cpu); } } std::map mtmd_backends::get_mem_compute() const { std::map res; for (ggml_backend_t b : ptrs) { res[ggml_backend_get_device(b)] += ggml_backend_sched_get_buffer_size(sched.get(), b); } return res; } mtmd_subcontext::mtmd_subcontext(const mtmd_model & model, mtmd_backends & backends, const mtmd_subctx_params & params) : model(model), backends(backends), max_nodes(model.max_nodes), flash_attn_type(params.flash_attn_type) { if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_AUTO && !model.flash_attn_auto) { flash_attn_type = MTMD_FLASH_ATTN_TYPE_DISABLED; } debug_output_embeddings = std::getenv("MTMD_DEBUG_EMBEDDINGS") != nullptr; buf_compute_meta.resize(max_nodes * ggml_tensor_overhead() + ggml_graph_overhead_custom(max_nodes, false)); auto batch = get_dummy_batch(); if (model.modality != MTMD_MODALITY_GEN_AUDIO) { // check batching support support_batch = mtmd_get_graph_builder(this, batch)->support_batch(); } if (params.warmup) { warmup(batch); } } mtmd_image_f32_batch mtmd_subcontext::get_dummy_batch() const { // create a fake batch const auto & hparams = model.hparams; mtmd_image_f32_batch batch; mtmd_image_f32 img; if (model.modality == MTMD_MODALITY_VISION) { const int sz = hparams.warmup_image_size; img.set_size({sz, sz}, false, false); LOG_INF("%s: warmup with image size = %d x %d\n", __func__, sz, sz); } else if (model.modality == MTMD_MODALITY_GEN_AUDIO) { // TODO: improve this, move to per-model impl const int n_embd = model.n_mmproj_embd(); img.set_size({n_embd, 1}, false, true); LOG_INF("%s: warmup with one gen-audio step\n", __func__); } else { // GEMMA4UA uses n_mel_bins as a raw-waveform frame size (640), not a mel-bin count, // so the [1, 256] bound only applies to FFT-based models. const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA; if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) { throw std::runtime_error(mtmd_format("%s: invalid n_mel_bins (%d), must be in [1, 256]\n", __func__, hparams.n_mel_bins)); } img.set_size({hparams.warmup_audio_size, hparams.n_mel_bins}, false, false); LOG_INF("%s: warmup with audio size = %d\n", __func__, hparams.warmup_audio_size); } batch.entries.push_back(img); return batch; } void mtmd_subcontext::warmup(const mtmd_image_f32_batch & batch) { support_info_graph info; if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_AUTO) { // try to enable flash attention to see if it's supported flash_attn_type = MTMD_FLASH_ATTN_TYPE_ENABLED; info = reserve_compute_meta(batch); if (!info.fattn && info.fattn_op) { auto op = info.fattn_op; LOG_WRN("%s: *****************************************************************\n", __func__); LOG_WRN("%s: WARNING: flash attention not supported by %s, memory usage will increase\n", __func__, ggml_backend_name(backends.backend)); LOG_WRN("%s: op params: \n", __func__); static auto print_shape = [](const char * fn, const char * name, ggml_tensor * t) { LOG_WRN("%s: %s: type = %s, ne = [%d %d %d %d], nb = [%d %d %d %d]\n", fn, name, ggml_type_name(t->type), t->ne[0], t->ne[1], t->ne[2], t->ne[3], t->nb[0], t->nb[1], t->nb[2], t->nb[3]); }; print_shape(__func__, " dst", op); print_shape(__func__, "src0", op->src[0]); print_shape(__func__, "src1", op->src[1]); print_shape(__func__, "src2", op->src[2]); LOG_WRN("%s: please report this on github as an issue\n", __func__); LOG_WRN("%s: *****************************************************************\n", __func__); flash_attn_type = MTMD_FLASH_ATTN_TYPE_DISABLED; reserve_compute_meta(batch); } } else { info = reserve_compute_meta(batch); if (!info.fattn && flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) { LOG_WRN("%s: flash attention is not supported by the current backend; falling back to CPU (performance will be degraded)\n", __func__); } } is_allocated = true; // mark buffers as allocated LOG_INF("%s: flash attention is %s\n", __func__, (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled"); // print ops that are not supported by the GPU backend (if there is one) if (backends.backend != backends.backend_cpu) { std::vector unsupported_ops; for (const auto & op : info.ops) { if (!op.is_accel) { unsupported_ops.push_back(op); } } if (!unsupported_ops.empty()) { LOG_WRN("%s: *****************************************************************\n", __func__); LOG_WRN("%s: WARNING: the mmproj graph uses unsupported operators by the backend\n", __func__); LOG_WRN("%s: the performance will be suboptimal \n", __func__); LOG_WRN("%s: list of unsupported ops (backend=%s):\n", __func__, ggml_backend_name(backends.backend)); for (const auto & op : unsupported_ops) { LOG_WRN("%s: %16s: type = %s, ne = [%d %d %d %d]\n", __func__, ggml_op_name(op.op->op), ggml_type_name(op.op->type), op.op->ne[0], op.op->ne[1], op.op->ne[2], op.op->ne[3]); } LOG_WRN("%s: flash attention is %s\n", __func__, (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled"); LOG_WRN("%s: please report this on github as an issue\n", __func__); LOG_WRN("%s: ref: https://github.com/ggml-org/llama.cpp/pull/16837#issuecomment-3461676118\n", __func__); LOG_WRN("%s: *****************************************************************\n", __func__); } } } mtmd_subcontext::support_info_graph mtmd_subcontext::reserve_compute_meta(const mtmd_image_f32_batch & batch) { ggml_cgraph * gf = mtmd_get_graph_builder(this, batch)->build(); ggml_backend_sched_reserve(backends.sched.get(), gf); const int n_splits = ggml_backend_sched_get_n_splits(backends.sched.get()); const int n_nodes = ggml_graph_n_nodes(gf); LOG_INF("%s: graph splits = %d, nodes = %d\n", __func__, n_splits, n_nodes); support_info_graph res { /*.fattn = */ true, /*.fattn_op = */ nullptr, /*.ops = */ {}, }; // check op support for (int i = 0; i < ggml_graph_n_nodes(gf); i++) { ggml_tensor * node = ggml_graph_node(gf, i); res.ops.push_back({node, true}); if (!ggml_backend_supports_op(backends.backend, node)) { res.ops.back().is_accel = false; if (node->op == GGML_OP_FLASH_ATTN_EXT) { res.fattn = false; res.fattn_op = node; } } } return res; } bool mtmd_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out) { ggml_backend_sched_t sched = config.sched; ggml_backend_t backend_cpu = ggml_backend_sched_get_backend(sched, ggml_backend_sched_get_n_backends(sched) - 1); mtmd_backend_cpu_set_n_threads(backend_cpu, std::max(1, config.n_threads)); const auto & cache = *config.cache; const auto & filters = cache.filters.data; const int64_t n_fft_bins = config.n_fft_bins; const int64_t frame_size = (n_fft_bins - 1) * 2; const int64_t n_mel = config.n_mel; GGML_ASSERT(filters.size() == (size_t) (n_mel * n_fft_bins)); GGML_ASSERT(cache.cos_vals.size() == (size_t) frame_size && cache.sin_vals.size() == (size_t) frame_size); // DFT matrix with the window folded in, the first n_fft_bins rows give the real part std::vector dft((size_t) 2 * n_fft_bins * frame_size); for (int64_t k = 0; k < n_fft_bins; k++) { for (int64_t j = 0; j < frame_size; j++) { const int64_t q = (j * k) % frame_size; dft[k * frame_size + j] = window[j] * cache.cos_vals[q]; dft[(k + n_fft_bins) * frame_size + j] = window[j] * cache.sin_vals[q]; } } // the spectrum is several times larger than the result, so long audio is processed in blocks const int64_t n_frames_max = 8192; for (int64_t off = 0; off < n_frames; off += n_frames_max) { const int64_t n_cur = std::min(n_frames_max, n_frames - off); ggml_init_params params = { /*.mem_size =*/ ggml_tensor_overhead() * 32 + ggml_graph_overhead(), /*.mem_buffer =*/ nullptr, /*.no_alloc =*/ true, }; ggml_context_ptr ctx_ptr(ggml_init(params)); ggml_context * ctx0 = ctx_ptr.get(); ggml_tensor * inp_samples = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, (n_cur - 1) * config.hop_length + frame_size); ggml_tensor * inp_dft = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, frame_size, 2 * n_fft_bins); ggml_tensor * inp_filters = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_fft_bins, n_mel); ggml_set_input(inp_samples); ggml_set_input(inp_dft); ggml_set_input(inp_filters); ggml_tensor * cur = ggml_cont(ctx0, ggml_view_2d(ctx0, inp_samples, frame_size, n_cur, config.hop_length * ggml_element_size(inp_samples), 0)); cur = ggml_mul_mat(ctx0, inp_dft, cur); // [2 * n_fft_bins, n_cur] ggml_prec_set_src(cur, GGML_PREC_F32, 1); cur = ggml_sqr(ctx0, cur); cur = ggml_add(ctx0, ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], 0), ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], n_fft_bins * ggml_element_size(cur))); if (config.use_magnitude) { cur = ggml_sqrt(ctx0, cur); } cur = ggml_mul_mat(ctx0, cur, inp_filters); // [n_cur, n_mel] ggml_prec_set_src(cur, GGML_PREC_F32, 1); cur = config.mel_floor_add ? ggml_scale_bias(ctx0, cur, 1.0f, config.mel_floor) : ggml_clamp(ctx0, cur, config.mel_floor, FLT_MAX); cur = ggml_log(ctx0, cur); if (!config.use_natural_log) { cur = ggml_scale(ctx0, cur, 1.0f / logf(10.0f)); } ggml_set_output(cur); ggml_cgraph * gf = ggml_new_graph(ctx0); ggml_build_forward_expand(gf, cur); ggml_backend_sched_reset(sched); if (!ggml_backend_sched_alloc_graph(sched, gf)) { LOG_ERR("%s: failed to allocate compute graph\n", __func__); return false; } ggml_backend_tensor_set(inp_samples, samples + off * config.hop_length, 0, ggml_nbytes(inp_samples)); ggml_backend_tensor_set(inp_dft, dft.data(), 0, ggml_nbytes(inp_dft)); ggml_backend_tensor_set(inp_filters, filters.data(), 0, ggml_nbytes(inp_filters)); auto status = ggml_backend_sched_graph_compute(sched, gf); if (status != GGML_STATUS_SUCCESS) { LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status); return false; } for (int64_t j = 0; j < n_mel; j++) { ggml_backend_tensor_get(cur, out + j * n_frames + off, j * cur->nb[1], n_cur * ggml_element_size(cur)); } } return true; } bool mtmd_subcontext::encode_image(int n_threads, const mtmd_image_f32 * img, std::vector & out_vec) { mtmd_image_f32_batch imgs; mtmd_image_f32 img_copy = *img; imgs.entries.push_back(std::move(img_copy)); return encode_batch(n_threads, &imgs, out_vec); } bool mtmd_subcontext::encode_batch(int n_threads, const mtmd_image_f32_batch * imgs_c_ptr, std::vector & out_batch_embd) { mtmd_subctx_encode_params params; params.imgs = imgs_c_ptr; params.n_threads = n_threads; params.out_embd = &out_batch_embd; if (model.proj_type == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) { return mtmd_graph_nemotron_transcribe::encode_windows(this, model, ¶ms); } return encode(¶ms); } // persisted state slots of the gen-audio decoders and the streaming audio encoders static std::vector list_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { switch (model.proj_type) { case PROJECTOR_TYPE_QWEN3TTS_GEN: return list_c2w_state_slots(hparams, model); case PROJECTOR_TYPE_FISH_S2_GEN: return list_c2w_state_slots(hparams, model); case PROJECTOR_TYPE_DRAMA3_GEN: return list_drama3_state_slots(hparams, model); case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model); case PROJECTOR_TYPE_LFM3A_GEN: return list_lfm3a_gen_state_slots(hparams, model); case PROJECTOR_TYPE_ZONOS3_GEN: return list_zonos3_gen_state_slots(hparams, model); default: return {}; } } // state slots of the streaming audio encoders static std::vector list_enc_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { switch (model.proj_type) { case PROJECTOR_TYPE_LFM3A: return list_lfm3a_state_slots(hparams, model); case PROJECTOR_TYPE_GEMMA4A: return list_gemma4a_state_slots(hparams, model); case PROJECTOR_TYPE_VIBEVOICE_ASR: return list_vibevoice_asr_state_slots(hparams, model); default: return {}; } } // state_out can have another shape than its slot, only the byte size must match static void copy_gen_state_out(ggml_tensor * src, ggml_tensor * dst) { GGML_ASSERT(ggml_nbytes(src) == ggml_nbytes(dst)); if (src->type == dst->type && ggml_are_same_shape(src, dst) && ggml_is_contiguous(src)) { ggml_backend_tensor_copy(src, dst); return; } std::vector tmp(ggml_nbytes(src)); ggml_backend_tensor_get(src, tmp.data(), 0, tmp.size()); ggml_backend_tensor_set(dst, tmp.data(), 0, tmp.size()); } mtmd_subctx_gen_state * mtmd_subcontext::gen_state_init(uint32_t seed) { auto st = std::make_unique(); st->rng.seed(seed == UINT32_MAX ? std::random_device{}() : seed); const auto slots = list_gen_state_slots(model.hparams, model); if (slots.empty()) { return st.release(); } ggml_init_params params = { /*.mem_size =*/ ggml_tensor_overhead() * slots.size(), /*.mem_buffer =*/ nullptr, /*.no_alloc =*/ true, }; st->ctx_data.reset(ggml_init(params)); for (const auto & slot : slots) { ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1); ggml_format_name(t, "gen_state_%s", slot.name.c_str()); st->slots.push_back(t); } st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backends.backend)); if (!st->buf) { LOG_ERR("%s: failed to allocate the gen state\n", __func__); return nullptr; } // a zero state is the cold start ggml_backend_buffer_clear(st->buf.get(), 0); return st.release(); } mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) { const auto & hparams = model.hparams; if (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A || model.proj_type == PROJECTOR_TYPE_VIBEVOICE_ASR) { // see list_enc_state_slots() const auto slots = list_enc_state_slots(hparams, model); auto st = std::make_unique(); ggml_init_params params = { /*.mem_size =*/ ggml_tensor_overhead() * slots.size(), /*.mem_buffer =*/ nullptr, /*.no_alloc =*/ true, }; st->ctx_data.reset(ggml_init(params)); for (const auto & slot : slots) { ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1); ggml_format_name(t, "enc_state_%s", slot.name.c_str()); st->slots.push_back(t); } st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backends.backend)); if (!st->buf) { LOG_ERR("%s: failed to allocate the encoder state\n", __func__); return nullptr; } // a zero state is the cold start ggml_backend_buffer_clear(st->buf.get(), 0); return st.release(); } if (model.proj_type != PROJECTOR_TYPE_VOXTRAL_RT || hparams.attn_window_size <= 0) { LOG_ERR("%s: the encoder does not support streaming input\n", __func__); return nullptr; } if (n_frames_max <= 0) { LOG_ERR("%s: invalid n_frames_max %d\n", __func__, n_frames_max); return nullptr; } const int n_layer = hparams.n_layer; const int d_head = hparams.n_embd_head > 0 ? hparams.n_embd_head : hparams.n_embd / hparams.n_head; const int n_embd_kv = hparams.n_head_kv * d_head; auto st = std::make_unique(); st->n_ctx = hparams.attn_window_size + n_frames_max; st->slot_pos.assign(st->n_ctx, -1); ggml_init_params params = { /*.mem_size =*/ ggml_tensor_overhead() * 2 * n_layer, /*.mem_buffer =*/ nullptr, /*.no_alloc =*/ true, }; st->ctx_data.reset(ggml_init(params)); for (int il = 0; il < n_layer; il++) { ggml_tensor * k = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F16, n_embd_kv, st->n_ctx); ggml_tensor * v = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F16, n_embd_kv, st->n_ctx); ggml_format_name(k, "enc_state_k_%d", il); ggml_format_name(v, "enc_state_v_%d", il); st->k.push_back(k); st->v.push_back(v); } st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backends.backend)); if (!st->buf) { LOG_ERR("%s: failed to allocate the encoder state\n", __func__); return nullptr; } // empty slots are masked out, but they must not hold NaN ggml_backend_buffer_clear(st->buf.get(), 0); LOG_INF("%s: encoder state: %d slots, %.2f MiB\n", __func__, st->n_ctx, ggml_backend_buffer_get_size(st->buf.get()) / 1024.0 / 1024.0); return st.release(); } bool mtmd_subcontext::encode_kokoro(mtmd_subctx_encode_params * params) { const int n_embd = model.n_mmproj_embd(); const int n_style = mtmd_graph_kokoro::n_style; if (params->gen_process != MTMD_SUBCTX_GEN_PROCESS_GEN_WAV || !params->codes || !params->feats || !params->gen_state || !params->out_audio) { LOG_ERR("%s: kokoro takes the phoneme tokens and their backbone states\n", __func__); return false; } const auto & tokens = *params->codes; const auto & feats = *params->feats; const int n_tok = (int) tokens.size(); if (n_tok < 2 || feats.size() != (size_t) 2 * n_style + (size_t) n_tok * n_embd) { LOG_ERR("%s: expected a voice row of %d values, then %d values for each of the %d tokens\n", __func__, 2 * n_style, n_embd, n_tok); return false; } const int64_t n_vocab = model.cbx_tensors.at("a.gen.text.embedding.weight")->ne[1]; for (int32_t t : tokens) { if (t < 0 || t >= n_vocab) { LOG_ERR("%s: phoneme token out of range (%d, vocabulary size %lld)\n", __func__, t, (long long) n_vocab); return false; } } const std::vector style_dec(feats.begin(), feats.begin() + n_style); const std::vector style_pred(feats.begin() + n_style, feats.begin() + 2 * n_style); const float * bert = feats.data() + 2 * n_style; auto run = [&](mtmd_subctx_kokoro_io & io) { mtmd_subctx_encode_params p = *params; p.kokoro = &io; p.out_audio = nullptr; return encode(&p); }; std::vector text_cnn; { mtmd_subctx_kokoro_io io; io.stage = mtmd_graph_kokoro::STAGE_TEXT; io.n_tok = n_tok; io.tokens = &tokens; io.out0 = &text_cnn; if (!run(io)) { return false; } } std::vector shared, asr; const int n_frames = mtmd_graph_kokoro::prosody(model, bert, n_embd, text_cnn.data(), n_tok, style_pred.data(), params->speed, shared, asr); std::vector f0, en; { mtmd_subctx_kokoro_io io; io.stage = mtmd_graph_kokoro::STAGE_F0N; io.n_frames = n_frames; io.x = &shared; io.style = &style_pred; io.out0 = &f0; io.out1 = &en; if (!run(io)) { return false; } } std::vector har, spec; { mtmd_subctx_kokoro_io io; io.stage = mtmd_graph_kokoro::STAGE_DEC; io.n_frames = n_frames; io.n_har = mtmd_graph_kokoro::har_source(model, f0, params->gen_state->rng, har); io.x = &asr; io.style = &style_dec; io.f0 = &f0; io.en = &en; io.har = &har; io.out0 = &spec; if (!run(io)) { return false; } } mtmd_graph_kokoro::istft(spec, *params->out_audio); return true; } bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { const mtmd_image_f32_batch & imgs = *params->imgs; int n_batch_cur = imgs.entries.size(); // [QWEN_VIDEO] for video models, the batch dimension is used as temporal dimension for merged frames if (!support_batch && n_batch_cur > model.n_temporal_merge()) { LOG_ERR("%s: batch size %d exceeds maximum supported batch/temporal-merge size %d\n", __func__, n_batch_cur, model.n_temporal_merge()); return false; } // if buffers are not allocated, we need to do a warmup run to allocate them if (!is_allocated) { warmup(*params->imgs); } if (model.proj_type == PROJECTOR_TYPE_KOKORO && !params->kokoro) { return encode_kokoro(params); } const bool needs_gen_state = params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE || params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV || params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS; if (needs_gen_state && params->gen_state == nullptr) { LOG_ERR("%s: audio gen requires a gen state\n", __func__); return false; } if (model.proj_type == PROJECTOR_TYPE_DRAMA3_GEN && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE) { float pos = 0.0f; ggml_backend_tensor_get(params->gen_state->slots[0], &pos, 0, sizeof(float)); if ((int) pos >= mtmd_graph_drama3_gen::n_pos_max) { LOG_ERR("%s: the codebook tower holds at most %d rows\n", __func__, mtmd_graph_drama3_gen::n_pos_max); return false; } } // build the inference graph ggml_backend_sched_reset(backends.sched.get()); ggml_cgraph * gf = mtmd_get_graph_builder(this, imgs, params)->build(); // the last node is the embedding tensor, code2wav has no out_embd // read it before scheduling: backends may reorder the nodes of graphs with several outputs (Metal) ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr; if (!ggml_backend_sched_alloc_graph(backends.sched.get(), gf)) { LOG_ERR("%s: failed to allocate compute graph\n", __func__); return false; } // set inputs const auto & hparams = model.hparams; const int image_size_width = imgs.entries[0].nx(); const int image_size_height = imgs.entries[0].ny(); const int patch_size = hparams.patch_size; const int num_patches = ((image_size_width / patch_size) * (image_size_height / patch_size)); const int n_pos = num_patches + (model.class_embedding ? 1 : 0); const int pos_w = image_size_width / patch_size; const int pos_h = image_size_height / patch_size; auto get_inp_tensor = [&gf](const char * name) { ggml_tensor * inp = ggml_graph_get_tensor(gf, name); if (inp == nullptr) { GGML_ABORT("Failed to get tensor %s", name); } if (!(inp->flags & GGML_TENSOR_FLAG_INPUT)) { GGML_ABORT("Tensor %s is not an input tensor", name); } return inp; }; auto set_input_f32 = [&get_inp_tensor](const char * name, const std::vector & values) { ggml_tensor * cur = get_inp_tensor(name); GGML_ASSERT(cur->type == GGML_TYPE_F32); GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size()); ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); }; // mask from build_inp_attn_mask(), f16 if flash attn is enabled auto set_input_attn_mask = [&get_inp_tensor](const std::vector & values, int idx = 0) { ggml_tensor * cur = get_inp_tensor(get_attn_mask_name(idx).c_str()); GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size()); if (cur->type == GGML_TYPE_F16) { std::vector values_f16(values.size()); ggml_fp32_to_fp16_row(values.data(), values_f16.data(), values.size()); ggml_backend_tensor_set(cur, values_f16.data(), 0, ggml_nbytes(cur)); } else { GGML_ASSERT(cur->type == GGML_TYPE_F32); ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); } }; auto set_input_i32 = [&get_inp_tensor](const char * name, std::vector & values) { ggml_tensor * cur = get_inp_tensor(name); GGML_ASSERT(cur->type == GGML_TYPE_I32); GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size()); ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); }; // state of the previous call, zero on a cold start auto set_gen_state_in = [&]() { const auto & slots = params->gen_state->slots; size_t i = 0; for (const auto & slot : list_gen_state_slots(hparams, model)) { GGML_ASSERT(i < slots.size()); if (slot.in_place) { i++; continue; } ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str())); } }; // returns the "pos" slot; without a stream, the state is zero auto set_enc_state_in = [&]() -> int { const auto state_slots = list_enc_state_slots(hparams, model); GGML_ASSERT(!state_slots.empty() && state_slots[0].name == "pos"); if (params->enc_state == nullptr) { for (const auto & slot : state_slots) { ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str()); ggml_backend_tensor_memset(t, 0, 0, ggml_nbytes(t)); } return 0; } const auto & slots = params->enc_state->slots; GGML_ASSERT(slots.size() == state_slots.size()); float base_f = 0.0f; ggml_backend_tensor_get(slots[0], &base_f, 0, sizeof(float)); size_t i = 0; for (const auto & slot : state_slots) { ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str())); } return (int) base_f; }; // lfm3a encoder and codec: causal sliding window over [cached window | this call] auto set_lfm3a_window_inputs = [&](int base, int n_pos, int window) { std::vector positions(n_pos); for (int i = 0; i < n_pos; i++) { positions[i] = base + i; } set_input_i32("positions", positions); // key j holds position base - prefix + j const int prefix = window - 1; const int n_kv = prefix + n_pos; std::vector mask((size_t) n_kv * n_pos, -INFINITY); for (int q = 0; q < n_pos; q++) { for (int j = std::max(0, q + prefix - window + 1); j <= q + prefix; j++) { if (base - prefix + j >= 0) { mask[(size_t) q * n_kv + j] = 0.0f; } } } set_input_attn_mask(mask); }; // rope positions and attention mask of the mimi encoder transformer (pocket-tts, qwen3tts reference encoder). // the mask is causal with a sliding window, see _build_attention_mask() in the reference auto set_mimi_tfm_inputs = [&](int64_t max_samples) { const int64_t n_pos = ggml_nelements(get_inp_tensor("inp_pos")); GGML_ASSERT(n_pos > 0); std::vector positions((size_t) n_pos); for (int64_t i = 0; i < n_pos; i++) { positions[(size_t) i] = (int32_t) i; } set_input_i32("inp_pos", positions); // the caller truncates the waveform to keep this mask bounded int64_t hop = 1; for (int32_t ratio : hparams.seanet_ratios) { hop *= ratio; } const int64_t max_pos = (max_samples + hop - 1) / hop; GGML_ASSERT(n_pos <= max_pos && "speaker reference too long for a dense mask"); const int64_t context = hparams.mimi_tfm_context; std::vector mask((size_t) n_pos * n_pos, -INFINITY); for (int64_t q = 0; q < n_pos; q++) { for (int64_t k = 0; k < n_pos; k++) { const int64_t delta = q - k; if (delta >= 0 && delta < context) { mask[(size_t) q * n_pos + k] = 0.0f; } } } set_input_attn_mask(mask); }; // set input pixel values if (!imgs.is_audio) { size_t nelem = 0; for (const auto & img : imgs.entries) { nelem += img.nx() * img.ny() * 3; } std::vector inp_raw(nelem); // layout of data (note: the channel dim is unrolled to better visualize the layout): // // ┌──W──┐ // │ H │ channel = R // ├─────┤ │ // │ H │ channel = G // ├─────┤ │ // │ H │ channel = B // └─────┘ │ // ──────┘ x B // IMPORTANT: [QWEN_VIDEO] the batch dim is currently used for temporal dim in Qwen-VL models // All entries must have the same spatial size (enforced by can_batch_with() during merging) { const int nx = imgs.entries[0].nx(); const int ny = imgs.entries[0].ny(); const int n = nx * ny; for (int b = 0; b < n_batch_cur; b++) { LOG_DBG("%s: copying image %d/%d to input buffer (nx=%d, ny=%d)\n", __func__, b+1, n_batch_cur, nx, ny); const auto & buf = imgs.entries[b].get_ro_buf(); float * batch_entry = inp_raw.data() + b * (3*n); for (int y = 0; y < ny; y++) { for (int x = 0; x < nx; x++) { size_t base_src = 3*(y * nx + x); size_t base_dst = y * nx + x; batch_entry[ base_dst] = buf[base_src ]; batch_entry[1*n + base_dst] = buf[base_src + 1]; batch_entry[2*n + base_dst] = buf[base_src + 2]; } } } } set_input_f32("inp_raw", inp_raw); } else if (params->gen_process != MTMD_SUBCTX_GEN_PROCESS_GEN_WAV && params->gen_process != MTMD_SUBCTX_GEN_PROCESS_TTS && params->gen_process != MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE && ggml_graph_get_tensor(gf, "inp_raw") != nullptr) { // audio input. GEN_WAV and the chatterbox TTS stages are not here: they take codes or feats, set in the switch below. // a GEN_CODE graph may have no hidden-state input (lfm3a_gen feedback embedding) GGML_ASSERT(imgs.entries.size() == 1); const auto & mel_inp = imgs.entries[0]; const auto & buf = mel_inp.get_ro_buf(); const int n_step = mel_inp.nx(); const int n_mel = mel_inp.ny(); GGML_ASSERT((size_t)n_step * n_mel == buf.size()); set_input_f32("inp_raw", buf); } // set input per projector switch (model.proj_type) { case PROJECTOR_TYPE_VOXTRAL_RT: { mtmd_subctx_enc_state * st = params->enc_state; if (!st) { LOG_ERR("%s: this encoder needs a mtmd_subctx_enc_state\n", __func__); return false; } const int n_window = hparams.attn_window_size; const int n_new = (imgs.entries[0].nx() - 2) / 2; GGML_ASSERT(n_new <= st->n_ctx - n_window); std::vector pos(n_new); std::vector idx(n_new); std::vector slot_pos = st->slot_pos; // including the frames written by this call for (int i = 0; i < n_new; i++) { pos[i] = st->n_past + i; idx[i] = pos[i] % st->n_ctx; slot_pos[idx[i]] = pos[i]; } set_input_i32("inp_pos", pos); ggml_backend_tensor_set(get_inp_tensor("inp_kv_idx"), idx.data(), 0, idx.size() * sizeof(int64_t)); // each frame sees itself and the n_window - 1 frames before it std::vector mask((size_t) st->n_ctx * n_new, -INFINITY); for (int i = 0; i < n_new; i++) { for (int s = 0; s < st->n_ctx; s++) { const int32_t p = slot_pos[s]; if (p >= 0 && p <= pos[i] && p > pos[i] - n_window) { mask[(size_t) i * st->n_ctx + s] = 0.0f; } } } set_input_attn_mask(mask); std::vector tail = st->stem_tail; tail.resize(hparams.n_embd, 0.0f); set_input_f32("inp_stem_tail", tail); } break; case PROJECTOR_TYPE_INKLING2V: { // CLS is at position 0, the patches are 1-indexed std::vector positions_w{0}, positions_h{0}; for (int y = 0; y < pos_h; ++y) { for (int x = 0; x < pos_w; ++x) { positions_w.push_back(x + 1); positions_h.push_back(y + 1); } } set_input_i32("pos_w", positions_w); set_input_i32("pos_h", positions_h); } break; case PROJECTOR_TYPE_MUSE_GLIMMER: { const int grid_w = pos_w; // image_size_width / patch_size const int grid_h = pos_h; // image_size_height / patch_size const int n_tok = grid_w * grid_h; const int pgrid = (int) std::sqrt((double) model.position_embeddings->ne[1]); // 32 const int f = hparams.n_merge; // downsample 2 // pixel patchify runs inside the graph via build_inp() (ggml_conv_2d); // pos-emb bilinear interp via resize_position_embeddings(). // --- sparse window grouping (pgrid x pgrid windows) --- const int win = pgrid; const int nwin_h = (grid_h + win - 1) / win; const int nwin_w = (grid_w + win - 1) / win; std::vector sp_perm; sp_perm.reserve(n_tok); std::vector sp_slens; for (int wy = 0; wy < nwin_h; wy++) { for (int wx = 0; wx < nwin_w; wx++) { int cnt = 0; for (int hh = 0; hh < win; hh++) { for (int ww = 0; ww < win; ww++) { const int gy = wy * win + hh; const int gx = wx * win + ww; if (gy < grid_h && gx < grid_w) { sp_perm.push_back(gy * grid_w + gx); cnt++; } } } if (cnt > 0) sp_slens.push_back(cnt); } } std::vector rpos_w(n_tok), rpos_h(n_tok), inv_perm(n_tok); for (int i = 0; i < n_tok; i++) { const int orig = sp_perm[i]; rpos_w[i] = (orig % grid_w) + 1; // 1-indexed rpos_h[i] = (orig / grid_w) + 1; inv_perm[orig] = i; } set_input_i32("muse_glimmer_sp_perm", sp_perm); set_input_i32("muse_glimmer_inv_perm", inv_perm); set_input_i32("muse_glimmer_pos_w", rpos_w); set_input_i32("muse_glimmer_pos_h", rpos_h); // block-diagonal window mask (permuted order) std::vector sp_mask((size_t) n_tok * n_tok, -INFINITY); { int off = 0; for (int s : sp_slens) { for (int a = 0; a < s; a++) for (int b = 0; b < s; b++) sp_mask[(size_t) (off + a) * n_tok + (off + b)] = 0.0f; off += s; } } set_input_attn_mask(sp_mask); // pixel-shuffle gather (original order): f*f spatial neighbours grouped std::vector dsp; dsp.reserve(n_tok); for (int oy = 0; oy < grid_h / f; oy++) for (int ox = 0; ox < grid_w / f; ox++) for (int ry = 0; ry < f; ry++) for (int rx = 0; rx < f; rx++) dsp.push_back((oy * f + ry) * grid_w + (ox * f + rx)); set_input_i32("muse_glimmer_ds_perm", dsp); } break; case PROJECTOR_TYPE_MINICPMV: { // inspired from siglip: // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit/blob/d66538faeba44480d0bfaa42145eef26f9423199/modeling_siglip.py#L316 std::vector positions(pos_h * pos_w); // note: sized by the actual patch counts; a tall/wide image produces more // than 1024 patches per side and a fixed [1024] array would be overrun std::vector bucket_coords_h(pos_h); std::vector bucket_coords_w(pos_w); for (int i = 0; i < pos_h; i++){ bucket_coords_h[i] = std::floor(70.0*i/pos_h); } for (int i = 0; i < pos_w; i++){ bucket_coords_w[i] = std::floor(70.0*i/pos_w); } for (int i = 0, id = 0; i < pos_h; i++){ for (int j = 0; j < pos_w; j++){ positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j]; } } set_input_i32("positions", positions); // inputs for resampler projector // set the 2D positions (using float for sinusoidal embedding) int n_patches_per_col = image_size_width / patch_size; std::vector pos_data(n_pos); // dimension H for (int i = 0; i < n_pos; i++) { pos_data[i] = static_cast(i / n_patches_per_col); } set_input_f32("pos_h", pos_data); // dimension W for (int i = 0; i < n_pos; i++) { pos_data[i] = static_cast(i % n_patches_per_col); } set_input_f32("pos_w", pos_data); // base frequency omega const float base_freq = 10000.0f; const int n_embd_proj = model.n_mmproj_embd(); std::vector omega(n_embd_proj / 4); for (int i = 0; i < n_embd_proj / 4; ++i) { omega[i] = 1.0f / std::pow(base_freq, static_cast(i) / (n_embd_proj / 4)); } set_input_f32("omega", omega); } break; case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: case PROJECTOR_TYPE_MINICPM_OCR_1_0: { const bool is_4x = hparams.n_merge == 2; // SigLIP position buckets (same as resampler path) std::vector positions(pos_h * pos_w); std::vector bucket_coords_h(pos_h); std::vector bucket_coords_w(pos_w); for (int i = 0; i < pos_h; i++){ bucket_coords_h[i] = std::floor(70.0*i/pos_h); } for (int i = 0; i < pos_w; i++){ bucket_coords_w[i] = std::floor(70.0*i/pos_w); } for (int i = 0, id = 0; i < pos_h; i++){ for (int j = 0; j < pos_w; j++){ positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j]; } } set_input_i32("positions", positions); const int half_h = pos_h / 2; const int half_w = pos_w / 2; auto make_ds_idx = [](int off_r, int off_c, int ds_h, int ds_w, int stride_w) { std::vector idx(ds_h * ds_w); for (int i = 0; i < ds_h; i++) { for (int j = 0; j < ds_w; j++) { idx[i * ds_w + j] = (2*i + off_r) * stride_w + (2*j + off_c); } } return idx; }; if (!is_4x) { // window reorder indices for 2x2 windows std::vector window_idx(n_pos); std::vector inv_window_idx(n_pos); { int k = 0; for (int wi = 0; wi < half_h; wi++) { for (int wj = 0; wj < half_w; wj++) { window_idx[k++] = (2*wi ) * pos_w + (2*wj ); window_idx[k++] = (2*wi ) * pos_w + (2*wj + 1); window_idx[k++] = (2*wi + 1) * pos_w + (2*wj ); window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1); } } for (int i = 0; i < n_pos; i++) { inv_window_idx[window_idx[i]] = i; } } set_input_i32("vit_merger_window_idx", window_idx); set_input_i32("vit_merger_inv_window_idx", inv_window_idx); // block-diagonal attention mask: tokens in the same 4-token // window attend to each other (mask = 0), all other positions // are masked out (-inf). matches the window-major reorder above. std::vector window_mask_data(n_pos * n_pos, std::numeric_limits::lowest()); for (int wi = 0; wi < n_pos / 4; wi++) { for (int i = 0; i < 4; i++) { for (int j = 0; j < 4; j++) { window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f; } } } set_input_attn_mask(window_mask_data); // ViT merger 2x2 downsample indices auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w); auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w); auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w); auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w); set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0); set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1); set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2); set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3); } const int merger_h = is_4x ? pos_h : half_h; const int merger_w = is_4x ? pos_w : half_w; auto m_ds_0 = make_ds_idx(0, 0, merger_h / 2, merger_w / 2, merger_w); auto m_ds_1 = make_ds_idx(0, 1, merger_h / 2, merger_w / 2, merger_w); auto m_ds_2 = make_ds_idx(1, 0, merger_h / 2, merger_w / 2, merger_w); auto m_ds_3 = make_ds_idx(1, 1, merger_h / 2, merger_w / 2, merger_w); set_input_i32("merger_ds_idx_0", m_ds_0); set_input_i32("merger_ds_idx_1", m_ds_1); set_input_i32("merger_ds_idx_2", m_ds_2); set_input_i32("merger_ds_idx_3", m_ds_3); } break; case PROJECTOR_TYPE_LFM3VL: { // patches in raster-scan order (no interleave before encoder layers) const int pw = image_size_width / patch_size; const int ph = image_size_height / patch_size; std::vector positions(n_pos * 4); int ptr = 0; for (int y = 0; y < ph; y++) { for (int x = 0; x < pw; x++) { positions[ ptr] = y; positions[ num_patches + ptr] = x; positions[2 * num_patches + ptr] = y; positions[3 * num_patches + ptr] = x; ptr++; } } set_input_i32("positions", positions); } break; case PROJECTOR_TYPE_MAGEVL: { // t, h, w in 2x2 block order, t is always 0 for an image const int merge_ratio = hparams.n_merge; const int pw = image_size_width / patch_size / merge_ratio; const int ph = image_size_height / patch_size / merge_ratio; std::vector positions(n_pos * 4, 0); int ptr = 0; for (int ty = 0; ty < ph; ty++) { for (int tx = 0; tx < pw; tx++) { for (int dy = 0; dy < merge_ratio; dy++) { for (int dx = 0; dx < merge_ratio; dx++) { positions[ num_patches + ptr] = ty * merge_ratio + dy; positions[2 * num_patches + ptr] = tx * merge_ratio + dx; ptr++; } } } } set_input_i32("positions", positions); } break; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN3VL: case PROJECTOR_TYPE_LING3VL: case PROJECTOR_TYPE_GLM4V: case PROJECTOR_TYPE_GLM5V: { const int merge_ratio = hparams.n_merge; const int pw = image_size_width / patch_size / merge_ratio; const int ph = image_size_height / patch_size / merge_ratio; std::vector positions(n_pos * 4); int ptr = 0; for (int ty = 0; ty < ph; ty++) { for (int tx = 0; tx < pw; tx++) { for (int dy = 0; dy < merge_ratio; dy++) { for (int dx = 0; dx < merge_ratio; dx++) { const int y = ty * merge_ratio + dy; const int x = tx * merge_ratio + dx; positions[ ptr] = y; positions[ num_patches + ptr] = x; positions[2 * num_patches + ptr] = y; positions[3 * num_patches + ptr] = x; ptr++; } } } } set_input_i32("positions", positions); } break; case PROJECTOR_TYPE_STEP3VL: { std::vector pos_data(n_pos); for (int i = 0; i < n_pos; i++) { pos_data[i] = i / pos_w; } set_input_i32("pos_h", pos_data); for (int i = 0; i < n_pos; i++) { pos_data[i] = i % pos_w; } set_input_i32("pos_w", pos_data); } break; case PROJECTOR_TYPE_PADDLEOCR: { const int merge_ratio = hparams.n_merge; const int pw = image_size_width / patch_size / merge_ratio; const int ph = image_size_height / patch_size / merge_ratio; std::vector positions(n_pos * 4); int ptr = 0; // NOTE: same as Qwen-VL, but x and y are swapped for (int ty = 0; ty < ph; ty++) { for (int dy = 0; dy < merge_ratio; dy++) { for (int tx = 0; tx < pw; tx++) { for (int dx = 0; dx < merge_ratio; dx++) { const int y = ty * merge_ratio + dy; const int x = tx * merge_ratio + dx; positions[ ptr] = y; positions[ num_patches + ptr] = x; positions[2 * num_patches + ptr] = y; positions[3 * num_patches + ptr] = x; ptr++; } } } } set_input_i32("positions", positions); } break; case PROJECTOR_TYPE_MINIMAX_M3: { const int n_merge = hparams.n_merge; const int gh = image_size_height / patch_size; const int gw = image_size_width / patch_size; std::vector pos_h, pos_w; pos_h.reserve(gh * gw); pos_w.reserve(gh * gw); for (int bh = 0; bh < gh / n_merge; bh++) for (int bw = 0; bw < gw / n_merge; bw++) for (int mh = 0; mh < n_merge; mh++) for (int mw = 0; mw < n_merge; mw++) { pos_h.push_back(bh * n_merge + mh); pos_w.push_back(bw * n_merge + mw); } set_input_i32("minimax_pos_h", pos_h); set_input_i32("minimax_pos_w", pos_w); } break; case PROJECTOR_TYPE_DOTS_OCR: case PROJECTOR_TYPE_DOTS3NOTE_V: { const int pw = image_size_width / patch_size; const int ph = image_size_height / patch_size; const int n_pos = ph * pw; std::vector positions(n_pos * 4); int ptr = 0; // flat layout: [h, w, h, w] for each patch // patches are in raster order (matching conv2d output) for (int y = 0; y < ph; y++) { for (int x = 0; x < pw; x++) { positions[ ptr] = y; positions[ n_pos + ptr] = x; positions[2*n_pos + ptr] = y; positions[3*n_pos + ptr] = x; ptr++; } } set_input_i32("positions", positions); } break; case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_EXAONE4_5: case PROJECTOR_TYPE_YOUTUVL: { // pw * ph = number of tokens output by ViT after apply patch merger // ipw * ipw = number of vision token been processed inside ViT const bool use_window_attn = (model.proj_type == PROJECTOR_TYPE_QWEN25VL || model.proj_type == PROJECTOR_TYPE_EXAONE4_5) ? hparams.n_wa_pattern > 0 : !hparams.wa_layer_indexes.empty(); const int merge_ratio = 2; const int pw = image_size_width / patch_size / merge_ratio; const int ph = image_size_height / patch_size / merge_ratio; const int ipw = image_size_width / patch_size; const int iph = image_size_height / patch_size; std::vector idx (ph * pw); std::vector inv_idx(ph * pw); if (use_window_attn) { const int attn_window_size = hparams.attn_window_size > 0 ? hparams.attn_window_size : 112; const int grid_window = attn_window_size / patch_size / merge_ratio; int dst = 0; // [num_vision_tokens, num_vision_tokens] attention mask tensor std::vector mask(pow(ipw * iph, 2), std::numeric_limits::lowest()); int mask_row = 0; for (int y = 0; y < ph; y += grid_window) { for (int x = 0; x < pw; x += grid_window) { const int win_h = std::min(grid_window, ph - y); const int win_w = std::min(grid_window, pw - x); const int dst_0 = dst; // group all tokens belong to the same window togather (to a continue range) for (int dy = 0; dy < win_h; dy++) { for (int dx = 0; dx < win_w; dx++) { const int src = (y + dy) * pw + (x + dx); GGML_ASSERT(src < (int)idx.size()); GGML_ASSERT(dst < (int)inv_idx.size()); idx [src] = dst; inv_idx[dst] = src; dst++; } } for (int r=0; r < win_h * win_w * merge_ratio * merge_ratio; r++) { int row_offset = mask_row * (ipw * iph); std::fill( mask.begin() + row_offset + (dst_0 * merge_ratio * merge_ratio), mask.begin() + row_offset + (dst * merge_ratio * merge_ratio), 0.0); mask_row++; } } } set_input_i32("window_idx", idx); set_input_i32("inv_window_idx", inv_idx); set_input_attn_mask(mask); } else { for (int i = 0; i < ph * pw; i++) { idx[i] = i; } } const int mpow = merge_ratio * merge_ratio; std::vector positions(n_pos * 4); int ptr = 0; for (int ty = 0; ty < ph; ty++) { for (int tx = 0; tx < pw; tx++) { for (int dy = 0; dy < merge_ratio; dy++) { for (int dx = 0; dx < merge_ratio; dx++) { auto remap = idx[ptr / mpow]; remap = (remap * mpow) + (ptr % mpow); const int y = ty * merge_ratio + dy; const int x = tx * merge_ratio + dx; positions[ remap] = y; positions[ num_patches + remap] = x; positions[2 * num_patches + remap] = y; positions[3 * num_patches + remap] = x; ptr++; } } } } set_input_i32("positions", positions); } break; case PROJECTOR_TYPE_MIMOVL: { const int merge = hparams.n_merge; // 2 const int merge_unit = merge * merge; // 4 const int patch = hparams.patch_size; // 16 const int H = image_size_height / patch; const int W = image_size_width / patch; const int n_pos_full = H * W; const int llm_h = H / merge; const int llm_w = W / merge; const int n_units = llm_h * llm_w; // n_pos / merge_unit // Row-major merge-tile-ordered (h, w) positions std::vector pos_h_row(n_pos_full); std::vector pos_w_row(n_pos_full); { int idx = 0; for (int ty = 0; ty < llm_h; ty++) { for (int tx = 0; tx < llm_w; tx++) { for (int dy = 0; dy < merge; dy++) { for (int dx = 0; dx < merge; dx++) { pos_h_row[idx] = ty * merge + dy; pos_w_row[idx] = tx * merge + dx; idx++; } } } } } // Col-major merge-unit permutation std::vector idx_col(n_units); for (int r = 0; r < llm_h; r++) { for (int c = 0; c < llm_w; c++) { int u_row = r * llm_w + c; int u_col = c * llm_h + r; idx_col[u_col] = (float) u_row; } } // Col-mode positions: permute pos_*_row by idx_col std::vector pos_h_col(n_pos_full); std::vector pos_w_col(n_pos_full); for (int u = 0; u < n_units; u++) { int src = (int) idx_col[u]; for (int k = 0; k < merge_unit; k++) { pos_h_col[u * merge_unit + k] = pos_h_row[src * merge_unit + k]; pos_w_col[u * merge_unit + k] = pos_w_row[src * merge_unit + k]; } } // Pack into ggml_rope_multi VISION-mode layout. The non-CPU kernels // only read slots 0 and 1, so pack h in slot 0, w in slot 1: // positions[0..n_pos) = h // positions[n_pos..2*n_pos) = w // positions[2*n_pos..3*n_pos) = 0 // positions[3*n_pos..4*n_pos) = 0 std::vector positions_row(static_cast(n_pos_full) * 4, 0); std::vector positions_col(static_cast(n_pos_full) * 4, 0); for (int i = 0; i < n_pos_full; i++) { positions_row[0 * n_pos_full + i] = pos_h_row[i]; positions_row[1 * n_pos_full + i] = pos_w_row[i]; positions_col[0 * n_pos_full + i] = pos_h_col[i]; positions_col[1 * n_pos_full + i] = pos_w_col[i]; } // Banded 1D sliding-window mask const int window = hparams.attn_window_size; GGML_ASSERT(window > 0); std::vector mask(static_cast(n_pos_full) * n_pos_full, std::numeric_limits::lowest()); for (int q = 0; q < n_pos_full; q++) { int lo = std::max(0, q - window); int hi = std::min(n_pos_full - 1, q + window); for (int k = lo; k <= hi; k++) { mask[static_cast(q) * n_pos_full + k] = 0.0f; } } set_input_i32("mimovl_positions_row", positions_row); set_input_i32("mimovl_positions_col", positions_col); set_input_f32("mimovl_idx_col", idx_col); set_input_attn_mask(mask); } break; case PROJECTOR_TYPE_PIXTRAL: case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_KIMIK25: case PROJECTOR_TYPE_KIMIK31: case PROJECTOR_TYPE_LIGHTONOCR: { // set the 2D positions int n_patches_per_col = image_size_width / patch_size; std::vector pos_data(n_pos); // dimension H for (int i = 0; i < n_pos; i++) { pos_data[i] = i / n_patches_per_col; } set_input_i32("pos_h", pos_data); // dimension W for (int i = 0; i < n_pos; i++) { pos_data[i] = i % n_patches_per_col; } set_input_i32("pos_w", pos_data); } break; case PROJECTOR_TYPE_SENSENOVA_U1: { const int n_patches_per_row = image_size_width / patch_size; std::vector pos_data(n_pos); for (int i = 0; i < n_pos; i++) { pos_data[i] = i / n_patches_per_row; } set_input_i32("pos_y", pos_data); for (int i = 0; i < n_pos; i++) { pos_data[i] = i % n_patches_per_row; } set_input_i32("pos_x", pos_data); } break; case PROJECTOR_TYPE_DEEPSEEK4V: { // set the 2D positions (mrope layout, only the first 2 channels are used) int n_patches_per_row = image_size_width / patch_size; std::vector positions(n_pos * 4, 0); for (int i = 0; i < n_pos; i++) { positions[i] = i / n_patches_per_row; // row positions[n_pos + i] = i % n_patches_per_row; // col } set_input_i32("positions", positions); // token block layout index (see mtmd_graph_deepseek4v::build) // rows [0, n_grid) are the aligner output, the sentinels follow const int n_merge = hparams.n_merge; const int n_llm_w = MTMD_ALIGN(pos_w, n_merge) / n_merge; const int n_llm_h = MTMD_ALIGN(pos_h, n_merge) / n_merge; const int n_grid = n_llm_w * n_llm_h; const int idx_start = n_grid; const int idx_end = n_grid + 1; const int idx_newline = n_grid + 2; const int idx_pad = n_grid + 3; const int lead_pad = imgs.entries[0].lead_pad; const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad); std::vector idx; idx.reserve(bl.n_out); for (int i = 0; i < lead_pad; i++) { idx.push_back(idx_pad); } idx.push_back(idx_start); // pairs of adjacent rows are interleaved column-wise ("N-layout") // ref: build_image_block in inference/image_processor.py for (int t = 0; t < bl.rows * bl.row_len; t++) { const int g = t / (2 * bl.row_len); const int rem = t % (2 * bl.row_len); const int c = rem / 2; // column const int r = 2 * g + rem % 2; // row if (r >= n_llm_h) { idx.push_back(idx_pad); } else if (c == n_llm_w) { idx.push_back(idx_newline); } else { idx.push_back(r * n_llm_w + c); } } for (int i = 0; i < bl.pad_last; i++) { idx.push_back(idx_pad); } idx.push_back(idx_end); set_input_i32("layout_idx", idx); } break; case PROJECTOR_TYPE_DEEPSEEK41V: { // mrope layout, only the first 2 channels are used const int n_patches_per_row = image_size_width / patch_size; std::vector positions(n_pos * 4, 0); for (int i = 0; i < n_pos; i++) { positions[i] = i / n_patches_per_row; positions[n_pos + i] = i % n_patches_per_row; } set_input_i32("positions", positions); // reading-order block: rows [0, n_grid) are the aligner output, then start / end / newline const int n_merge = hparams.n_merge; const int n_llm_w = MTMD_ALIGN(pos_w, n_merge) / n_merge; const int n_llm_h = MTMD_ALIGN(pos_h, n_merge) / n_merge; const int n_grid = n_llm_w * n_llm_h; std::vector idx; idx.reserve(dsv41_n_block_tokens(n_llm_w, n_llm_h)); idx.push_back(n_grid); // IMAGE_START for (int r = 0; r < n_llm_h; r++) { for (int c = 0; c < n_llm_w; c++) { idx.push_back(r * n_llm_w + c); } idx.push_back(n_grid + 2); // IMAGE_NEW_LINE } idx.push_back(n_grid + 1); // IMAGE_END set_input_i32("layout_idx", idx); } break; case PROJECTOR_TYPE_GLM_EDGE: { // llava and other models std::vector positions(n_pos); for (int i = 0; i < n_pos; i++) { positions[i] = i; } set_input_i32("positions", positions); } break; case PROJECTOR_TYPE_MLP: case PROJECTOR_TYPE_MLP_NORM: case PROJECTOR_TYPE_LDP: case PROJECTOR_TYPE_LDPV2: { // llava and other models std::vector positions(n_pos); for (int i = 0; i < n_pos; i++) { positions[i] = i; } set_input_i32("positions", positions); // The patches vector is used to get rows to index into the embeds with; // we should skip dim 0 only if we have CLS to avoid going out of bounds // when retrieving the rows. int patch_offset = model.class_embedding ? 1 : 0; std::vector patches(num_patches); for (int i = 0; i < num_patches; i++) { patches[i] = i + patch_offset; } set_input_i32("patches", patches); } break; case PROJECTOR_TYPE_POCKETTTS_SPKENC: { set_mimi_tfm_inputs((int64_t) mtmd_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate); } break; case PROJECTOR_TYPE_POCKETTTS_GEN: { if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { GGML_ASSERT(params->feats != nullptr); set_input_f32("inp_feats", *params->feats); // positions and mask are derived in-graph from the persisted counter set_gen_state_in(); } else { // flow matching starts from gaussian noise, std = sqrt(temp), no noise at temp 0 ggml_tensor * t = get_inp_tensor("inp_noise"); std::vector noise(ggml_nelements(t), 0.0f); if (params->temp > 0.0f) { std::normal_distribution dist(0.0f, std::sqrt(params->temp)); for (auto & v : noise) { v = dist(params->gen_state->rng); } } set_input_f32("inp_noise", noise); } } break; case PROJECTOR_TYPE_SOPRANO: case PROJECTOR_TYPE_YUE2_GEN: { set_input_f32("inp_feats", *params->feats); } break; case PROJECTOR_TYPE_KOKORO: { const auto * io = params->kokoro; GGML_ASSERT(io); switch (io->stage) { case mtmd_graph_kokoro::STAGE_TEXT: { std::vector tokens = *io->tokens; set_input_i32("inp_tokens", tokens); } break; case mtmd_graph_kokoro::STAGE_F0N: set_input_f32("inp_x", *io->x); set_input_f32("inp_style", *io->style); break; case mtmd_graph_kokoro::STAGE_DEC: set_input_f32("inp_asr", *io->x); set_input_f32("inp_style", *io->style); set_input_f32("inp_f0", *io->f0); set_input_f32("inp_n", *io->en); set_input_f32("inp_har", *io->har); break; } } break; case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN: { set_input_f32("inp_feats", *params->feats); if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE) { if (params->code0 < 0 || (int64_t) params->code0 >= model.gen_code_out_embd_w->ne[1]) { LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) model.gen_code_out_embd_w->ne[1]); return false; } std::vector code0 = { params->code0 }; set_input_i32("inp_code0", code0); // greedy (temp <= 0) has no random input if (ggml_graph_get_tensor(gf, "inp_rand") != nullptr) { std::uniform_real_distribution dist(0.0f, 1.0f); std::vector r(model.gen_code_head_w->ne[2]); for (auto & v : r) { v = dist(params->gen_state->rng); } set_input_f32("inp_rand", r); } } else if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_FLOW_COND) { // F.interpolate(mode="nearest") of the reference ggml_tensor * t = get_inp_tensor("inp_idx"); const int64_t n_in = (int64_t) params->feats->size() / model.hparams.n_embd; const int64_t n_out = t->ne[0]; const float scale = (float) n_in / (float) n_out; std::vector idx(n_out); for (int64_t i = 0; i < n_out; i++) { idx[i] = (int32_t) std::min((int64_t) floorf((float) i * scale), n_in - 1); } set_input_i32("inp_idx", idx); } else if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_FLOW_STEP) { set_input_f32("inp_t", { params->flow_t }); } } break; case PROJECTOR_TYPE_GEMMA4V: case PROJECTOR_TYPE_GEMMA4UV: { // set (col, row) patch positions for learned positional embedding const int n_cols = image_size_width / patch_size; std::vector pos_x(num_patches), pos_y(num_patches); for (int i = 0; i < num_patches; i++) { pos_x[i] = i % n_cols; pos_y[i] = i / n_cols; } set_input_i32("pos_x", pos_x); set_input_i32("pos_y", pos_y); } break; case PROJECTOR_TYPE_DEEPSEEKOCR: case PROJECTOR_TYPE_DEEPSEEKOCR2: { GGML_ASSERT( (pos_w == pos_h) // overview image || (pos_h >= pos_w && pos_h % pos_w == 0) // tile images ); const int window = hparams.attn_window_size; const int pos = pos_w; std::vector rel_pos_indices_local(window * window); std::vector rel_pos_indices_global(pos * pos); for (int q = 0; q < window; q++) { for (int k = 0; k < window; k++) { rel_pos_indices_local[q * window + k] = q - k + window - 1; } } for (int q = 0; q < pos; q++) { for (int k = 0; k < pos; k++) { rel_pos_indices_global[q * pos + k] = q - k + pos - 1; } } set_input_i32("rel_pos_indices_local", rel_pos_indices_local); set_input_i32("rel_pos_indices_global", rel_pos_indices_global); if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2) { // qwen2 encoder attention mask // num_image_tokens = num_patches / 16 // 256 for 1024 global view // 144 for 768 tile views const int num_image_tokens = num_patches / 16; const int seq_len = num_image_tokens * 2; std::vector qwen2_mask(static_cast(seq_len) * seq_len, 0.0f); // attention mask layout // +--------------+---------------+ // | all 0 | all -inf | // +--------------+---------------+ // | all 0 | lower tri 0 | // +--------------+---------------+ for (int i = 0; i < seq_len; i++) { for (int j = 0; j < seq_len; j++) { const bool zero = i < num_image_tokens ? j < num_image_tokens : j < num_image_tokens || j <= i; qwen2_mask[static_cast(i) * seq_len + j] = zero ? 0.0f : -1e9f; } } set_input_attn_mask(qwen2_mask); } } break; case PROJECTOR_TYPE_CHATTERBOX: { if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE) { set_input_f32("inp_mel", *params->mel_in); set_input_f32("inp_sstft", *params->sstft_in); break; } if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) { const int T1 = (imgs.entries[0].nx() - 1) / 2 + 1; const int T2 = (T1 - 1) / 2 + 1; std::vector pos(T2); for (int i = 0; i < T2; i++) { pos[(size_t) i] = i; } set_input_i32("inp_pos", pos); break; } if (params->gen_process != MTMD_SUBCTX_GEN_PROCESS_TTS) { break; } const int n_gen = (int) params->codes->size(); GGML_ASSERT(params->ref); const int n_prompt = (int) params->ref->tokens.size(); std::vector tokens = params->ref->tokens; tokens.resize((size_t) n_prompt + n_gen); memcpy(tokens.data() + n_prompt, params->codes->data(), (size_t) n_gen * sizeof(int32_t)); const int64_t n_codes = model.cbx.input_embedding_w->ne[1]; for (int32_t t : tokens) { if (t < 0 || t >= n_codes) { LOG_ERR("%s: speech token out of range (%d, codebook size %lld)\n", __func__, t, (long long) n_codes); return false; } } const int T1 = n_prompt + n_gen; const int T2 = 2 * T1; set_input_i32("inp_tokens", tokens); // mel-rate reference conditioning of the voice set_input_f32("inp_prompt_feat", params->ref->feat); set_input_f32("inp_xvec", params->ref->spk); // espnet relative positional encoding, entry k holds the sinusoid of relative position (T-1) - k auto fill_pos = [&](const char * name, int T) { const int d = 512; std::vector pos((size_t) (2 * T - 1) * d); for (int k = 0; k < 2 * T - 1; k++) { const double rel = (double) (T - 1 - k); for (int i = 0; i < d / 2; i++) { const double div = exp(-(double) (2 * i) * log(10000.0) / d); pos[(size_t) k * d + 2 * i ] = (float) sin(rel * div); pos[(size_t) k * d + 2 * i + 1] = (float) cos(rel * div); } } set_input_f32(name, pos); }; fill_pos("inp_pos1", T1); fill_pos("inp_pos2", T2); // noise and the time embeddings of the solver span points, matcha layout (sines then cosines, scale 1000) std::vector noise((size_t) 80 * T2); std::normal_distribution nd(0.0f, 1.0f); for (auto & f : noise) f = nd(params->gen_state->rng); set_input_f32("inp_noise", noise); const bool meanflow = model.cbx.time_embed_mixer_w != nullptr; const int n_steps = meanflow ? 2 : 10; std::vector temb((size_t) 320 * (n_steps + 1)); for (int s = 0; s <= n_steps; s++) { const double u = (double) s / n_steps; const double t = meanflow ? u : 1.0 - cos(u * M_PI / 2.0); for (int i = 0; i < 160; i++) { const double div = exp(-(double) i * log(10000.0) / 159.0); temb[(size_t) s * 320 + i ] = (float) sin(1000.0 * t * div); temb[(size_t) s * 320 + i + 160] = (float) cos(1000.0 * t * div); } } set_input_f32("inp_temb", temb); } break; case PROJECTOR_TYPE_GEMMA3: case PROJECTOR_TYPE_GEMMA3NV: case PROJECTOR_TYPE_IDEFICS3: case PROJECTOR_TYPE_COHERE2V: case PROJECTOR_TYPE_INTERNVL: case PROJECTOR_TYPE_NEMOTRON_V2_VL: case PROJECTOR_TYPE_NEMOTRON_OMNI: case PROJECTOR_TYPE_QWEN2A: case PROJECTOR_TYPE_HIGGS_AUDIO: case PROJECTOR_TYPE_QWEN3A: case PROJECTOR_TYPE_GLMA: case PROJECTOR_TYPE_ULTRAVOX: case PROJECTOR_TYPE_LFM2: case PROJECTOR_TYPE_D1OMNI_V: case PROJECTOR_TYPE_VOXTRAL: case PROJECTOR_TYPE_MERALION: case PROJECTOR_TYPE_MUSIC_FLAMINGO: case PROJECTOR_TYPE_JANUS_PRO: case PROJECTOR_TYPE_PHI4: case PROJECTOR_TYPE_COGVLM: case PROJECTOR_TYPE_YASA2: case PROJECTOR_TYPE_GEMMA4UA: case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: case PROJECTOR_TYPE_QWEN3TTS_SPKENC: case PROJECTOR_TYPE_ZONOS3_SPKENC: case PROJECTOR_TYPE_FISH_S2_SPKENC: { // do nothing } break; case PROJECTOR_TYPE_CHATTERBOX_SPKENC: { // the ceil-mode seg pooling divides by the full segment length, rescale the last partial segment const int T1 = (imgs.entries[0].nx() - 1) / 2 + 1; const int S = (T1 + 99) / 100; std::vector segfix((size_t) S, 1.0f); const int last = T1 - (S - 1) * 100; segfix[(size_t) S - 1] = 100.0f / (float) last; set_input_f32("inp_segfix", segfix); } break; case PROJECTOR_TYPE_ZONOS3_GEN: if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { // frame-major codes -> codebook-major rows; a residual code of -1 is missing and left out const int64_t n_cb = mtmd_graph_zonos3_gen::n_codebooks; const int64_t n_codes = mtmd_graph_zonos3_gen::n_cb_vocab; GGML_ASSERT(params->codes != nullptr && params->codes->size() % n_cb == 0); const int64_t n_frames = (int64_t) params->codes->size() / n_cb; std::vector rows(params->codes->size()); std::vector keep((size_t) n_frames * (n_cb - 1), 1.0f); for (int64_t t = 0; t < n_frames; t++) { for (int64_t c = 0; c < n_cb; c++) { int32_t code = (*params->codes)[t * n_cb + c]; if (code == -1 && c > 0) { keep[(c - 1) * n_frames + t] = 0.0f; code = 0; } if (code < 0 || code >= n_codes) { LOG_ERR("%s: code out of range (%d, codebook size %lld)\n", __func__, code, (long long) n_codes); return false; } rows[c * n_frames + t] = code + (int32_t) (c == 0 ? 0 : (c - 1) * n_codes); } } set_input_i32("inp_codes", rows); set_input_f32("inp_keep", keep); set_gen_state_in(); } else { const int64_t n_cb = mtmd_graph_zonos3_gen::n_codebooks; const int64_t n_vocab = model.gen_code_out_embd_w->ne[1] / n_cb; // no codes: logits only, the row of codes 0 is ignored std::vector codes(n_cb, 0); if (params->codes) { if (params->codes->size() != (size_t) n_cb) { LOG_ERR("%s: expected %lld codes, got %zu\n", __func__, (long long) n_cb, params->codes->size()); return false; } codes = *params->codes; } for (int64_t i = 0; i < n_cb; i++) { if (codes[i] < 0 || codes[i] >= n_vocab) { LOG_ERR("%s: code out of range (%d, vocab %lld)\n", __func__, codes[i], (long long) n_vocab); return false; } codes[i] += (int32_t) (i * n_vocab); } set_input_i32("inp_codes", codes); } break; case PROJECTOR_TYPE_LFM3A_GEN: { const int64_t n_cb = model.gen_code_embd_w->ne[2]; const int64_t n_vocab = model.gen_code_embd_w->ne[1]; if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { GGML_ASSERT(params->codes != nullptr); // codes index the FSQ table via ggml_get_rows, the end-of-audio code has no waveform const int64_t n_codes = model.c2w.quant_first_cb_w->ne[1]; for (int32_t c : *params->codes) { if (c < 0 || c >= n_codes) { LOG_ERR("%s: code out of range (%d, codebook size %lld)\n", __func__, c, (long long) n_codes); return false; } } std::vector codes = *params->codes; set_input_i32("inp_codes", codes); set_gen_state_in(); // the "pos" slot leads the state float base = 0.0f; ggml_backend_tensor_get(params->gen_state->slots[0], &base, 0, sizeof(float)); set_lfm3a_window_inputs((int) base, (int) ggml_nelements(get_inp_tensor("positions")), hparams.wav_tfm_swa); } else { // rows of the merged per-codebook tables; a masked codebook (-1) takes row 0, the mask drops it std::vector codes(n_cb, -1); if (params->codes) { GGML_ASSERT(params->codes->size() == (size_t) n_cb); codes = *params->codes; } std::vector mask(n_cb, 0.0f); for (int64_t i = 0; i < n_cb; i++) { if (codes[i] >= n_vocab) { LOG_ERR("%s: code out of range (%d, vocab %lld)\n", __func__, codes[i], (long long) n_vocab); return false; } if (codes[i] < 0) { codes[i] = 0; } else { mask[i] = 1.0f; } codes[i] += (int32_t) (i * n_vocab); } set_input_i32("inp_codes", codes); if (ggml_graph_get_tensor(gf, "inp_mask") != nullptr) { set_input_f32("inp_mask", mask); } } } break; case PROJECTOR_TYPE_DRAMA3_GEN: case PROJECTOR_TYPE_FISH_S2_GEN: { const int64_t n_cb = model.gen_code_embd_w->ne[2]; const int64_t n_vocab = model.gen_code_embd_w->ne[1]; if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { // frame-major to group-major, clamp: codec residual codebooks are smaller than the tower vocab GGML_ASSERT(params->codes != nullptr && params->codes->size() % n_cb == 0); const int64_t n_frames = (int64_t) params->codes->size() / n_cb; const int64_t n_first = model.c2w.quant_first_cb_w->ne[1]; const int64_t n_rest = model.c2w.quant_rest_cb_w->ne[1]; std::vector codes(params->codes->size()); for (int64_t f = 0; f < n_frames; f++) { for (int64_t g = 0; g < n_cb; g++) { const int64_t n = g == 0 ? n_first : n_rest; codes[g * n_frames + f] = std::clamp((*params->codes)[f * n_cb + g], 0, (int32_t) n - 1); } } set_input_i32("inp_codes", codes); set_gen_state_in(); } else if (model.proj_type == PROJECTOR_TYPE_FISH_S2_GEN) { if (params->code0 < 0 || params->code0 >= n_vocab) { LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) n_vocab); return false; } std::vector code0 = { params->code0 }; set_input_i32("inp_code0", code0); } else { // code0 < 0: next row is text (prompt), the tower only fills its KV // codes: input frame codes for a speech row, empty for a text row const bool next_speech = params->code0 >= 0; const bool cur_speech = params->codes && (int64_t) params->codes->size() == n_cb; if (params->code0 >= n_vocab) { LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) n_vocab); return false; } std::vector code0 = { std::max(params->code0, 0) }; set_input_i32("inp_code0", code0); std::vector extra(n_cb - 2); for (int64_t g = 2; g < n_cb; g++) { const int32_t c = cur_speech ? std::clamp((*params->codes)[g], 0, (int32_t) n_vocab - 1) : 0; extra[g - 2] = c + (int32_t) (g * n_vocab); } set_input_i32("inp_extra", extra); set_input_f32("inp_scale", { next_speech ? 1.0f : 0.0f, cur_speech ? 1.0f : 0.0f }); float pos = 0.0f; ggml_backend_tensor_get(params->gen_state->slots[0], &pos, 0, sizeof(float)); std::vector inp_pos = { (int32_t) pos }; set_input_i32("inp_pos", inp_pos); std::uniform_real_distribution dist(0.0f, 1.0f); std::vector r(model.gen_code_head_w->ne[2]); for (auto & v : r) { v = dist(params->gen_state->rng); } set_input_f32("inp_rand", r); } } break; case PROJECTOR_TYPE_QWEN3TTS_GEN: { if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) { set_mimi_tfm_inputs((int64_t) mtmd_hparams::qwen3tts_max_ref_seconds * hparams.audio_sample_rate); break; } if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { GGML_ASSERT(params->codes != nullptr); // frame-major input to group-major, rear-padded with code 0 up to one window const int64_t n_codes = model.gen_code_head_w->ne[2] + 1; const int64_t n_frames_w = hparams.wav_tfm_swa; const int64_t n_frames = (int64_t) params->codes->size() / n_codes; GGML_ASSERT(n_frames > 0 && n_frames <= n_frames_w); // codes are used as ggml_get_rows indices, so check them against the codebook vocab const int64_t vocab_first = model.c2w.quant_first_cb_w->ne[1]; const int64_t vocab_rest = model.c2w.quant_rest_cb_w->ne[1]; for (int64_t f = 0; f < n_frames; f++) { for (int64_t g = 0; g < n_codes; g++) { const int32_t c = (*params->codes)[f * n_codes + g]; const int64_t vocab = (g == 0) ? vocab_first : vocab_rest; if (c < 0 || (int64_t) c >= vocab) { LOG_ERR("%s: code out of range (frame %lld, group %lld, code %d, vocab %lld)\n", __func__, (long long) f, (long long) g, c, (long long) vocab); return false; } } } std::vector codes(n_frames_w * n_codes, 0); for (int64_t f = 0; f < n_frames; f++) { for (int64_t g = 0; g < n_codes; g++) { codes[g * n_frames_w + f] = (*params->codes)[f * n_codes + g]; } } set_input_i32("inp_codes", codes); set_gen_state_in(); } else { // code0 indexes gen_code_out_embd_w via ggml_get_rows; bound it const int64_t vocab0 = model.gen_code_out_embd_w->ne[1]; if (params->code0 < 0 || (int64_t) params->code0 >= vocab0) { LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) vocab0); return false; } std::vector code0 = { params->code0 }; set_input_i32("inp_code0", code0); // one uniform(0,1) draw per codebook, used by do_sampling() std::uniform_real_distribution dist(0.0f, 1.0f); const int64_t n_acoustic = model.gen_code_head_w->ne[2]; for (int64_t g = 0; g < n_acoustic; g++) { std::vector r = { dist(params->gen_state->rng) }; set_input_f32(("inp_rand_" + std::to_string(g)).c_str(), r); } } } break; case PROJECTOR_TYPE_HUNYUANVL: { // Compute the HunyuanVL 2D position embedding on CPU (with the // custom sf=(target+0.1)/n_grid bilinear sampling that the // reference implementation uses) and upload it to the graph // input declared in mtmd_graph_hunyuanvl::build(). GGML_ASSERT(model.position_embeddings != nullptr); ggml_tensor * src_t = model.position_embeddings; const int64_t n_embd = src_t->ne[0]; const int64_t n_pos = src_t->ne[1]; // = n_grid * n_grid const int n_grid = (int)std::lround(std::sqrt((double)n_pos)); GGML_ASSERT((int64_t)n_grid * n_grid == n_pos); const int out_w = pos_w; // pw const int out_h = pos_h; // ph // Pull weight to host. std::vector src(n_embd * n_pos); ggml_backend_tensor_get(src_t, src.data(), 0, ggml_nbytes(src_t)); // Output layout matches ggml_new_tensor_2d(F32, n_embd, out_h*out_w): // ne[0] = n_embd (fastest), ne[1] = out_h*out_w // dst[(y*out_w + x) * n_embd + c] std::vector dst((size_t)n_embd * out_h * out_w); const float sx = (float)(out_w + 0.1f) / (float)n_grid; const float sy = (float)(out_h + 0.1f) / (float)n_grid; for (int y = 0; y < out_h; ++y) { // Match ggml_compute_forward_upscale_f32 pixel-center // convention (align_corners=False): src_y = (y+0.5)/sy - 0.5. const float fy = ((float)y + 0.5f) / sy - 0.5f; int y0 = (int)std::floor(fy); int y1 = y0 + 1; y0 = std::clamp(y0, 0, n_grid - 1); y1 = std::clamp(y1, 0, n_grid - 1); float wy1 = std::clamp(fy - (float)y0, 0.0f, 1.0f); const float wy0 = 1.0f - wy1; for (int x = 0; x < out_w; ++x) { const float fx = ((float)x + 0.5f) / sx - 0.5f; int x0 = (int)std::floor(fx); int x1 = x0 + 1; x0 = std::clamp(x0, 0, n_grid - 1); x1 = std::clamp(x1, 0, n_grid - 1); float wx1 = std::clamp(fx - (float)x0, 0.0f, 1.0f); const float wx0 = 1.0f - wx1; const float w00 = wy0 * wx0; const float w01 = wy0 * wx1; const float w10 = wy1 * wx0; const float w11 = wy1 * wx1; const float * s00 = &src[((size_t)y0 * n_grid + x0) * n_embd]; const float * s01 = &src[((size_t)y0 * n_grid + x1) * n_embd]; const float * s10 = &src[((size_t)y1 * n_grid + x0) * n_embd]; const float * s11 = &src[((size_t)y1 * n_grid + x1) * n_embd]; float * d = &dst[((size_t)y * out_w + x) * n_embd]; for (int c = 0; c < n_embd; ++c) { d[c] = w00 * s00[c] + w01 * s01[c] + w10 * s10[c] + w11 * s11[c]; } } } set_input_f32("hunyuanvl_pos_embd", dst); } break; case PROJECTOR_TYPE_LLAMA4: { // set the 2D positions int n_patches_per_col = image_size_width / patch_size; std::vector pos_data(num_patches + 1, 0); // +1 for the [CLS] token // last pos is always kept 0, it's for CLS // dimension H for (int i = 0; i < num_patches; i++) { pos_data[i] = (i / n_patches_per_col) + 1; } set_input_i32("pos_h", pos_data); // dimension W for (int i = 0; i < num_patches; i++) { pos_data[i] = (i % n_patches_per_col) + 1; } set_input_i32("pos_w", pos_data); } break; case PROJECTOR_TYPE_DOTS3NOTE_A: { GGML_ASSERT(imgs.entries.size() == 1); const int n_pos = (imgs.entries.front().nx() + 7) / 8; // 3x stride-2 conv2d std::vector positions(n_pos); for (int i = 0; i < n_pos; i++) { positions[i] = i; } set_input_i32("positions", positions); } break; case PROJECTOR_TYPE_VIBEVOICE_ASR: { set_enc_state_in(); } break; case PROJECTOR_TYPE_GEMMA4A: { GGML_ASSERT(imgs.entries.size() == 1); const auto & img0 = imgs.entries.front(); // Compute n_pos matching SSCP output: two stride-2 convs int n_pos = img0.nx(); for (int i = 0; i < 2; i++) { n_pos = (n_pos - 1) / 2 + 1; } // Chunked local attention: blocked causal mask and RPE const int chunk_size = mtmd_graph_gemma4a::chunk_size; const int max_past = mtmd_graph_gemma4a::max_past; const int context_size = chunk_size + max_past; const int num_blocks = (n_pos + chunk_size - 1) / chunk_size; // cached keys of a stream are at positions -n_hist .. -1 const int n_hist = std::min(set_enc_state_in(), max_past); // Blocked causal attention mask: [context_size, chunk_size, num_blocks] { std::vector mask(context_size * chunk_size * num_blocks, -1e9f); for (int b = 0; b < num_blocks; b++) { for (int q = 0; q < chunk_size; q++) { int gq = b * chunk_size + q; for (int k = 0; k < context_size; k++) { int gk = b * chunk_size - max_past + k; if (gq < n_pos && gk >= -n_hist && gk < n_pos && gk <= gq && (gq - gk) < max_past) { mask[k + q * context_size + b * context_size * chunk_size] = 0.0f; } } } } set_input_f32("kq_mask", mask); } // Sinusoidal RPE: 13 positions [12, 11, ..., 0] { const int n_embd = model.hparams.n_embd; const int num_timescales = n_embd / 2; const float log_timescale_increment = logf(10000.0f) / std::max(num_timescales - 1, 1); const int rpe_len = max_past + 1; std::vector pos_emb(n_embd * rpe_len, 0.0f); for (int p = 0; p < rpe_len; p++) { float position = (float)(max_past - p); for (int i = 0; i < num_timescales; i++) { float inv_ts = expf(-(float)i * log_timescale_increment); float scaled = position * inv_ts; pos_emb[p * n_embd + i] = sinf(scaled); pos_emb[p * n_embd + i + num_timescales] = cosf(scaled); } } set_input_f32("pos_emb", pos_emb); } } break; case PROJECTOR_TYPE_MIMO_AUDIO: { GGML_ASSERT(imgs.entries.size() == 1); const int n_frames = imgs.entries.front().nx(); const int n_pos = (n_frames - 1) / 2 + 1; // matches conv1(s=1)+conv2(s=2) output length std::vector positions(n_pos); for (int i = 0; i < n_pos; i++) { positions[i] = i; } set_input_i32("mimo_audio_positions", positions); const int window = hparams.attn_window_size; GGML_ASSERT(window > 0); const float neg_inf = std::numeric_limits::lowest(); std::vector full_mask((size_t) n_pos * n_pos); std::vector window_mask((size_t) n_pos * n_pos); for (int q = 0; q < n_pos; q++) { for (int k = 0; k < n_pos; k++) { const bool causal_ok = k <= q; full_mask[(size_t) q * n_pos + k] = causal_ok ? 0.0f : neg_inf; window_mask[(size_t) q * n_pos + k] = (causal_ok && (q - k) <= window) ? 0.0f : neg_inf; } } set_input_attn_mask(full_mask, 0); set_input_attn_mask(window_mask, 1); // input_local_transformer: block-diagonal mask + in-group positions { const int n_pos_ds = (n_pos - 2) / 2 + 1; // matches downsample conv (k=2,s=2,p=0) const int group_size = hparams.audio_local_group_size; GGML_ASSERT(group_size > 0); const int n_groups = (n_pos_ds + group_size - 1) / group_size; const int n_padded = n_groups * group_size; std::vector local_positions(n_padded); for (int i = 0; i < n_padded; i++) { local_positions[i] = i % group_size; } set_input_i32("mimo_audio_local_positions", local_positions); std::vector local_mask((size_t) n_padded * n_padded); for (int q = 0; q < n_padded; q++) { for (int k = 0; k < n_padded; k++) { const bool same_group = (q / group_size) == (k / group_size); local_mask[(size_t) q * n_padded + k] = same_group ? 0.0f : neg_inf; } } set_input_attn_mask(local_mask, 2); } } break; case PROJECTOR_TYPE_LFM2A: case PROJECTOR_TYPE_D1OMNI_A: { GGML_ASSERT(imgs.entries.size() == 1); const auto n_frames = model.n_output_tokens(&imgs.entries.front()); auto d_model = 512; auto seq_len = n_frames * 2 - 1; std::vector pos_emb(d_model*seq_len); std::vector inv_freq(d_model / 2); for (size_t i = 0; i < inv_freq.size(); ++i) { inv_freq[i] = std::exp(-(std::log(10000.0) / (float)d_model) * (2.0f * (float)(i))); } for (int64_t pos = 0; pos < seq_len; ++pos) { for (size_t i = 0; i < inv_freq.size(); ++i) { const float ang = (n_frames - pos - 1) * inv_freq[i]; pos_emb[pos*d_model + 2*i + 0] = sinf(ang); // even pos_emb[pos*d_model + 2*i + 1] = cosf(ang); // odd } } set_input_f32("pos_emb", pos_emb); } break; case PROJECTOR_TYPE_LFM3A: { GGML_ASSERT(imgs.entries.size() == 1); const int n_pos = model.n_output_tokens(&imgs.entries.front()); const int base = set_enc_state_in(); set_lfm3a_window_inputs(base, n_pos, hparams.attn_window_size); } break; case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: { GGML_ASSERT(imgs.entries.size() == 1); std::vector positions(ggml_nelements(get_inp_tensor("positions"))); for (size_t i = 0; i < positions.size(); i++) { positions[i] = (int32_t) i; } set_input_i32("positions", positions); GGML_ASSERT(params->enc_state != nullptr); set_input_f32("inp_norm", params->enc_state->mel_norm); if (ggml_graph_get_tensor(gf, "spk_prev") != nullptr) { set_input_f32("spk_prev", params->enc_state->spk_prev); } } break; case PROJECTOR_TYPE_PARAKEET: { GGML_ASSERT(imgs.entries.size() == 1); struct ggml_tensor * attn_mask = ggml_graph_get_tensor(gf, "attn_mask"); const int n_q = attn_mask->ne[1]; const int n_k = attn_mask->ne[0]; const int n_frames = imgs.entries.front().nx(); const int n_tokens_real = (n_frames + hparams.subsampling_factor-1) / hparams.subsampling_factor; const float mask_value = -1e30f; std::vector mask_data(n_q * n_k); if (n_k == n_q) { // full attention: mask keys that are padding for (int q = 0; q < n_q; ++q) { for (int k = 0; k < n_k; ++k) { mask_data[q * n_k + k] = (k >= n_tokens_real) ? mask_value : 0.0f; } } } else { // local attention: mask keys outside the valid window const int att_left = n_k / 2; for (int q = 0; q < n_q; ++q) { for (int k = 0; k < n_k; ++k) { const int key = q - att_left + k; mask_data[q * n_k + k] = (key >= 0 && key < n_tokens_real) ? 0.0f : mask_value; } } } set_input_f32(attn_mask->name, mask_data); // local attention skew mask: zeroes out the probs that were // computed for keys outside the valid sliding window. if (struct ggml_tensor * local_mask = ggml_graph_get_tensor(gf, "local_mask")) { const int lm_k = local_mask->ne[0]; const int lm_q = local_mask->ne[1]; const int window_size = lm_k - lm_q + 1; std::vector lm_data(lm_q * lm_k); for (int q = 0; q < lm_q; ++q) { for (int k = 0; k < lm_k; ++k) { const int rel = k - q; lm_data[q * lm_k + k] = (rel >= 0 && rel < window_size) ? 1.0f : 0.0f; } } set_input_f32(local_mask->name, lm_data); } // Generate rotation frequencies for relative positional encoding. { const int n_state = hparams.n_embd; const int d_half = n_state / 2; const float log_10000 = logf(10000.0f); std::vector freqs(d_half); for (int k = 0; k < d_half; ++k) { freqs[k] = expf(-(float(k * 2) * log_10000 / float(n_state))); } set_input_f32("pos_freqs", freqs); } // Generate relative positional distance values which scaled by // the frequency to produce the angles for sin/cos. { // window_size is only known after graph construction since it depends on // n_time from the conv output, so we read it back from the graph tensor. struct ggml_tensor * rel_pos = ggml_graph_get_tensor(gf, "rel_positions"); const int window_size = rel_pos->ne[1]; std::vector pos(window_size); // local attention: window is fixed at [att_left, att_right] // full attention: window covers the full sequence, centered if (ggml_graph_get_tensor(gf, "local_mask")) { const int att_left = window_size / 2; for (int t = 0; t < window_size; ++t) { pos[t] = float(att_left - t); } } else { const int n_time = (window_size + 1) / 2; for (int t = 0; t < window_size; ++t) { pos[t] = float(n_time - 1 - t); } } set_input_f32(rel_pos->name, pos); } } break; case PROJECTOR_TYPE_GRANITE_SPEECH: { const int context_size = model.hparams.audio_chunk_size; const int max_pos_emb = model.hparams.audio_max_pos_emb; std::vector dists((size_t) context_size * (size_t) context_size); for (int i = 0; i < context_size; i++) { for (int j = 0; j < context_size; j++) { int d = i - j; if (d < -context_size) d = -context_size; if (d > context_size) d = context_size; dists[(size_t) i * (size_t) context_size + (size_t) j] = d + max_pos_emb; } } set_input_i32("attn_dists", dists); const int n_frames = image_size_width; const int remainder = n_frames % context_size; if (remainder > 0) { const int num_blocks = (n_frames + context_size - 1) / context_size; std::vector mask((size_t) context_size * (size_t) context_size * (size_t) num_blocks, 0.0f); const float neg_inf = -INFINITY; const size_t last_block_offset = (size_t) (num_blocks - 1) * (size_t) context_size * (size_t) context_size; for (int q = 0; q < context_size; q++) { for (int k = 0; k < context_size; k++) { if (q >= remainder || k >= remainder) { mask[last_block_offset + (size_t) q * (size_t) context_size + (size_t) k] = neg_inf; } } } set_input_f32("attn_mask", mask); } } break; case PROJECTOR_TYPE_GRANITE4_VISION: { // Granite Vision 4.1 uses precomputed permutation index // tensors to express the _win / _unwin / spatial sampling // reshapes as ggml_get_rows gathers. The names are set // by g4v_gather() in models/granite4-vision.cpp. const int patch_size = model.hparams.patch_size; const int image_side = imgs.entries.front().nx() / patch_size; const int window_side = hparams.downsample_window_side; const int query_side = hparams.downsample_query_side; const int n = image_side / window_side; const int new_side = n * query_side; // Builds the raster→window permutation indices for a // (side, side) grid split into (n × n) windows of (win × win) // tokens each. dst[w * win*win + p] = source raster index. auto make_win_idx = [](int side, int win) { const int nn = side / win; std::vector idx(static_cast(side) * side); for (int wy = 0; wy < nn; ++wy) { for (int wx = 0; wx < nn; ++wx) { for (int iy = 0; iy < win; ++iy) { for (int ix = 0; ix < win; ++ix) { const int w = wy * nn + wx; const int p = iy * win + ix; const int y = wy * win + iy; const int x = wx * win + ix; idx[static_cast(w) * (win*win) + p] = y * side + x; } } } } return idx; }; auto make_unwin_idx = [&](int side, int win) { const std::vector fwd = make_win_idx(side, win); std::vector inv(fwd.size()); for (size_t i = 0; i < fwd.size(); ++i) { inv[fwd[i]] = static_cast(i); } return inv; }; auto make_spatial_idx = [](int side, int offset) { const int off_y = (offset >> 1) & 1; const int off_x = offset & 1; const int new_s = side / 2; std::vector idx(static_cast(new_s) * new_s); for (int y = 0; y < new_s; ++y) { for (int x = 0; x < new_s; ++x) { idx[y * new_s + x] = (y * 2 + off_y) * side + (x * 2 + off_x); } } return idx; }; // the same permutation is applied to every tile of the stacked image auto upload = [&](const std::string & name, const std::vector & idx) { ggml_tensor * t = ggml_graph_get_tensor(gf, name.c_str()); GGML_ASSERT(t); GGML_ASSERT(ggml_nelements(t) % (int64_t) idx.size() == 0); const int n_rep = ggml_nelements(t) / idx.size(); std::vector buf; buf.reserve(idx.size() * n_rep); for (int i = 0; i < n_rep; ++i) { buf.insert(buf.end(), idx.begin(), idx.end()); } ggml_backend_tensor_set(t, buf.data(), 0, ggml_nbytes(t)); }; // Stage 1b only uses block 0's permutations; future stages // will upload all blocks. for (size_t bid = 0; bid < hparams.feature_layers.size(); ++bid) { const std::string prefix = "g4v_blk" + std::to_string(bid) + "_"; upload(prefix + "win_idx", make_win_idx(image_side, window_side)); upload(prefix + "qwin_idx", make_win_idx(new_side, query_side)); upload(prefix + "unwin_idx", make_unwin_idx(new_side, query_side)); const auto spatial_offset = hparams.proj_spatial_offsets[bid]; if (spatial_offset >= 0) { upload(prefix + "spatial_idx", make_spatial_idx(image_side,spatial_offset)); } } } break; default: GGML_ABORT("Unknown projector type"); } mtmd_backend_cpu_set_n_threads(backends.backend_cpu, params->n_threads); auto status = ggml_backend_sched_graph_compute(backends.sched.get(), gf); if (status != GGML_STATUS_SUCCESS) { LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status); return false; } if (model.proj_type == PROJECTOR_TYPE_KOKORO) { const auto * io = params->kokoro; auto read = [&](const char * name, std::vector * out) { ggml_tensor * t = ggml_graph_get_tensor(gf, name); GGML_ASSERT(t != nullptr && out != nullptr); out->resize(ggml_nelements(t)); ggml_backend_tensor_get(t, out->data(), 0, ggml_nbytes(t)); }; switch (io->stage) { case mtmd_graph_kokoro::STAGE_TEXT: read("out_text", io->out0); break; case mtmd_graph_kokoro::STAGE_F0N: read("out_f0", io->out0); read("out_n", io->out1); break; case mtmd_graph_kokoro::STAGE_DEC: read("out_spec", io->out0); break; } return true; } if (embeddings != nullptr) { // sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one) const int n_tokens_out = embeddings->ne[1]; const int expected_n_tokens_out = model.n_output_tokens(&imgs.entries[0]); if (n_tokens_out != expected_n_tokens_out) { LOG_ERR("%s: expected output %d tokens, got %d\n", __func__, expected_n_tokens_out, n_tokens_out); GGML_ABORT("Invalid number of output tokens"); } LOG_DBG("%s: output embedding shape [%d, %d, %d]\n", __func__, (int)embeddings->ne[0], (int)embeddings->ne[1], (int)embeddings->ne[2]); // copy output to user buffer if provided // if output is empty, skip the copy auto & out_batch_embd = *params->out_embd; if (!out_batch_embd.empty()) { if (out_batch_embd.size() != (size_t)ggml_nelements(embeddings)) { LOG_ERR("%s: output buffer has %zu elements but expected %zu\n", __func__, out_batch_embd.size(), (size_t)ggml_nelements(embeddings)); GGML_ABORT("Output buffer size mismatch"); } ggml_backend_tensor_get(embeddings, out_batch_embd.data(), 0, ggml_nbytes(embeddings)); } else { LOG_WRN("%s: output buffer is empty, skipping copy\n", __func__); } } // // for audio gen models // if (model.proj_type == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) { GGML_ASSERT(params->out_codes && params->out_code_embd); // frame-major codes, the layout of GEN_WAV's input std::vector> groups; for (int g = 0; ; g++) { ggml_tensor * t = ggml_graph_get_tensor(gf, ("out_ref_code_" + std::to_string(g)).c_str()); if (!t) { break; } groups.emplace_back((size_t) ggml_nelements(t)); ggml_backend_tensor_get(t, groups.back().data(), 0, ggml_nbytes(t)); } GGML_ASSERT(!groups.empty()); const size_t n_frames = groups[0].size(); params->out_codes->resize(n_frames * groups.size()); for (size_t f = 0; f < n_frames; f++) { for (size_t g = 0; g < groups.size(); g++) { (*params->out_codes)[f * groups.size() + g] = groups[g][f]; } } ggml_tensor * rows = ggml_graph_get_tensor(gf, "out_ref_rows"); GGML_ASSERT(rows && rows->type == GGML_TYPE_F32 && (size_t) rows->ne[1] == n_frames); params->out_code_embd->resize((size_t) ggml_nelements(rows)); ggml_backend_tensor_get(rows, params->out_code_embd->data(), 0, ggml_nbytes(rows)); return true; } if (model.proj_type == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) { ggml_tensor * fsq = ggml_graph_get_tensor(gf, "out_fsq"); GGML_ASSERT(fsq != nullptr && params->out_codes); const int n_tok = (int) fsq->ne[1]; std::vector h((size_t) 8 * n_tok); ggml_backend_tensor_get(fsq, h.data(), 0, ggml_nbytes(fsq)); // fsq round to base 3: h in (-1, 1) maps to digits {0, 1, 2} params->out_codes->resize(n_tok); for (int t = 0; t < n_tok; t++) { int32_t code = 0; for (int i = 7; i >= 0; i--) { const int32_t d = (int32_t) roundf(h[(size_t) t * 8 + i] * 0.9990000128746033f) + 1; code = code * 3 + d; } (*params->out_codes)[(size_t) t] = code; } // speech table rows of the codes, the multilingual variant adds its learned positions as its perceiver expects if (params->out_code_embd) { const auto & tensors = model.cbx_tensors; auto tab_it = tensors.find("a.gen.code.out_embd.weight"); GGML_ASSERT(tab_it != tensors.end()); ggml_tensor * tab = tab_it->second; const int n_e = (int) tab->ne[0]; auto pos_it = tensors.find("a.gen.t3.speech_pos_emb"); ggml_tensor * pos = pos_it == tensors.end() ? nullptr : pos_it->second; if ((tab->type != GGML_TYPE_F16 && tab->type != GGML_TYPE_F32) || (pos && (pos->type != GGML_TYPE_F32 || pos->ne[0] != n_e || pos->ne[1] < n_tok))) { LOG_ERR("%s: unexpected speech table or positions\n", __func__); return false; } params->out_code_embd->resize((size_t) n_tok * n_e); std::vector h16(n_e); std::vector pr(n_e); for (int t = 0; t < n_tok; t++) { float * dst = params->out_code_embd->data() + (size_t) t * n_e; if ((*params->out_codes)[(size_t) t] >= tab->ne[1]) { LOG_ERR("%s: speech code beyond the speech table\n", __func__); return false; } const size_t r = (size_t) (*params->out_codes)[(size_t) t] * n_e; if (tab->type == GGML_TYPE_F16) { ggml_backend_tensor_get(tab, h16.data(), r * sizeof(ggml_fp16_t), (size_t) n_e * sizeof(ggml_fp16_t)); for (int j = 0; j < n_e; j++) { dst[j] = ggml_fp16_to_fp32(h16[j]); } } else { ggml_backend_tensor_get(tab, dst, r * sizeof(float), (size_t) n_e * sizeof(float)); } if (pos) { ggml_backend_tensor_get(pos, pr.data(), (size_t) t * n_e * sizeof(float), (size_t) n_e * sizeof(float)); for (int j = 0; j < n_e; j++) { dst[j] += pr[j]; } } } } return true; } if (model.proj_type == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE) { ggml_tensor * sp = ggml_graph_get_tensor(gf, "out_spec"); GGML_ASSERT(sp != nullptr && params->out_spec); params->out_spec->resize(ggml_nelements(sp)); ggml_backend_tensor_get(sp, params->out_spec->data(), 0, ggml_nbytes(sp)); return true; } if (model.proj_type == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS) { ggml_tensor * mel = ggml_graph_get_tensor(gf, "out_mel"); GGML_ASSERT(mel != nullptr); std::vector mel_data(ggml_nelements(mel)); ggml_backend_tensor_get(mel, mel_data.data(), 0, ggml_nbytes(mel)); // hift: f0 -> harmonic source -> source stft on the host, then the vocoder graph and the istft ggml_tensor * f0_t = ggml_graph_get_tensor(gf, "out_f0"); GGML_ASSERT(f0_t != nullptr); const int n_mel_out = (int) ggml_nelements(f0_t); std::vector f0(n_mel_out); ggml_backend_tensor_get(f0_t, f0.data(), 0, ggml_nbytes(f0_t)); // SineGen: f0 upsampled x480, 9 harmonics with cumulative phase, uv gating and noise, merged by l_linear + tanh const int ups_total = 480; const double sr = 24000.0; const int64_t n_wav = (int64_t) n_mel_out * ups_total; std::vector lw(9); float lb = 0.0f; if (model.cbx_read_tensor("a.gen.hift.m_source.l_linear.weight", lw.data(), lw.size()) != 9 || model.cbx_read_tensor("a.gen.hift.m_source.l_linear.bias", &lb, 1) != 1) { LOG_ERR("%s: unexpected hift source module\n", __func__); return false; } std::mt19937 & srng = params->gen_state->rng; std::uniform_real_distribution ud(-M_PI, M_PI); std::normal_distribution snd(0.0f, 1.0f); double phase[9]; for (int h = 0; h < 9; h++) { phase[h] = h == 0 ? 0.0 : ud(srng); } std::vector src((size_t) n_wav); double cum[9] = {0.0}; for (int64_t t = 0; t < n_wav; t++) { const float f = f0[(size_t) (t / ups_total)]; const float uv = f > 10.0f ? 1.0f : 0.0f; // nsf_voiced_threshold const float namp = uv * 0.003f + (1.0f - uv) * 0.1f / 3.0f; float merged = lb; for (int h = 0; h < 9; h++) { cum[h] += (double) f * (h + 1) / sr; cum[h] -= floor(cum[h]); float sine = 0.1f * (float) sin(2.0 * M_PI * cum[h] + phase[h]); sine = sine * uv + namp * snd(srng); merged += lw[(size_t) h] * sine; } src[(size_t) t] = tanhf(merged); } // stft of the source: n_fft 16, hop 4, hann window, centered const int n_fft = 16, hop = 4, n_bins = 9; std::vector win(n_fft); for (int i = 0; i < n_fft; i++) win[(size_t) i] = 0.5f - 0.5f * cosf(2.0f * (float) M_PI * i / n_fft); const int n_stft = (int) (n_wav / hop) + 1; std::vector sstft((size_t) n_stft * 18); for (int fr = 0; fr < n_stft; fr++) { const int64_t c0 = (int64_t) fr * hop - n_fft / 2; // centered, reflect padded for (int k = 0; k < n_bins; k++) { double re = 0.0, im = 0.0; for (int i = 0; i < n_fft; i++) { int64_t idx = c0 + i; if (idx < 0) idx = -idx; if (idx >= n_wav) idx = 2 * (n_wav - 1) - idx; const double v = (double) src[(size_t) idx] * win[(size_t) i]; const double a = 2.0 * M_PI * k * i / n_fft; re += v * cos(a); im -= v * sin(a); } sstft[(size_t) fr * 18 + k ] = (float) re; sstft[(size_t) fr * 18 + 9 + k] = (float) im; } } // vocoder graph on mel + source stft std::vector spec; { mtmd_subctx_encode_params vp = *params; vp.gen_process = MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE; vp.mel_in = &mel_data; vp.sstft_in = &sstft; vp.vocode_n_mel = n_mel_out; vp.vocode_n_stft = n_stft; vp.out_audio = nullptr; vp.out_spec = &spec; if (!encode(&vp)) { LOG_ERR("%s: vocoder stage failed\n", __func__); return false; } } // istft: mag = clipped exp, phase = sin, hann overlap-add const int n_frames_out = (int) (spec.size() / 18); const int64_t n_out = (int64_t) (n_frames_out - 1) * hop; std::vector acc((size_t) n_out + n_fft, 0.0); std::vector wsum((size_t) n_out + n_fft, 0.0); for (int fr = 0; fr < n_frames_out; fr++) { double frame[16]; for (int i = 0; i < n_fft; i++) { double v = 0.0; for (int k = 0; k < n_bins; k++) { const double mag = fmin(exp((double) spec[(size_t) fr * 18 + k]), 1e2); const double ph = sin((double) spec[(size_t) fr * 18 + 9 + k]); const double re = mag * cos(ph), im = mag * sin(ph); const double a = 2.0 * M_PI * k * i / n_fft; const double w = (k == 0 || k == n_fft / 2) ? 1.0 : 2.0; v += w * (re * cos(a) - im * sin(a)); } frame[i] = v / n_fft; } const int64_t o = (int64_t) fr * hop; for (int i = 0; i < n_fft; i++) { acc [(size_t) (o + i)] += frame[i] * win[(size_t) i]; wsum[(size_t) (o + i)] += (double) win[(size_t) i] * win[(size_t) i]; } } GGML_ASSERT(params->out_audio); auto & out_audio = *params->out_audio; out_audio.resize((size_t) std::min(n_out - n_fft / 2, n_wav)); for (size_t i = 0; i < out_audio.size(); i++) { const size_t j = i + n_fft / 2; // drop the centering pad const double v = wsum[j] > 1e-11 ? acc[j] / wsum[j] : 0.0; out_audio[(size_t) i] = (float) fmax(-0.99, fmin(0.99, v)); } // trim_fade of the reference: silence 20 ms then fade in 20 ms, hides the artifact of the prompt boundary const size_t n_trim = 24000 / 50; for (size_t i = 0; i < 2 * n_trim && i < out_audio.size(); i++) { const double g = i < n_trim ? 0.0 : (cos(M_PI * (1.0 - (double) (i - n_trim) / n_trim)) + 1.0) / 2.0; out_audio[(size_t) i] *= (float) g; } return true; } // optional outputs: a pipeline yields codes or feats, and not all have an eos head if (params->out_codes != nullptr) { ggml_tensor * codes = ggml_graph_get_tensor(gf, "out_codes"); if (codes != nullptr) { auto & out_codes = *params->out_codes; out_codes.resize(ggml_nelements(codes)); ggml_backend_tensor_get(codes, out_codes.data(), 0, ggml_nbytes(codes)); } } if (params->out_feats != nullptr) { ggml_tensor * feats = ggml_graph_get_tensor(gf, "out_feats"); if (feats != nullptr) { auto & out_feats = *params->out_feats; out_feats.resize(ggml_nelements(feats)); ggml_backend_tensor_get(feats, out_feats.data(), 0, ggml_nbytes(feats)); } } if (params->out_is_eos != nullptr) { ggml_tensor * eos = ggml_graph_get_tensor(gf, "out_eos_score"); if (eos != nullptr) { GGML_ASSERT(ggml_nelements(eos) == 1); float score = 0.0f; ggml_backend_tensor_get(eos, &score, 0, sizeof(float)); *params->out_is_eos = score > hparams.gen_eos_threshold; } } if (params->out_audio != nullptr) { ggml_tensor * audio = ggml_graph_get_tensor(gf, "out_audio"); if (audio == nullptr) { GGML_ABORT("out_audio requested but graph has no \"out_audio\" tensor"); } auto & out_audio = *params->out_audio; out_audio.resize(ggml_nelements(audio)); ggml_backend_tensor_get(audio, out_audio.data(), 0, ggml_nbytes(audio)); if (model.proj_type == PROJECTOR_TYPE_SOPRANO) { std::vector spectrum = std::move(out_audio); mtmd_graph_soprano::decode_spectrum(model, spectrum, out_audio); } // drop the tail audio that comes from the code-0 rear padding, drama3, fish_s2, yue2 and minimax_music3 have no padding if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN && model.proj_type != PROJECTOR_TYPE_FISH_S2_GEN && model.proj_type != PROJECTOR_TYPE_YUE2_GEN && model.proj_type != PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN) { const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0; const int64_t n_frames_w = hparams.wav_tfm_swa; const int64_t n_frames = params->codes ? (int64_t) params->codes->size() / n_codes : n_frames_w; if (n_frames < n_frames_w) { const size_t hop = out_audio.size() / n_frames_w; out_audio.resize((size_t) n_frames * hop); } } } if (params->enc_state != nullptr && (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A || model.proj_type == PROJECTOR_TYPE_VIBEVOICE_ASR)) { const auto & slots = params->enc_state->slots; size_t i = 0; for (const auto & slot : list_enc_state_slots(hparams, model)) { ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str()); GGML_ASSERT(t != nullptr); copy_gen_state_out(t, slots[i++]); } } else if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) { mtmd_subctx_enc_state * st = params->enc_state; ggml_tensor * chunk = ggml_graph_get_tensor(gf, "spk_chunk"); ggml_tensor * preds = ggml_graph_get_tensor(gf, "spk_preds"); GGML_ASSERT(chunk != nullptr && preds != nullptr); st->spk_chunk.resize(ggml_nelements(chunk)); st->spk_preds.resize(ggml_nelements(preds)); ggml_backend_tensor_get(chunk, st->spk_chunk.data(), 0, ggml_nbytes(chunk)); ggml_backend_tensor_get(preds, st->spk_preds.data(), 0, ggml_nbytes(preds)); } else if (params->enc_state != nullptr) { mtmd_subctx_enc_state * st = params->enc_state; ggml_tensor * tail = ggml_graph_get_tensor(gf, "out_stem_tail"); GGML_ASSERT(tail != nullptr); st->stem_tail.resize(ggml_nelements(tail)); ggml_backend_tensor_get(tail, st->stem_tail.data(), 0, ggml_nbytes(tail)); const int n_new = (imgs.entries[0].nx() - 2) / 2; for (int i = 0; i < n_new; i++) { st->slot_pos[(st->n_past + i) % st->n_ctx] = st->n_past + i; } st->n_past += n_new; } if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { const auto & slots = params->gen_state->slots; size_t i = 0; for (const auto & slot : list_gen_state_slots(hparams, model)) { if (slot.in_place) { i++; continue; } ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str()); if (t == nullptr) { GGML_ABORT("graph has no \"state_out_%s\" tensor", slot.name.c_str()); } GGML_ASSERT(i < slots.size()); copy_gen_state_out(t, slots[i++]); } } // // Debug: dump final embeddings if MTMD_DEBUG_EMBEDDINGS is set // if (debug_output_embeddings && embeddings != nullptr) { const int64_t n_embd = embeddings->ne[0]; const int64_t n_tokens = embeddings->ne[1]; std::vector emb_data(ggml_nelements(embeddings)); ggml_backend_tensor_get(embeddings, emb_data.data(), 0, ggml_nbytes(embeddings)); LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n"); LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens); // TEMP debugging (parity validation), will be removed before merge // when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data] const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS"); if (dump_path && strcmp(dump_path, "1") != 0) { FILE * f = fopen(dump_path, "wb"); if (f) { const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd }; fwrite(hdr, sizeof(hdr), 1, f); fwrite(emb_data.data(), sizeof(float), emb_data.size(), f); fclose(f); } } // Print first few values of first token LOG_INF("Token 0 (first 16 values): "); for (int i = 0; i < std::min((int64_t)16, n_embd); i++) { LOG_INF("%.6f ", emb_data[i]); } LOG_INF("\n"); // Print last few values of first token if (n_embd > 16) { LOG_INF("Token 0 (last 16 values): "); for (int64_t i = n_embd - 16; i < n_embd; i++) { LOG_INF("%.6f ", emb_data[i]); } LOG_INF("\n"); } // Compute and print statistics float sum = 0.0f, sum_sq = 0.0f, min_val = emb_data[0], max_val = emb_data[0]; for (size_t i = 0; i < emb_data.size(); i++) { sum += emb_data[i]; sum_sq += emb_data[i] * emb_data[i]; min_val = std::min(min_val, emb_data[i]); max_val = std::max(max_val, emb_data[i]); } float mean = sum / emb_data.size(); float variance = (sum_sq / emb_data.size()) - (mean * mean); LOG_INF("Stats: mean=%.6f, std=%.6f, min=%.6f, max=%.6f, sum=%.6f\n", mean, sqrtf(variance), min_val, max_val, sum); LOG_INF("=== END MTMD_DEBUG_EMBEDDINGS ===\n\n"); } return true; }