#pragma once #include "../mtmd-graph.h" #include #include #include #include #include /* * IMPORTANT: The mtmd module does NOT accept pull requests that are fully or predominantly AI-generated. * We encourage human contributors to ensure the quality and reliability of the codebase. */ struct mtmd_graph_siglip : mtmd_graph { mtmd_graph_siglip(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_gemma4v : mtmd_graph { mtmd_graph_gemma4v(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override; bool support_batch() const override { return true; } }; struct mtmd_graph_gemma4uv : mtmd_graph { mtmd_graph_gemma4uv(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_nemotron_transcribe : mtmd_graph { const mtmd_subctx_enc_state * state; // nullptr during warmup mtmd_graph_nemotron_transcribe(mtmd_subcontext * ctx, const mtmd_image_f32 & img, const mtmd_subctx_enc_state * state) : mtmd_graph(ctx, img), state(state) {} ggml_cgraph * build() override; ggml_tensor * build_spk_encoder(ggml_tensor * cur, ggml_tensor * pos); // one recording: encodes its windows in turn, carrying the speaker state between them static bool encode_windows(mtmd_subcontext * ctx, const mtmd_model & model, mtmd_subctx_encode_params * params); // NeMo ParallelExpertEncoder online defaults, in output frames: 40 s windows with 4 s of context on each side static constexpr int window_core = 500; static constexpr int window_left = 50; static constexpr int window_right = 50; }; struct mtmd_graph_pixtral : mtmd_graph { mtmd_graph_pixtral(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_deepseek4v : mtmd_graph { mtmd_graph_deepseek4v(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_qwen2vl : mtmd_graph { mtmd_graph_qwen2vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_qwen3vl : mtmd_graph_qwen2vl { mtmd_graph_qwen3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_qwen2vl(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_magevl : mtmd_graph { mtmd_graph_magevl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_sensenova_u1 : mtmd_graph { mtmd_graph_sensenova_u1(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_lfm3vl : mtmd_graph { mtmd_graph_lfm3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_ling3vl : mtmd_graph_qwen3vl { mtmd_graph_ling3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_qwen3vl(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_minimax_m3 : mtmd_graph { mtmd_graph_minimax_m3(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * apply_rope(ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w); }; struct mtmd_graph_mimovl : mtmd_graph { mtmd_graph_mimovl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; // Force F32 mat-mul accumulation to avoid F16 overflow in the FFN down-proj // when the mmproj is stored in F16 (the source weights are BF16; downcasting // to F16 reduces dynamic range below the SwiGLU output magnitude on the last few layers). ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override; }; struct mtmd_graph_step3vl : mtmd_graph { mtmd_graph_step3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_youtuvl : mtmd_graph { mtmd_graph_youtuvl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_yasa2 : mtmd_graph { mtmd_graph_yasa2(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * layer_norm_channels(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b, float eps = 1e-6f); ggml_tensor * convnext_grn(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b); }; struct mtmd_graph_minicpmv : mtmd_graph { mtmd_graph_minicpmv(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_minicpmv4_6 : mtmd_graph { mtmd_graph_minicpmv4_6(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_internvl : mtmd_graph { mtmd_graph_internvl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; bool support_batch() const override { return true; } }; struct mtmd_graph_nemotron_v2_vl : mtmd_graph { mtmd_graph_nemotron_v2_vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_nemotron_omni : mtmd_graph { mtmd_graph_nemotron_omni(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * build_inp_with_temporal_merge(); }; struct mtmd_graph_llama4 : mtmd_graph { mtmd_graph_llama4(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_kimivl : mtmd_graph { mtmd_graph_kimivl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_paddleocr : mtmd_graph { mtmd_graph_paddleocr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_dotsocr : mtmd_graph { mtmd_graph_dotsocr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_dots3note_a : mtmd_graph { mtmd_graph_dots3note_a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_cogvlm : mtmd_graph { mtmd_graph_cogvlm(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_llava : mtmd_graph { mtmd_graph_llava(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_whisper_enc : mtmd_graph { mtmd_graph_whisper_enc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; // causal whisper-like encoder with a sliding window KV cache (voxtral realtime) struct mtmd_graph_voxtral_rt : mtmd_graph { mtmd_subctx_enc_state * state; // nullptr during warmup, the cache is then a graph input mtmd_graph_voxtral_rt(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_enc_state * state) : mtmd_graph(ctx, img), state(state) {} ggml_cgraph * build() override; }; struct mtmd_graph_deepseekocr : mtmd_graph { mtmd_graph_deepseekocr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * build_sam(ggml_tensor * inp); // build the SAM model bool support_batch() const override { return true; } }; struct mtmd_graph_deepseekocr2 : mtmd_graph_deepseekocr { mtmd_graph_deepseekocr2(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_deepseekocr(ctx, img) {} ggml_cgraph * build() override; // reuses build_sam() from base bool support_batch() const override { return true; } }; struct mtmd_graph_conformer : mtmd_graph { mtmd_graph_conformer(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; // builders shared by the chatterbox gen and spkenc graphs, defined in chatterbox-gen.cpp struct mtmd_graph_chatterbox_base : mtmd_graph { using mtmd_graph::mtmd_graph; protected: ggml_tensor * cbx_linear(ggml_tensor * w, ggml_tensor * b, ggml_tensor * x) const; ggml_tensor * cbx_conv1d(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, int stride, int pad_l, int pad_r) const; ggml_tensor * cbx_conv1d_dil(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, int pad, int dil) const; }; struct mtmd_graph_chatterbox_spkenc : mtmd_graph_chatterbox_base { mtmd_graph_chatterbox_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_chatterbox_base(ctx, img) {} ggml_cgraph * build() override; private: ggml_tensor * bn1d(const mtmd_chatterbox::bn & n, ggml_tensor * x); ggml_tensor * bn2d_relu(const mtmd_chatterbox::bn & n, ggml_tensor * x); ggml_tensor * res2d(const mtmd_chatterbox::spk_res2d & r, ggml_tensor * x, int stride); ggml_tensor * cam_layer(const mtmd_chatterbox::spk_cam_layer & l, ggml_tensor * x, int dil, ggml_tensor * segfix); }; struct mtmd_graph_chatterbox : mtmd_graph_chatterbox_base { mtmd_subctx_gen_process gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; int n_tokens = 0; int n_prompt_mel = 0; int vocode_n_mel = 0; int vocode_n_stft = 0; mtmd_graph_chatterbox(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int n_tokens, int n_prompt_mel, int vocode_n_mel, int vocode_n_stft) : mtmd_graph_chatterbox_base(ctx, img), gen_process(gen_process), n_tokens(n_tokens), n_prompt_mel(n_prompt_mel), vocode_n_mel(vocode_n_mel), vocode_n_stft(vocode_n_stft) {} ggml_cgraph * build() override; private: ggml_tensor * enc_layer(const mtmd_chatterbox::enc_layer & l, ggml_tensor * x, ggml_tensor * pos, int T); ggml_tensor * causal_block(const mtmd_chatterbox::causal_block & b, ggml_tensor * x); ggml_tensor * resnet(const mtmd_chatterbox::resnet & r, ggml_tensor * x, ggml_tensor * temb); ggml_tensor * tfm_block(const mtmd_chatterbox::tfm_block & b, ggml_tensor * x); ggml_tensor * estimator(ggml_tensor * x_noise, ggml_tensor * mu, ggml_tensor * spks, ggml_tensor * cond, ggml_tensor * temb, int T); ggml_tensor * hift_resblock(const mtmd_chatterbox::hift_res & r, ggml_tensor * x); ggml_cgraph * build_s3tok(int T); ggml_cgraph * build_vocoder(int n_mel, int n_stft); }; struct mtmd_graph_lfm3a : mtmd_graph { mtmd_graph_lfm3a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_granite_speech : mtmd_graph { mtmd_graph_granite_speech(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_vibevoice_asr : mtmd_graph { mtmd_graph_vibevoice_asr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; std::map state_in; mutable std::vector> state_out; ggml_tensor * with_left_ctx(ggml_tensor * x, const std::string & name) const; ggml_tensor * causal_conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & name) const; ggml_tensor * encode(int e, ggml_tensor * x) const; }; struct mtmd_graph_muse_spark_audio : mtmd_graph { mtmd_graph_muse_spark_audio(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_gemma4a : mtmd_graph { // attention_chunk_size, attention_context_left - 1 static constexpr int64_t chunk_size = 12; static constexpr int64_t max_past = 12; mtmd_graph_gemma4a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override; }; struct mtmd_graph_gemma4ua : mtmd_graph { mtmd_graph_gemma4ua(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_glm4v : mtmd_graph { mtmd_graph_glm4v(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_hunyuanvl : mtmd_graph { mtmd_graph_hunyuanvl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_mobilenetv5 : mtmd_graph { mtmd_graph_mobilenetv5(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * rms_norm_2d( ggml_tensor * inp, ggml_tensor * weight, float eps = 1e-6f); ggml_tensor* pad_same_2d( ggml_tensor* inp, int kernel_h, int kernel_w, int stride_h, int stride_w, int dilation_h = 1, int dilation_w = 1); ggml_tensor * build_edge_residual( ggml_tensor * inp, const mobilenetv5_block & block, int stride); ggml_tensor * build_inverted_residual( ggml_tensor * inp, const mobilenetv5_block & block, int stride); ggml_tensor * build_mobilenet_attn( ggml_tensor * inp, const mobilenetv5_block & block); }; struct mtmd_graph_qwen3a : mtmd_graph { mtmd_graph_qwen3a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_mimo_audio : mtmd_graph { mtmd_graph_mimo_audio(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; // zonos3 speaker encoder: MECT-B2 + ASTP, then the speaker projection of the backbone struct mtmd_graph_zonos3_spkenc : mtmd_graph { mtmd_graph_zonos3_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * W(const std::string & name) const; bool has(const std::string & name) const; ggml_tensor * chan(ggml_tensor * v) const; ggml_tensor * bn(ggml_tensor * x, const std::string & p) const; ggml_tensor * conv(ggml_tensor * x, const std::string & p, int s_t, int s_f, bool bias, int pad) const; ggml_tensor * grouped3x3(ggml_tensor * x, const std::string & p, int s_t, int s_f) const; ggml_tensor * res_block(ggml_tensor * x, const std::string & q, int s_f, int s_t) const; ggml_tensor * linear(ggml_tensor * x, const std::string & p) const; ggml_tensor * layer_norm(ggml_tensor * x, const std::string & p) const; ggml_tensor * flatten(ggml_tensor * x) const; ggml_tensor * unflatten(ggml_tensor * x, int64_t F, int64_t C) const; ggml_tensor * attention(ggml_tensor * h, const std::string & p) const; ggml_tensor * moe_ffn(ggml_tensor * h, const std::string & p) const; ggml_tensor * pos_conv(ggml_tensor * h, const std::string & p) const; }; struct mtmd_graph_qwen3tts_spkenc : mtmd_graph { mtmd_graph_qwen3tts_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const; ggml_tensor * res2net(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const; ggml_tensor * se_block(ggml_tensor * x, const mtmd_layer & layer) const; ggml_tensor * se_res2net_block(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const; ggml_tensor * attentive_stats_pool(ggml_tensor * x) const; }; struct mtmd_graph_qwen3tts_gen : mtmd_graph { mtmd_graph_qwen3tts_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int top_k, float top_p) : mtmd_graph(ctx, img), gen_process(gen_process), top_k(top_k), top_p(top_p) {} ggml_cgraph * build() override; // which sub-graph build() constructs, fixed at graph-build time mtmd_subctx_gen_process gen_process; // sampling params, fixed at graph-build time (GEN_CODE only) int top_k; float top_p; // // code_gen: backbone hidden state + sampled code0 -> 16 RVQ codes // MTP-style code predictor, one token per codebook // struct code_gen : mtmd_graph { code_gen(const mtmd_graph & parent, int top_k, float top_p) : mtmd_graph(parent), top_k(top_k), top_p(top_p) {} ggml_cgraph * build() override { GGML_ABORT("call prefill()/step() instead"); } int top_k; float top_p; ggml_tensor * cache_set(ggml_tensor * cache, int row_idx, ggml_tensor * value) const; ggml_tensor * do_sampling(ggml_tensor * logits, ggml_tensor * inp_rand) const; ggml_tensor * const_i32(ggml_tensor * anchor, float value) const; ggml_tensor * causal_mask_row(int64_t n_kv_pad, int pos) const; ggml_tensor * project_in(ggml_tensor * cur) const; ggml_tensor * layer_forward( ggml_tensor * cur, const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, ggml_tensor *& k_cache_layer, ggml_tensor *& v_cache_layer, int64_t n_kv_pad, int pos, int il) const; void prefill( std::vector & k_cache, std::vector & v_cache, ggml_tensor *& out_code_cache, ggml_tensor * h_state, ggml_tensor * code0_embd, ggml_tensor * inp_rand) const; ggml_tensor * step( std::vector & k_cache, std::vector & v_cache, ggml_tensor * out_code_cache, ggml_tensor * inp_rand, int step_idx) const; }; // // code2wav: RVQ codes -> raw PCM (quantizer + pre_conv + pre_transformer + upsample + DAC). // struct code2wav : mtmd_graph { code2wav(const mtmd_graph & parent) : mtmd_graph(parent) {} ggml_cgraph * build() override { GGML_ABORT("call decode() instead"); } // state_in: previous call's persisted state, by slot name (see list_c2w_state_slots()) std::map state_in; // state_out: this call's state to persist, added to the graph outputs by build() mutable std::vector> state_out; // stateful conv ops: read/update their state via state_in/state_out[state_name] ggml_tensor * causal_conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation, const std::string & state_name) const; ggml_tensor * causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name) const; ggml_tensor * causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const; ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const; ggml_tensor * quant_decode(ggml_tensor * inp_codes) const; ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il) const; ggml_tensor * convnext_block(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk, const std::string & state_prefix) const; ggml_tensor * dac_res_unit(ggml_tensor * x, const mtmd_code2wav::dac_res & res, int dilation, const std::string & state_name) const; // drama3: erf GELU in ConvNeXt, not the tanh approximation bool gelu_erf = false; // inp_codes [1, n_codes] I32 -> this frame's audio samples [n_samples] F32, clamped to [-1, 1] ggml_tensor * decode(ggml_tensor * inp_codes) const; }; }; // // pocket-tts: SEANet convolution stack, shared by the voice encoder and the mimi decoder. // stateless unless state_in is populated: convs then pad instead of carrying left-context. // struct mtmd_graph_pockettts_seanet : mtmd_graph { mtmd_graph_pockettts_seanet(const mtmd_graph & parent) : mtmd_graph(parent) {} ggml_cgraph * build() override { GGML_ABORT("call encode()/decode() instead"); } // per-call streaming state, keyed by slot name (see list_pockettts_state_slots) std::map state_in; mutable std::vector> state_out; ggml_tensor * conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation, bool pad_replicate = false, const std::string & state_name = "") const; ggml_tensor * conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name = "") const; ggml_tensor * res_unit(ggml_tensor * x, const mtmd_seanet::stage & stage, int dilation, const std::string & state_prefix = "") const; // x: [T, C] -> [T / hop, dim] ggml_tensor * encode(ggml_tensor * x) const; // x: [T, dim] -> [T * hop, 1], streams when state_in is populated ggml_tensor * decode(ggml_tensor * x) const; }; // mimi encoder + speaker_proj: reference waveform -> voice conditioning rows struct mtmd_graph_pockettts_spkenc : mtmd_graph { mtmd_graph_pockettts_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const; }; // // pocket-tts generation: // GEN_CODE = flow-matching decoder + end-of-speech head, one latent per call // GEN_WAV = mimi decoder, a window of latents -> PCM // struct mtmd_graph_pockettts_gen : mtmd_graph { mtmd_graph_pockettts_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int n_step, int n_frames) : mtmd_graph(ctx, img), gen_process(gen_process), n_step(n_step), n_frames(n_frames) {} ggml_cgraph * build() override; mtmd_subctx_gen_process gen_process; int n_step; // lsd_decode steps, fixed at graph-build time int n_frames; // GEN_WAV only: number of latents to decode // AdaLN modulation: x * (1 + scale) + shift ggml_tensor * modulate(ggml_tensor * x, ggml_tensor * shift, ggml_tensor * scale) const; ggml_tensor * time_embed(const mtmd_flow_net::time_embd & te, float t) const; ggml_tensor * flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const; }; // one persisted state buffer used by code2wav, see qwen3tts-gen.cpp struct c2w_state_slot { std::string name; int64_t ne0; int64_t ne1; bool in_place = false; // graph writes the slot directly, no state_in/state_out copy }; std::vector list_c2w_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); // drama3: tower KV slots, then list_c2w_state_slots() std::vector list_drama3_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); // same, for the streaming mimi decoder (pocket-tts GEN_WAV) std::vector list_pockettts_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); // // drama3 generation: // GEN_CODE = one codebook tower row, each layer attends over the whole sequence (KV in gen state) // GEN_WAV = Fish DAC decoder, frames -> PCM // struct mtmd_graph_drama3_gen : mtmd_graph { mtmd_graph_drama3_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, mtmd_subctx_gen_state * gen_state, int n_frames, int top_k, float top_p, float temp) : mtmd_graph(ctx, img), gen_process(gen_process), gen_state(gen_state), n_frames(n_frames), top_k(top_k), top_p(top_p), temp(temp) {} ggml_cgraph * build() override; // tower KV rows, shared by the prompt and the generated frames static constexpr int n_pos_max = 4096; mtmd_subctx_gen_process gen_process; mtmd_subctx_gen_state * gen_state; // nullptr on buffer reserve int n_frames; // GEN_WAV only int top_k; float top_p; float temp; ggml_cgraph * build_tower(); ggml_cgraph * build_codec(); ggml_tensor * tower_layer(ggml_tensor * x, int il, ggml_tensor * code, ggml_tensor * mask, ggml_tensor * extra, ggml_tensor * inp_pos, ggml_tensor * k_slot, ggml_tensor * v_slot, int pos); }; // // zonos3 generation: // GEN_CODE = 10 codebook heads, or the backbone row of a sampled frame // GEN_WAV = ZAC codec decoder up to the iSTFT spectra // struct mtmd_graph_zonos3_gen : mtmd_graph { mtmd_graph_zonos3_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, bool all_known, int n_frames) : mtmd_graph(ctx, img), gen_process(gen_process), all_known(all_known), n_frames(n_frames) {} ggml_cgraph * build() override; static constexpr int n_codebooks = 10; static constexpr int n_cb_vocab = 4096; static constexpr float softcap = 15.0f; // ZAC decoder, fixed by the codec config static constexpr int n_stages = 5; static constexpr int stage_ctx[n_stages] = { 150, 300, 480, 480, 480 }; // attention window static constexpr int head_dim = 64; static constexpr float ln_eps = 1e-5f; static constexpr float convnext_eps = 1e-6f; static constexpr int hop = 184; // at 44160 Hz static constexpr int n_fft[2] = { 736, 2208 }; static constexpr int rs_orig = 736; // 44160 -> 44100 Hz static constexpr int rs_new = 735; static constexpr int samples_per_frame = 2940; mtmd_subctx_gen_process gen_process; bool all_known; // GEN_CODE: only the backbone row int n_frames; // GEN_WAV std::map state_in; std::vector> state_out; ggml_cgraph * build_gen_code(); ggml_cgraph * build_gen_wav(); ggml_tensor * convnext(ggml_tensor * x, const mtmd_zonos3_wav::convnext & b, const std::string & state_name); ggml_tensor * moss_layer(ggml_tensor * x, const mtmd_zonos3_wav::layer & l, int s, int il, ggml_tensor * base); }; std::vector list_zonos3_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); struct mtmd_graph_soprano : mtmd_graph { mtmd_graph_soprano(mtmd_subcontext * ctx, const mtmd_image_f32 & img, int n_frames) : mtmd_graph(ctx, img), n_frames(n_frames) {} ggml_cgraph * build() override; static void decode_spectrum(const mtmd_model & model, const std::vector & spectrum, std::vector & pcm); // SopranoDecoder default static constexpr int upscale = 4; int n_frames; }; // // LFM3-Audio generation: // GEN_CODE = one MaskGIT pass of the depthformer, or only the feedback embedding once all codes are known // GEN_WAV = liquid-codec decoder, a window of frames -> polar STFT frames // struct mtmd_graph_lfm3a_gen : mtmd_graph { mtmd_graph_lfm3a_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, bool all_known, int n_frames) : mtmd_graph(ctx, img), gen_process(gen_process), all_known(all_known), n_frames(n_frames) {} ggml_cgraph * build() override; mtmd_subctx_gen_process gen_process; bool all_known; // GEN_CODE only int n_frames; // GEN_WAV only ggml_tensor * build_gen_code(); ggml_tensor * build_gen_wav(); ggml_tensor * build_lfm_layer(ggml_tensor * inpL, const mtmd_layer & layer, float eps, int il, const std::function & op); ggml_tensor * build_lfm_attn(ggml_tensor * cur, const mtmd_layer & layer, int n_head, int n_head_kv, float eps, float rope_theta, int il, ggml_tensor * inp_pos, ggml_tensor * kq_mask, ggml_tensor * k_cache, ggml_tensor * v_cache, ggml_tensor ** k_all, ggml_tensor ** v_all); ggml_tensor * build_lfm_conv(ggml_tensor * cur, const mtmd_layer & layer, int il, ggml_tensor * conv_state, ggml_tensor ** conv_state_out); }; // state slots of the streaming codec decoder (lfm3a_gen GEN_WAV) std::vector list_lfm3a_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); // state slots of the streaming LFM3-Audio encoder (lfm3a) std::vector list_lfm3a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); // state slots of the streaming Gemma 4 audio encoder (gemma4a) std::vector list_gemma4a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); // vibevoice-asr: the left context of every causal conv, a zero state is the causal zero padding std::vector list_vibevoice_asr_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); struct mtmd_graph_kimik25 : mtmd_graph { mtmd_graph_kimik25(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode); }; struct mtmd_graph_parakeet : mtmd_graph { mtmd_graph_parakeet(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_exaone4_5 : mtmd_graph { mtmd_graph_exaone4_5(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_granite4_vision : mtmd_graph { mtmd_graph_granite4_vision(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img), anyres(img.anyres), n_tiles(img.ny() / img.nx()), tile_side(img.nx() / patch_size) {} ggml_cgraph * build() override; private: // the input image is a stack of tiles on the Y axis: [overview, tile(0,0), tile(0,1), ...] const mtmd_image_f32::anyres_info anyres; const int n_tiles; const int tile_side; // patches per tile side ggml_tensor * build_tile_inp(); ggml_tensor * gather(ggml_tensor * src, const std::string & name, int idx_len); ggml_tensor * interp_down(ggml_tensor * src, int side, int new_side); ggml_tensor * build_block(const qf_block & blk, ggml_tensor * h, int bid, int spatial_offset, int image_side, int window_side, int query_side, float qformer_eps); ggml_tensor * build_newline_row(ggml_context * ctx0); ggml_tensor * build_anyres_assembly(ggml_tensor * cur, int out_side); }; struct mtmd_graph_muse_glimmer : mtmd_graph { mtmd_graph_muse_glimmer(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; struct mtmd_graph_inkling2v : mtmd_graph { using mtmd_graph::mtmd_graph; ggml_cgraph * build() override; };