Files
llama.cpp/tools/mtmd/models/models.h
T
Xuan-Son NguyenandPascal 92192d31e4 mtmd: support YuE2-3B (#32369)
* mtmd: support YuE2-3B

* improve speed

* add CFG

* fix streaming

* add cfg control from api

* check ctx size

* fix n_predict

* add flow_steps config

* nits

* fix batch api

* nits

* adapt

* mtmd: yue2 samples from its own saved logits

The context logits belong to the last decode of any slot, so with
several server slots the score and the unguided codes were sampled
from another sequence. Keep the logits of each own decode in
save_logits(), which also does the guidance mix.

* mtmd: let the yue2 snake chain fuse

The beta reshape landed between sqr and mul, so the mul, sin, sqr,
mul, add chain was not consecutive and the backends never fused it.
Expand both reshapes first, like qwen3tts does.

---------

Co-authored-by: Pascal <admin@serveurperso.com>
2026-11-16 20:22:23 +01:00

767 lines
34 KiB
C++

#pragma once
#include "../mtmd-graph.h"
#include <functional>
#include <map>
#include <string>
#include <utility>
#include <vector>
/*
* IMPORTANT: The mtmd module does NOT accept pull requests that are fully or predominantly AI-generated.
* We encourage human contributors to ensure the quality and reliability of the codebase.
*/
struct mtmd_graph_siglip : mtmd_graph {
mtmd_graph_siglip(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_gemma4v : mtmd_graph {
mtmd_graph_gemma4v(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override;
bool support_batch() const override { return true; }
};
struct mtmd_graph_gemma4uv : mtmd_graph {
mtmd_graph_gemma4uv(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_nemotron_transcribe : mtmd_graph {
const mtmd_subctx_enc_state * state; // nullptr during warmup
mtmd_graph_nemotron_transcribe(mtmd_subcontext * ctx, const mtmd_image_f32 & img, const mtmd_subctx_enc_state * state)
: mtmd_graph(ctx, img), state(state) {}
ggml_cgraph * build() override;
ggml_tensor * build_spk_encoder(ggml_tensor * cur, ggml_tensor * pos);
// one recording: encodes its windows in turn, carrying the speaker state between them
static bool encode_windows(mtmd_subcontext * ctx, const mtmd_model & model, mtmd_subctx_encode_params * params);
// NeMo ParallelExpertEncoder online defaults, in output frames: 40 s windows with 4 s of context on each side
static constexpr int window_core = 500;
static constexpr int window_left = 50;
static constexpr int window_right = 50;
};
struct mtmd_graph_pixtral : mtmd_graph {
mtmd_graph_pixtral(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_deepseek4v : mtmd_graph {
mtmd_graph_deepseek4v(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_qwen2vl : mtmd_graph {
mtmd_graph_qwen2vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_qwen3vl : mtmd_graph_qwen2vl {
mtmd_graph_qwen3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_qwen2vl(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_magevl : mtmd_graph {
mtmd_graph_magevl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_sensenova_u1 : mtmd_graph {
mtmd_graph_sensenova_u1(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_lfm3vl : mtmd_graph {
mtmd_graph_lfm3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_ling3vl : mtmd_graph_qwen3vl {
mtmd_graph_ling3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_qwen3vl(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_minimax_m3 : mtmd_graph {
mtmd_graph_minimax_m3(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * apply_rope(ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w);
};
struct mtmd_graph_mimovl : mtmd_graph {
mtmd_graph_mimovl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
// Force F32 mat-mul accumulation to avoid F16 overflow in the FFN down-proj
// when the mmproj is stored in F16 (the source weights are BF16; downcasting
// to F16 reduces dynamic range below the SwiGLU output magnitude on the last few layers).
ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override;
};
struct mtmd_graph_step3vl : mtmd_graph {
mtmd_graph_step3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_youtuvl : mtmd_graph {
mtmd_graph_youtuvl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_yasa2 : mtmd_graph {
mtmd_graph_yasa2(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * layer_norm_channels(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b, float eps = 1e-6f);
ggml_tensor * convnext_grn(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b);
};
struct mtmd_graph_minicpmv : mtmd_graph {
mtmd_graph_minicpmv(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_minicpmv4_6 : mtmd_graph {
mtmd_graph_minicpmv4_6(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_internvl : mtmd_graph {
mtmd_graph_internvl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
bool support_batch() const override { return true; }
};
struct mtmd_graph_nemotron_v2_vl : mtmd_graph {
mtmd_graph_nemotron_v2_vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_nemotron_omni : mtmd_graph {
mtmd_graph_nemotron_omni(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * build_inp_with_temporal_merge();
};
struct mtmd_graph_llama4 : mtmd_graph {
mtmd_graph_llama4(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_kimivl : mtmd_graph {
mtmd_graph_kimivl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_paddleocr : mtmd_graph {
mtmd_graph_paddleocr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_dotsocr : mtmd_graph {
mtmd_graph_dotsocr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_dots3note_a : mtmd_graph {
mtmd_graph_dots3note_a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_cogvlm : mtmd_graph {
mtmd_graph_cogvlm(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_llava : mtmd_graph {
mtmd_graph_llava(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_whisper_enc : mtmd_graph {
mtmd_graph_whisper_enc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
// causal whisper-like encoder with a sliding window KV cache (voxtral realtime)
struct mtmd_graph_voxtral_rt : mtmd_graph {
mtmd_subctx_enc_state * state; // nullptr during warmup, the cache is then a graph input
mtmd_graph_voxtral_rt(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_enc_state * state) : mtmd_graph(ctx, img), state(state) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_deepseekocr : mtmd_graph {
mtmd_graph_deepseekocr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * build_sam(ggml_tensor * inp); // build the SAM model
bool support_batch() const override { return true; }
};
struct mtmd_graph_deepseekocr2 : mtmd_graph_deepseekocr {
mtmd_graph_deepseekocr2(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_deepseekocr(ctx, img) {}
ggml_cgraph * build() override; // reuses build_sam() from base
bool support_batch() const override { return true; }
};
struct mtmd_graph_conformer : mtmd_graph {
mtmd_graph_conformer(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
// builders shared by the chatterbox gen and spkenc graphs, defined in chatterbox-gen.cpp
struct mtmd_graph_chatterbox_base : mtmd_graph {
using mtmd_graph::mtmd_graph;
protected:
ggml_tensor * cbx_linear(ggml_tensor * w, ggml_tensor * b, ggml_tensor * x) const;
ggml_tensor * cbx_conv1d(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x,
int stride, int pad_l, int pad_r) const;
ggml_tensor * cbx_conv1d_dil(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x,
int pad, int dil) const;
};
struct mtmd_graph_chatterbox_spkenc : mtmd_graph_chatterbox_base {
mtmd_graph_chatterbox_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_chatterbox_base(ctx, img) {}
ggml_cgraph * build() override;
private:
ggml_tensor * bn1d(const mtmd_chatterbox::bn & n, ggml_tensor * x);
ggml_tensor * bn2d_relu(const mtmd_chatterbox::bn & n, ggml_tensor * x);
ggml_tensor * res2d(const mtmd_chatterbox::spk_res2d & r, ggml_tensor * x, int stride);
ggml_tensor * cam_layer(const mtmd_chatterbox::spk_cam_layer & l, ggml_tensor * x, int dil, ggml_tensor * segfix);
};
struct mtmd_graph_chatterbox : mtmd_graph_chatterbox_base {
mtmd_subctx_gen_process gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
int n_tokens = 0;
int n_prompt_mel = 0;
int vocode_n_mel = 0;
int vocode_n_stft = 0;
mtmd_graph_chatterbox(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int n_tokens,
int n_prompt_mel, int vocode_n_mel, int vocode_n_stft)
: mtmd_graph_chatterbox_base(ctx, img), gen_process(gen_process), n_tokens(n_tokens), n_prompt_mel(n_prompt_mel),
vocode_n_mel(vocode_n_mel), vocode_n_stft(vocode_n_stft) {}
ggml_cgraph * build() override;
private:
ggml_tensor * enc_layer(const mtmd_chatterbox::enc_layer & l, ggml_tensor * x, ggml_tensor * pos, int T);
ggml_tensor * causal_block(const mtmd_chatterbox::causal_block & b, ggml_tensor * x);
ggml_tensor * resnet(const mtmd_chatterbox::resnet & r, ggml_tensor * x, ggml_tensor * temb);
ggml_tensor * tfm_block(const mtmd_chatterbox::tfm_block & b, ggml_tensor * x);
ggml_tensor * estimator(ggml_tensor * x_noise, ggml_tensor * mu, ggml_tensor * spks,
ggml_tensor * cond, ggml_tensor * temb, int T);
ggml_tensor * hift_resblock(const mtmd_chatterbox::hift_res & r, ggml_tensor * x);
ggml_cgraph * build_s3tok(int T);
ggml_cgraph * build_vocoder(int n_mel, int n_stft);
};
struct mtmd_graph_lfm3a : mtmd_graph {
mtmd_graph_lfm3a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_granite_speech : mtmd_graph {
mtmd_graph_granite_speech(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_vibevoice_asr : mtmd_graph {
mtmd_graph_vibevoice_asr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
std::map<std::string, ggml_tensor *> state_in;
mutable std::vector<std::pair<std::string, ggml_tensor *>> state_out;
ggml_tensor * with_left_ctx(ggml_tensor * x, const std::string & name) const;
ggml_tensor * causal_conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & name) const;
ggml_tensor * encode(int e, ggml_tensor * x) const;
};
struct mtmd_graph_muse_spark_audio : mtmd_graph {
mtmd_graph_muse_spark_audio(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_gemma4a : mtmd_graph {
// attention_chunk_size, attention_context_left - 1
static constexpr int64_t chunk_size = 12;
static constexpr int64_t max_past = 12;
mtmd_graph_gemma4a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override;
};
struct mtmd_graph_gemma4ua : mtmd_graph {
mtmd_graph_gemma4ua(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_glm4v : mtmd_graph {
mtmd_graph_glm4v(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_hunyuanvl : mtmd_graph {
mtmd_graph_hunyuanvl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_mobilenetv5 : mtmd_graph {
mtmd_graph_mobilenetv5(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * rms_norm_2d(
ggml_tensor * inp,
ggml_tensor * weight,
float eps = 1e-6f);
ggml_tensor* pad_same_2d(
ggml_tensor* inp,
int kernel_h,
int kernel_w,
int stride_h,
int stride_w,
int dilation_h = 1,
int dilation_w = 1);
ggml_tensor * build_edge_residual(
ggml_tensor * inp,
const mobilenetv5_block & block,
int stride);
ggml_tensor * build_inverted_residual(
ggml_tensor * inp,
const mobilenetv5_block & block,
int stride);
ggml_tensor * build_mobilenet_attn(
ggml_tensor * inp,
const mobilenetv5_block & block);
};
struct mtmd_graph_qwen3a : mtmd_graph {
mtmd_graph_qwen3a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_mimo_audio : mtmd_graph {
mtmd_graph_mimo_audio(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
// zonos3 speaker encoder: MECT-B2 + ASTP, then the speaker projection of the backbone
struct mtmd_graph_zonos3_spkenc : mtmd_graph {
mtmd_graph_zonos3_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * W(const std::string & name) const;
bool has(const std::string & name) const;
ggml_tensor * chan(ggml_tensor * v) const;
ggml_tensor * bn(ggml_tensor * x, const std::string & p) const;
ggml_tensor * conv(ggml_tensor * x, const std::string & p, int s_t, int s_f, bool bias, int pad) const;
ggml_tensor * grouped3x3(ggml_tensor * x, const std::string & p, int s_t, int s_f) const;
ggml_tensor * res_block(ggml_tensor * x, const std::string & q, int s_f, int s_t) const;
ggml_tensor * linear(ggml_tensor * x, const std::string & p) const;
ggml_tensor * layer_norm(ggml_tensor * x, const std::string & p) const;
ggml_tensor * flatten(ggml_tensor * x) const;
ggml_tensor * unflatten(ggml_tensor * x, int64_t F, int64_t C) const;
ggml_tensor * attention(ggml_tensor * h, const std::string & p) const;
ggml_tensor * moe_ffn(ggml_tensor * h, const std::string & p) const;
ggml_tensor * pos_conv(ggml_tensor * h, const std::string & p) const;
};
struct mtmd_graph_qwen3tts_spkenc : mtmd_graph {
mtmd_graph_qwen3tts_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const;
ggml_tensor * res2net(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const;
ggml_tensor * se_block(ggml_tensor * x, const mtmd_layer & layer) const;
ggml_tensor * se_res2net_block(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const;
ggml_tensor * attentive_stats_pool(ggml_tensor * x) const;
};
struct mtmd_graph_qwen3tts_gen : mtmd_graph {
mtmd_graph_qwen3tts_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int top_k, float top_p)
: mtmd_graph(ctx, img), gen_process(gen_process), top_k(top_k), top_p(top_p) {}
ggml_cgraph * build() override;
// which sub-graph build() constructs, fixed at graph-build time
mtmd_subctx_gen_process gen_process;
// sampling params, fixed at graph-build time (GEN_CODE only)
int top_k;
float top_p;
//
// code_gen: backbone hidden state + sampled code0 -> 16 RVQ codes
// MTP-style code predictor, one token per codebook
//
struct code_gen : mtmd_graph {
code_gen(const mtmd_graph & parent, int top_k, float top_p)
: mtmd_graph(parent), top_k(top_k), top_p(top_p) {}
ggml_cgraph * build() override { GGML_ABORT("call prefill()/step() instead"); }
int top_k;
float top_p;
ggml_tensor * cache_set(ggml_tensor * cache, int row_idx, ggml_tensor * value) const;
ggml_tensor * do_sampling(ggml_tensor * logits, ggml_tensor * inp_rand) const;
ggml_tensor * const_i32(ggml_tensor * anchor, float value) const;
ggml_tensor * causal_mask_row(int64_t n_kv_pad, int pos) const;
ggml_tensor * project_in(ggml_tensor * cur) const;
ggml_tensor * layer_forward(
ggml_tensor * cur,
const mtmd_layer & layer,
ggml_tensor * inp_pos,
ggml_tensor * kq_mask,
ggml_tensor *& k_cache_layer,
ggml_tensor *& v_cache_layer,
int64_t n_kv_pad,
int pos,
int il) const;
void prefill(
std::vector<ggml_tensor *> & k_cache,
std::vector<ggml_tensor *> & v_cache,
ggml_tensor *& out_code_cache,
ggml_tensor * h_state,
ggml_tensor * code0_embd,
ggml_tensor * inp_rand) const;
ggml_tensor * step(
std::vector<ggml_tensor *> & k_cache,
std::vector<ggml_tensor *> & v_cache,
ggml_tensor * out_code_cache,
ggml_tensor * inp_rand,
int step_idx) const;
};
//
// code2wav: RVQ codes -> raw PCM (quantizer + pre_conv + pre_transformer + upsample + DAC).
//
struct code2wav : mtmd_graph {
code2wav(const mtmd_graph & parent) : mtmd_graph(parent) {}
ggml_cgraph * build() override { GGML_ABORT("call decode() instead"); }
// state_in: previous call's persisted state, by slot name (see list_c2w_state_slots())
std::map<std::string, ggml_tensor *> state_in;
// state_out: this call's state to persist, added to the graph outputs by build()
mutable std::vector<std::pair<std::string, ggml_tensor *>> state_out;
// stateful conv ops: read/update their state via state_in/state_out[state_name]
ggml_tensor * causal_conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation, const std::string & state_name) const;
ggml_tensor * causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name) const;
ggml_tensor * causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const;
ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const;
ggml_tensor * quant_decode(ggml_tensor * inp_codes) const;
// window <= 0 uses hparams.wav_tfm_swa
ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il, int64_t window = 0) const;
ggml_tensor * convnext_block(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk, const std::string & state_prefix) const;
ggml_tensor * dac_res_unit(ggml_tensor * x, const mtmd_code2wav::dac_res & res, int dilation, const std::string & state_name) const;
// drama3: erf GELU in ConvNeXt, not the tanh approximation
bool gelu_erf = false;
// inp_codes [1, n_codes] I32 -> this frame's audio samples [n_samples] F32, clamped to [-1, 1]
ggml_tensor * decode(ggml_tensor * inp_codes) const;
};
};
//
// pocket-tts: SEANet convolution stack, shared by the voice encoder and the mimi decoder.
// stateless unless state_in is populated: convs then pad instead of carrying left-context.
//
struct mtmd_graph_pockettts_seanet : mtmd_graph {
mtmd_graph_pockettts_seanet(const mtmd_graph & parent) : mtmd_graph(parent) {}
ggml_cgraph * build() override { GGML_ABORT("call encode()/decode() instead"); }
// per-call streaming state, keyed by slot name (see list_pockettts_state_slots)
std::map<std::string, ggml_tensor *> state_in;
mutable std::vector<std::pair<std::string, ggml_tensor *>> state_out;
ggml_tensor * conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation,
bool pad_replicate = false, const std::string & state_name = "") const;
ggml_tensor * conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride,
const std::string & state_name = "") const;
ggml_tensor * res_unit(ggml_tensor * x, const mtmd_seanet::stage & stage, int dilation,
const std::string & state_prefix = "") const;
// x: [T, C] -> [T / hop, dim]
ggml_tensor * encode(ggml_tensor * x) const;
// x: [T, dim] -> [T * hop, 1], streams when state_in is populated
ggml_tensor * decode(ggml_tensor * x) const;
};
// mimi encoder + speaker_proj: reference waveform -> voice conditioning rows
struct mtmd_graph_pockettts_spkenc : mtmd_graph {
mtmd_graph_pockettts_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const;
};
//
// pocket-tts generation:
// GEN_CODE = flow-matching decoder + end-of-speech head, one latent per call
// GEN_WAV = mimi decoder, a window of latents -> PCM
//
struct mtmd_graph_pockettts_gen : mtmd_graph {
mtmd_graph_pockettts_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int n_step, int n_frames)
: mtmd_graph(ctx, img), gen_process(gen_process), n_step(n_step), n_frames(n_frames) {}
ggml_cgraph * build() override;
mtmd_subctx_gen_process gen_process;
int n_step; // lsd_decode steps, fixed at graph-build time
int n_frames; // GEN_WAV only: number of latents to decode
// AdaLN modulation: x * (1 + scale) + shift
ggml_tensor * modulate(ggml_tensor * x, ggml_tensor * shift, ggml_tensor * scale) const;
ggml_tensor * time_embed(const mtmd_flow_net::time_embd & te, float t) const;
ggml_tensor * flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const;
};
// one persisted state buffer used by code2wav, see qwen3tts-gen.cpp
struct c2w_state_slot {
std::string name;
int64_t ne0;
int64_t ne1;
bool in_place = false; // graph writes the slot directly, no state_in/state_out copy
};
std::vector<c2w_state_slot> list_c2w_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
// drama3: tower KV slots, then list_c2w_state_slots()
std::vector<c2w_state_slot> list_drama3_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
// same, for the streaming mimi decoder (pocket-tts GEN_WAV)
std::vector<c2w_state_slot> list_pockettts_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
//
// drama3 generation:
// GEN_CODE = one codebook tower row, each layer attends over the whole sequence (KV in gen state)
// GEN_WAV = Fish DAC decoder, frames -> PCM
//
struct mtmd_graph_drama3_gen : mtmd_graph {
mtmd_graph_drama3_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, mtmd_subctx_gen_state * gen_state,
int n_frames, int top_k, float top_p, float temp)
: mtmd_graph(ctx, img), gen_process(gen_process), gen_state(gen_state), n_frames(n_frames), top_k(top_k), top_p(top_p), temp(temp) {}
ggml_cgraph * build() override;
// tower KV rows, shared by the prompt and the generated frames
static constexpr int n_pos_max = 4096;
mtmd_subctx_gen_process gen_process;
mtmd_subctx_gen_state * gen_state; // nullptr on buffer reserve
int n_frames; // GEN_WAV only
int top_k;
float top_p;
float temp;
ggml_cgraph * build_tower();
ggml_cgraph * build_codec();
ggml_tensor * tower_layer(ggml_tensor * x, int il, ggml_tensor * code, ggml_tensor * mask, ggml_tensor * extra,
ggml_tensor * inp_pos, ggml_tensor * k_slot, ggml_tensor * v_slot, int pos);
};
//
// fish_s2 generation:
// GEN_CODE = fast AR, backbone hidden + semantic code -> other codes of the frame + next backbone input
// GEN_WAV = Fish DAC decoder, same as drama3
//
struct mtmd_graph_fish_s2_gen : mtmd_graph_drama3_gen {
using mtmd_graph_drama3_gen::mtmd_graph_drama3_gen;
ggml_cgraph * build() override;
ggml_cgraph * build_fast_ar();
};
// fish_s2 voice cloning: raw 44.1 kHz clip -> codec encoder + RVQ -> one backbone input row per frame
struct mtmd_graph_fish_s2_spkenc : mtmd_graph {
mtmd_graph_fish_s2_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
// codec encoder transformer layers, the rest of model.layers is the quantizer pre_module
static constexpr int n_enc_tfm_layer = 4;
static constexpr int n_tfm_layer = 12;
ggml_tensor * conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation) const;
ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const;
ggml_tensor * convnext(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk) const;
};
//
// zonos3 generation:
// GEN_CODE = 10 codebook heads, or the backbone row of a sampled frame
// GEN_WAV = ZAC codec decoder up to the iSTFT spectra
//
struct mtmd_graph_zonos3_gen : mtmd_graph {
mtmd_graph_zonos3_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, bool all_known, int n_frames)
: mtmd_graph(ctx, img), gen_process(gen_process), all_known(all_known), n_frames(n_frames) {}
ggml_cgraph * build() override;
static constexpr int n_codebooks = 10;
static constexpr int n_cb_vocab = 4096;
static constexpr float softcap = 15.0f;
// ZAC decoder, fixed by the codec config
static constexpr int n_stages = 5;
static constexpr int stage_ctx[n_stages] = { 150, 300, 480, 480, 480 }; // attention window
static constexpr int head_dim = 64;
static constexpr float ln_eps = 1e-5f;
static constexpr float convnext_eps = 1e-6f;
static constexpr int hop = 184; // at 44160 Hz
static constexpr int n_fft[2] = { 736, 2208 };
static constexpr int rs_orig = 736; // 44160 -> 44100 Hz
static constexpr int rs_new = 735;
static constexpr int samples_per_frame = 2940;
mtmd_subctx_gen_process gen_process;
bool all_known; // GEN_CODE: only the backbone row
int n_frames; // GEN_WAV
std::map<std::string, ggml_tensor *> state_in;
std::vector<std::pair<std::string, ggml_tensor *>> state_out;
ggml_cgraph * build_gen_code();
ggml_cgraph * build_gen_wav();
ggml_tensor * convnext(ggml_tensor * x, const mtmd_zonos3_wav::convnext & b, const std::string & state_name);
ggml_tensor * moss_layer(ggml_tensor * x, const mtmd_zonos3_wav::layer & l, int s, int il, ggml_tensor * base);
};
std::vector<c2w_state_slot> list_zonos3_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
//
// YuE2 generation, the NAR transformer itself runs on a llama_context, see mtmd_gen_flow_yue2():
// LATENT_IN = latent rows -> backbone embd rows
// LATENT_OUT = backbone hidden rows -> velocity rows
// GEN_WAV = Oobleck VAE decoder, latent rows -> interleaved stereo PCM
//
struct mtmd_graph_yue2_gen : mtmd_graph {
mtmd_graph_yue2_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int n_rows)
: mtmd_graph(ctx, img), gen_process(gen_process), n_rows(n_rows) {}
ggml_cgraph * build() override;
static constexpr int n_latent = 64;
static constexpr int n_freq = 256; // time embedder input
static constexpr int n_channels = 2;
static constexpr int vae_ratio = 1920; // output samples per latent frame
static constexpr int vae_trim = 64; // a decode of n frames has vae_ratio * n - vae_trim samples
static constexpr float rope_theta = 1000000.0f;
static constexpr int n_ctx_max = 24576; // the AR prefix and the NAR rows of one chunk
static constexpr int ode_steps = 32; // midpoint solver
mtmd_subctx_gen_process gen_process;
int n_rows;
ggml_tensor * snake(ggml_tensor * x, const mtmd_yue2::snake & s) const;
ggml_tensor * conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const;
};
struct mtmd_graph_soprano : mtmd_graph {
mtmd_graph_soprano(mtmd_subcontext * ctx, const mtmd_image_f32 & img, int n_frames) : mtmd_graph(ctx, img), n_frames(n_frames) {}
ggml_cgraph * build() override;
static void decode_spectrum(const mtmd_model & model, const std::vector<float> & spectrum, std::vector<float> & pcm);
// SopranoDecoder default
static constexpr int upscale = 4;
int n_frames;
};
//
// LFM3-Audio generation:
// GEN_CODE = one MaskGIT pass of the depthformer, or only the feedback embedding once all codes are known
// GEN_WAV = liquid-codec decoder, a window of frames -> polar STFT frames
//
struct mtmd_graph_lfm3a_gen : mtmd_graph {
mtmd_graph_lfm3a_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, bool all_known, int n_frames)
: mtmd_graph(ctx, img), gen_process(gen_process), all_known(all_known), n_frames(n_frames) {}
ggml_cgraph * build() override;
mtmd_subctx_gen_process gen_process;
bool all_known; // GEN_CODE only
int n_frames; // GEN_WAV only
ggml_tensor * build_gen_code();
ggml_tensor * build_gen_wav();
ggml_tensor * build_lfm_layer(ggml_tensor * inpL, const mtmd_layer & layer, float eps, int il,
const std::function<ggml_tensor *(ggml_tensor *)> & op);
ggml_tensor * build_lfm_attn(ggml_tensor * cur, const mtmd_layer & layer, int n_head, int n_head_kv,
float eps, float rope_theta, int il, ggml_tensor * inp_pos,
ggml_tensor * kq_mask, ggml_tensor * k_cache, ggml_tensor * v_cache,
ggml_tensor ** k_all, ggml_tensor ** v_all);
ggml_tensor * build_lfm_conv(ggml_tensor * cur, const mtmd_layer & layer, int il,
ggml_tensor * conv_state, ggml_tensor ** conv_state_out);
};
// state slots of the streaming codec decoder (lfm3a_gen GEN_WAV)
std::vector<c2w_state_slot> list_lfm3a_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
// state slots of the streaming LFM3-Audio encoder (lfm3a)
std::vector<c2w_state_slot> list_lfm3a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
// state slots of the streaming Gemma 4 audio encoder (gemma4a)
std::vector<c2w_state_slot> list_gemma4a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
// vibevoice-asr: the left context of every causal conv, a zero state is the causal zero padding
std::vector<c2w_state_slot> list_vibevoice_asr_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
struct mtmd_graph_kimik25 : mtmd_graph {
mtmd_graph_kimik25(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode);
};
struct mtmd_graph_parakeet : mtmd_graph {
mtmd_graph_parakeet(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_exaone4_5 : mtmd_graph {
mtmd_graph_exaone4_5(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_granite4_vision : mtmd_graph {
mtmd_graph_granite4_vision(mtmd_subcontext * ctx, const mtmd_image_f32 & img)
: mtmd_graph(ctx, img),
anyres(img.anyres),
n_tiles(img.ny() / img.nx()),
tile_side(img.nx() / patch_size) {}
ggml_cgraph * build() override;
private:
// the input image is a stack of tiles on the Y axis: [overview, tile(0,0), tile(0,1), ...]
const mtmd_image_f32::anyres_info anyres;
const int n_tiles;
const int tile_side; // patches per tile side
ggml_tensor * build_tile_inp();
ggml_tensor * gather(ggml_tensor * src, const std::string & name, int idx_len);
ggml_tensor * interp_down(ggml_tensor * src, int side, int new_side);
ggml_tensor * build_block(const qf_block & blk, ggml_tensor * h, int bid,
int spatial_offset, int image_side, int window_side,
int query_side, float qformer_eps);
ggml_tensor * build_newline_row(ggml_context * ctx0);
ggml_tensor * build_anyres_assembly(ggml_tensor * cur, int out_side);
};
struct mtmd_graph_muse_glimmer : mtmd_graph {
mtmd_graph_muse_glimmer(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct mtmd_graph_inkling2v : mtmd_graph {
using mtmd_graph::mtmd_graph;
ggml_cgraph * build() override;
};