Files
llama.cpp/tools/mtmd/mtmd-subcontext.h
T
Xuan-Son Nguyen e4b620af19 mtmd: support voxcpm2 (#32443)
* mtmd: support voxcpm

* speed up a bit

* various fixes
2026-11-19 02:39:05 +01:00

236 lines
9.4 KiB
C++

#pragma once
#include "ggml.h"
#include "ggml-backend.h"
#include "ggml-cpp.h"
#include "mtmd-impl.h"
#include "mtmd-model.h"
#include <stddef.h>
#include <stdint.h>
#include <map>
#include <memory>
#include <random>
#include <vector>
// !!! Internal header, to be used by mtmd only !!!
// runs the graphs of one mtmd_model on the shared backends and scheduler
enum mtmd_flash_attn_type {
MTMD_FLASH_ATTN_TYPE_AUTO = -1,
MTMD_FLASH_ATTN_TYPE_DISABLED = 0,
MTMD_FLASH_ATTN_TYPE_ENABLED = 1,
};
struct mtmd_subctx_params {
enum mtmd_flash_attn_type flash_attn_type;
bool warmup;
};
// the device the weights and the graphs are placed on, nullptr means CPU
ggml_backend_dev_t mtmd_select_device(bool use_gpu, ggml_backend_dev_t device);
// shared by the subcontexts, not thread-safe
struct mtmd_backends {
ggml_backend_t backend = nullptr; // backend_cpu if no accelerator
ggml_backend_t backend_cpu = nullptr;
// for ggml_backend_sched_new(), backend_cpu last
std::vector<ggml_backend_t> ptrs;
std::vector<ggml_backend_buffer_type_t> bufts;
std::shared_ptr<ggml_threadpool> threadpool; // can be nullptr
// its buffers fit the largest graph reserved by the subcontexts
ggml_backend_sched_ptr sched;
// device nullptr means CPU
mtmd_backends(ggml_backend_dev_t device, int n_threads, int max_nodes,
ggml_backend_sched_eval_callback cb_eval, void * cb_eval_user_data);
~mtmd_backends();
std::map<ggml_backend_dev_t, size_t> get_mem_compute() const;
mtmd_backends(const mtmd_backends &) = delete;
mtmd_backends & operator=(const mtmd_backends &) = delete;
};
// compute the log-mel of n_frames frames, the waveform must already be padded and window must hold one whole frame
// out has the layout [n_frames, n_mel]; must not run at the same time as another pass
struct mtmd_audio_mel_config;
bool mtmd_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out);
enum mtmd_subctx_gen_process {
MTMD_SUBCTX_GEN_PROCESS_GEN_UNKNOWN,
MTMD_SUBCTX_GEN_PROCESS_GEN_CODE, // h_state to codes
MTMD_SUBCTX_GEN_PROCESS_GEN_WAV, // codes to raw PCM audio
MTMD_SUBCTX_GEN_PROCESS_TTS, // full utterance of codes to raw PCM audio
MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE, // internal: mel + source stft to istft input
MTMD_SUBCTX_GEN_PROCESS_TOKENIZE, // raw PCM audio to semantic speech tokens
MTMD_SUBCTX_GEN_PROCESS_LATENT_IN, // internal: flow latent rows to backbone embd rows (tiny-voca: token states to durations)
MTMD_SUBCTX_GEN_PROCESS_LATENT_OUT, // internal: backbone hidden rows to flow velocity rows (tiny-voca: to source-filter params)
MTMD_SUBCTX_GEN_PROCESS_RES_INP, // internal: backbone row to residual LM input row (voxcpm2)
MTMD_SUBCTX_GEN_PROCESS_FLOW_COND, // internal: frame rows to flow condition rows
MTMD_SUBCTX_GEN_PROCESS_FLOW_STEP, // internal: one velocity pass of the flow transformer
};
// persistent state of a causal audio encoder, for streaming input (one per stream)
// it holds the sliding window of past keys/values, so each encode() call only takes the new frames
struct mtmd_subctx_enc_state {
int32_t n_ctx = 0; // ring slots: attention window + frames per call
int32_t n_past = 0; // absolute position of the next frame
std::vector<int32_t> slot_pos; // absolute position held by each slot, -1 if empty
std::vector<float> stem_tail; // last conv stem output frame, empty on a cold start
ggml_context_ptr ctx_data;
ggml_backend_buffer_ptr buf;
std::vector<ggml_tensor *> k; // per layer [n_embd_kv, n_ctx]
std::vector<ggml_tensor *> v;
// lfm3a: same order as list_lfm3a_state_slots(), F32 [ne0, ne1], updated in place
std::vector<ggml_tensor *> slots;
// nemotron transcribe: the window's normalized log-mel, cache and FIFO rows in; its speaker rows and predictions out
std::vector<float> mel_norm;
std::vector<float> spk_prev;
std::vector<float> spk_chunk, spk_preds;
};
// per-sequence state of audio generation: RNG and the slots kept between GEN_WAV calls
struct mtmd_subctx_gen_state {
std::mt19937 rng;
ggml_context_ptr ctx_data;
ggml_backend_buffer_ptr buf;
std::vector<ggml_tensor *> slots; // same order as list_gen_state_slots(), F32 [ne0, ne1]
// clears the slots and keeps the RNG stream
void reset() {
if (buf) {
ggml_backend_buffer_clear(buf.get(), 0);
}
}
};
// kokoro: inputs and outputs of one graph stage, see mtmd_graph_kokoro
struct mtmd_subctx_kokoro_io {
int stage = 0;
int n_tok = 0;
int n_frames = 0;
int n_har = 0;
const std::vector<int32_t> * tokens = nullptr; // STAGE_TEXT
const std::vector<float> * x = nullptr; // STAGE_F0N: shared LSTM output, STAGE_DEC: aligned text features
const std::vector<float> * style = nullptr; // predictor or decoder half of the voice row
const std::vector<float> * f0 = nullptr; // STAGE_DEC
const std::vector<float> * en = nullptr;
const std::vector<float> * har = nullptr;
std::vector<float> * out0 = nullptr; // text features, F0, or STFT coefficients
std::vector<float> * out1 = nullptr; // STAGE_F0N: energy
};
struct mtmd_subctx_encode_params {
int n_threads = 1;
const mtmd_image_f32_batch * imgs = nullptr;
std::vector<float> * out_embd = nullptr;
// for audio gen, imgs has exactly one entry: hidden state from backbone (GEN_CODE) or unused (GEN_WAV)
mtmd_subctx_gen_process gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_UNKNOWN;
// GEN_CODE: out_embd receives the embd to feed back to the backbone
int32_t code0 = 0; // semantic code sampled by the backbone
int32_t top_k = 50;
float top_p = 1.0f;
std::vector<int32_t> * out_codes = nullptr; // this frame's 16 sampled codes
std::vector<float> * out_feats = nullptr; // continuous counterpart of out_codes
float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders
float cfg_scale = 1.0f; // classifier-free guidance, 1.0 = off
float flow_t = 0.0f; // FLOW_STEP: flow-matching time, 0 = noise
int n_steps = 0; // steps of the flow-matching ODE solver (voxcpm2)
bool * out_is_eos = nullptr;
// TOKENIZE: speech embedding rows of the produced codes
std::vector<float> * out_code_embd = nullptr;
// GEN_WAV
const std::vector<int32_t> * codes = nullptr; // this frame's 16 RVQ codes
const std::vector<float> * feats = nullptr; // continuous counterpart of codes
std::vector<float> * out_audio = nullptr; // decoded PCM samples, F32
const mtmd_gen_ref * ref = nullptr; // TTS reference conditioning of the voice
// TTS_VOCODE internal stage inputs
const std::vector<float> * mel_in = nullptr;
const std::vector<float> * sstft_in = nullptr;
int vocode_n_mel = 0;
int vocode_n_stft = 0;
std::vector<float> * out_spec = nullptr;
// kokoro: set by encode_kokoro() for each of its graph stages
const mtmd_subctx_kokoro_io * kokoro = nullptr;
float speed = 1.0f; // kokoro: the predicted durations are divided by it
// audio gen: state of one sequence, encode() updates it in place
struct mtmd_subctx_gen_state * gen_state = nullptr;
// streaming audio input, updated in place by encode()
struct mtmd_subctx_enc_state * enc_state = nullptr;
};
struct mtmd_subcontext {
const mtmd_model & model;
mtmd_backends & backends;
std::vector<uint8_t> buf_compute_meta;
int max_nodes = 8192;
mtmd_flash_attn_type flash_attn_type = MTMD_FLASH_ATTN_TYPE_AUTO;
bool is_allocated = false;
bool debug_output_embeddings = false;
bool support_batch = false;
// model and backends must outlive the subcontext
mtmd_subcontext(const mtmd_model & model, mtmd_backends & backends, const mtmd_subctx_params & params);
bool encode(mtmd_subctx_encode_params * params);
// reserve the compute buffers for the given batch
void warmup(const mtmd_image_f32_batch & batch);
// TODO: remove encode_image() and always use batched version
bool encode_image(int n_threads, const mtmd_image_f32 * img, std::vector<float> & out_vec);
bool encode_batch(int n_threads, const mtmd_image_f32_batch * imgs, std::vector<float> & out_batch_embd);
// n_frames_max is the maximum number of encoder frames per encode() call, free the result with delete
mtmd_subctx_enc_state * enc_state_init(int32_t n_frames_max);
// seed UINT32_MAX means random, free the result with delete
mtmd_subctx_gen_state * gen_state_init(uint32_t seed);
private:
struct support_info_op {
ggml_tensor * op;
// true if the op runs on the accelerated backend
bool is_accel = true;
};
struct support_info_graph {
// whether the backend supports flash attention
bool fattn = true;
ggml_tensor * fattn_op = nullptr; // for debugging
std::vector<support_info_op> ops;
};
mtmd_image_f32_batch get_dummy_batch() const;
// kokoro GEN_WAV: codes are the phoneme tokens, feats the voice row followed by the backbone state of each token
bool encode_kokoro(mtmd_subctx_encode_params * params);
// only initialize backend buffers, but do not allocate them yet
support_info_graph reserve_compute_meta(const mtmd_image_f32_batch & batch);
};