mtmd: add mtmd_audio_stream + support voxtral-realtime (#30629)

* mtmd: add mtmd_audio_stream + support voxtral-realtime

* address review
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-10-16 01:46:42 +02:00
1 parent 59d5fad7c1
commit 2851b3bcbf
27 files changed
+1000 -22

No files matched your search

+61 -1
View File
@@ -8,6 +8,8 @@ void llama_model_mistral3::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ROPE_SCALING_YARN_BETA_SLOW, hparams.yarn_beta_slow, false);
ml.get_key(LLM_KV_ROPE_SCALING_YARN_LOG_MUL, hparams.rope_yarn_log_mul, false);
ml.get_key(LLM_KV_REALTIME_DELAY_TOKENS, hparams.n_tokens_delay, false);
hparams.f_attn_temp_offset = 0.0f;
// TODO: maybe add n_attn_temp_floor_scale as a separate KV?
@@ -26,7 +28,7 @@ void llama_model_mistral3::load_arch_hparams(llama_model_loader & ml) {
}
}
void llama_model_mistral3::load_arch_tensors(llama_model_loader &) {
void llama_model_mistral3::load_arch_tensors(llama_model_loader & ml) {
LLAMA_LOAD_LOCALS;
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
@@ -53,6 +55,13 @@ void llama_model_mistral3::load_arch_tensors(llama_model_loader &) {
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
// optional ada rms norm (voxtral realtime), the hidden size is taken from the tensor itself
if (const ggml_tensor * meta = ml.get_tensor_meta(tn(LLM_TENSOR_FFN_NORM_ADA_DOWN, "weight", i).str().c_str())) {
const int64_t n_ada = meta->ne[1];
layer.ffn_norm_ada_down = create_tensor(tn(LLM_TENSOR_FFN_NORM_ADA_DOWN, "weight", i), {n_embd, n_ada}, 0);
layer.ffn_norm_ada_up = create_tensor(tn(LLM_TENSOR_FFN_NORM_ADA_UP, "weight", i), {n_ada, n_embd}, 0);
}
if (hparams.rope_scaling_type_train == LLAMA_ROPE_SCALING_TYPE_LONGROPE) {
layer.rope_long = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
@@ -86,6 +95,34 @@ void llama_model_mistral3::load_arch_tensors(llama_model_loader &) {
}
}
// sinusoidal embedding of the output delay, constant for the whole session
class llama_model_mistral3::input_t_cond : public llm_graph_input_i {
public:
input_t_cond(uint32_t n_delay, int64_t n_embd) : data(n_embd) {
const int64_t half = n_embd / 2;
for (int64_t i = 0; i < half; i++) {
const double freq = exp(-log(10000.0) * i / half) * n_delay;
data[i] = cos(freq);
data[half + i] = sin(freq);
}
}
void set_input(const llama_ubatch * ubatch) override {
GGML_UNUSED(ubatch);
ggml_backend_tensor_set(t_cond, data.data(), 0, ggml_nbytes(t_cond));
}
// the input does not depend on the batch
bool can_reuse(const llm_graph_params & params) override {
GGML_UNUSED(params);
return true;
}
ggml_tensor * t_cond = nullptr; // F32 [n_embd]
std::vector<float> data;
};
std::unique_ptr<llm_graph_context> llama_model_mistral3::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(*this, params);
}
@@ -112,6 +149,17 @@ llama_model_mistral3::graph::graph(const llama_model & model, const llm_graph_pa
auto * inp_attn = build_attn_inp_kv();
// (optional) time conditioning of the ada rms norm
ggml_tensor * t_cond = nullptr;
if (model.layers[0].ffn_norm_ada_down) {
auto inp = std::make_unique<input_t_cond>(hparams.n_tokens_delay, n_embd);
t_cond = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_embd);
ggml_set_input(t_cond);
cb(t_cond, "inp_t_cond", -1);
inp->t_cond = t_cond;
res->add_input(std::move(inp));
}
const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
ggml_tensor * inp_out_ids = build_inp_out_ids();
@@ -176,6 +224,18 @@ llama_model_mistral3::graph::graph(const llama_model & model, const llm_graph_pa
LLM_NORM_RMS, il);
cb(cur, "ffn_norm", il);
if (t_cond) {
// ada rms norm: scale the normalized input by (1 + mlp(t_cond))
ggml_tensor * ada = build_ffn(t_cond,
model.layers[il].ffn_norm_ada_down, NULL, NULL,
NULL, NULL, NULL,
model.layers[il].ffn_norm_ada_up, NULL, NULL,
NULL,
LLM_FFN_GELU_ERF, LLM_FFN_SEQ, il);
cur = ggml_add(ctx0, cur, ggml_mul(ctx0, cur, ada));
cb(cur, "ffn_norm_ada", il);
}
cur = build_ffn(cur,
model.layers[il].ffn_up, model.layers[il].ffn_up_b, model.layers[il].ffn_up_s,
model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, model.layers[il].ffn_gate_s,