mtmd: add mtmd_audio_stream + support voxtral-realtime (#30629)
* mtmd: add mtmd_audio_stream + support voxtral-realtime * address review
This commit is contained in:
1 parent
59d5fad7c1
commit
2851b3bcbf
27 files changed
+1000
-22
No files matched your search
+61
-1
@@ -8,6 +8,8 @@ void llama_model_mistral3::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_ROPE_SCALING_YARN_BETA_SLOW, hparams.yarn_beta_slow, false);
|
||||
ml.get_key(LLM_KV_ROPE_SCALING_YARN_LOG_MUL, hparams.rope_yarn_log_mul, false);
|
||||
|
||||
ml.get_key(LLM_KV_REALTIME_DELAY_TOKENS, hparams.n_tokens_delay, false);
|
||||
|
||||
hparams.f_attn_temp_offset = 0.0f;
|
||||
|
||||
// TODO: maybe add n_attn_temp_floor_scale as a separate KV?
|
||||
@@ -26,7 +28,7 @@ void llama_model_mistral3::load_arch_hparams(llama_model_loader & ml) {
|
||||
}
|
||||
}
|
||||
|
||||
void llama_model_mistral3::load_arch_tensors(llama_model_loader &) {
|
||||
void llama_model_mistral3::load_arch_tensors(llama_model_loader & ml) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
|
||||
@@ -53,6 +55,13 @@ void llama_model_mistral3::load_arch_tensors(llama_model_loader &) {
|
||||
|
||||
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
|
||||
|
||||
// optional ada rms norm (voxtral realtime), the hidden size is taken from the tensor itself
|
||||
if (const ggml_tensor * meta = ml.get_tensor_meta(tn(LLM_TENSOR_FFN_NORM_ADA_DOWN, "weight", i).str().c_str())) {
|
||||
const int64_t n_ada = meta->ne[1];
|
||||
layer.ffn_norm_ada_down = create_tensor(tn(LLM_TENSOR_FFN_NORM_ADA_DOWN, "weight", i), {n_embd, n_ada}, 0);
|
||||
layer.ffn_norm_ada_up = create_tensor(tn(LLM_TENSOR_FFN_NORM_ADA_UP, "weight", i), {n_ada, n_embd}, 0);
|
||||
}
|
||||
|
||||
if (hparams.rope_scaling_type_train == LLAMA_ROPE_SCALING_TYPE_LONGROPE) {
|
||||
layer.rope_long = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
|
||||
layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
|
||||
@@ -86,6 +95,34 @@ void llama_model_mistral3::load_arch_tensors(llama_model_loader &) {
|
||||
}
|
||||
}
|
||||
|
||||
// sinusoidal embedding of the output delay, constant for the whole session
|
||||
class llama_model_mistral3::input_t_cond : public llm_graph_input_i {
|
||||
public:
|
||||
input_t_cond(uint32_t n_delay, int64_t n_embd) : data(n_embd) {
|
||||
const int64_t half = n_embd / 2;
|
||||
for (int64_t i = 0; i < half; i++) {
|
||||
const double freq = exp(-log(10000.0) * i / half) * n_delay;
|
||||
data[i] = cos(freq);
|
||||
data[half + i] = sin(freq);
|
||||
}
|
||||
}
|
||||
|
||||
void set_input(const llama_ubatch * ubatch) override {
|
||||
GGML_UNUSED(ubatch);
|
||||
ggml_backend_tensor_set(t_cond, data.data(), 0, ggml_nbytes(t_cond));
|
||||
}
|
||||
|
||||
// the input does not depend on the batch
|
||||
bool can_reuse(const llm_graph_params & params) override {
|
||||
GGML_UNUSED(params);
|
||||
return true;
|
||||
}
|
||||
|
||||
ggml_tensor * t_cond = nullptr; // F32 [n_embd]
|
||||
|
||||
std::vector<float> data;
|
||||
};
|
||||
|
||||
std::unique_ptr<llm_graph_context> llama_model_mistral3::build_arch_graph(const llm_graph_params & params) const {
|
||||
return std::make_unique<graph>(*this, params);
|
||||
}
|
||||
@@ -112,6 +149,17 @@ llama_model_mistral3::graph::graph(const llama_model & model, const llm_graph_pa
|
||||
|
||||
auto * inp_attn = build_attn_inp_kv();
|
||||
|
||||
// (optional) time conditioning of the ada rms norm
|
||||
ggml_tensor * t_cond = nullptr;
|
||||
if (model.layers[0].ffn_norm_ada_down) {
|
||||
auto inp = std::make_unique<input_t_cond>(hparams.n_tokens_delay, n_embd);
|
||||
t_cond = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_embd);
|
||||
ggml_set_input(t_cond);
|
||||
cb(t_cond, "inp_t_cond", -1);
|
||||
inp->t_cond = t_cond;
|
||||
res->add_input(std::move(inp));
|
||||
}
|
||||
|
||||
const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
|
||||
|
||||
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
||||
@@ -176,6 +224,18 @@ llama_model_mistral3::graph::graph(const llama_model & model, const llm_graph_pa
|
||||
LLM_NORM_RMS, il);
|
||||
cb(cur, "ffn_norm", il);
|
||||
|
||||
if (t_cond) {
|
||||
// ada rms norm: scale the normalized input by (1 + mlp(t_cond))
|
||||
ggml_tensor * ada = build_ffn(t_cond,
|
||||
model.layers[il].ffn_norm_ada_down, NULL, NULL,
|
||||
NULL, NULL, NULL,
|
||||
model.layers[il].ffn_norm_ada_up, NULL, NULL,
|
||||
NULL,
|
||||
LLM_FFN_GELU_ERF, LLM_FFN_SEQ, il);
|
||||
cur = ggml_add(ctx0, cur, ggml_mul(ctx0, cur, ada));
|
||||
cb(cur, "ffn_norm_ada", il);
|
||||
}
|
||||
|
||||
cur = build_ffn(cur,
|
||||
model.layers[il].ffn_up, model.layers[il].ffn_up_b, model.layers[il].ffn_up_s,
|
||||
model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, model.layers[il].ffn_gate_s,
|
||||
|
||||
Reference in new issue
Block a user