* mtmd: support hibiki realtime duplex * fix and clean up * fix long session quality * fix ci * tests : add the moshi fusion baseline for Metal The Metal fusion check found 5 RMS_NORM+MUL fusions in the new moshi arch and no baseline for them. --------- Co-authored-by: Pascal <admin@serveurperso.com>
207 lines
8.4 KiB
C++
207 lines
8.4 KiB
C++
#include "models.h"
|
|
|
|
#include <cmath>
|
|
|
|
// pocket-tts generation stages
|
|
//
|
|
// GEN_CODE: backbone hidden state -> next 32-d latent (flow matching) + end-of-speech score
|
|
// GEN_WAV : a window of latents -> PCM, through the mimi decoder
|
|
//
|
|
// there is no codebook anywhere, "codes" in the mtmd API are continuous features here
|
|
|
|
ggml_tensor * mtmd_graph_pockettts_gen::modulate(ggml_tensor * x, ggml_tensor * shift, ggml_tensor * scale) const {
|
|
ggml_tensor * cur = ggml_mul(ctx0, x, ggml_scale_bias(ctx0, scale, 1.0f, 1.0f));
|
|
return ggml_add(ctx0, cur, shift);
|
|
}
|
|
|
|
// see TimestepEmbedder in the reference
|
|
ggml_tensor * mtmd_graph_pockettts_gen::time_embed(const mtmd_flow_net::time_embd & te, float t) const {
|
|
// t is a graph-build constant, so the cos/sin table can be folded into a scaled copy
|
|
ggml_tensor * args = ggml_scale(ctx0, te.freqs, t);
|
|
ggml_tensor * emb = ggml_concat(ctx0, ggml_cos(ctx0, args), ggml_sin(ctx0, args), 0);
|
|
|
|
ggml_tensor * cur = build_mm(te.up_w, emb);
|
|
cur = ggml_add(ctx0, cur, te.up_b);
|
|
cur = ggml_silu(ctx0, cur);
|
|
cur = build_mm(te.down_w, cur);
|
|
cur = ggml_add(ctx0, cur, te.down_b);
|
|
|
|
// this "RMSNorm" divides by the unbiased variance, not the mean square
|
|
// it also rescales the input, not the centered value, see _rms_norm() in mlp.py
|
|
{
|
|
const int64_t n = cur->ne[0];
|
|
ggml_tensor * mean = ggml_mean(ctx0, cur);
|
|
ggml_tensor * dev = ggml_sub(ctx0, cur, mean);
|
|
ggml_tensor * var = ggml_mean(ctx0, ggml_sqr(ctx0, dev));
|
|
var = ggml_scale_bias(ctx0, var, (float) n / (float) (n - 1), 1e-5f);
|
|
cur = ggml_div(ctx0, cur, ggml_sqrt(ctx0, var));
|
|
cur = ggml_mul(ctx0, cur, te.norm);
|
|
}
|
|
|
|
return cur;
|
|
}
|
|
|
|
// one velocity evaluation: v(cond, s, t, x)
|
|
ggml_tensor * mtmd_graph_pockettts_gen::flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const {
|
|
const auto & flow = model.flow;
|
|
|
|
ggml_tensor * cur = build_mm(flow.input_proj_w, x);
|
|
cur = ggml_add(ctx0, cur, flow.input_proj_b);
|
|
|
|
// the two time conditions are averaged, then added to the projected backbone state
|
|
ggml_tensor * ts = ggml_add(ctx0, time_embed(flow.time[0], s), time_embed(flow.time[1], t));
|
|
ts = ggml_scale(ctx0, ts, 1.0f / (float) flow.time.size());
|
|
|
|
ggml_tensor * c = build_mm(flow.cond_embd_w, cond);
|
|
c = ggml_add(ctx0, c, flow.cond_embd_b);
|
|
|
|
ggml_tensor * y = ggml_add(ctx0, ts, c);
|
|
cb(y, "flow_cond", -1);
|
|
|
|
const int64_t n_ch = flow.blocks.empty() ? 0 : flow.blocks[0].norm_w->ne[0];
|
|
|
|
for (size_t il = 0; il < flow.blocks.size(); il++) {
|
|
const auto & blk = flow.blocks[il];
|
|
|
|
ggml_tensor * mod = build_mm(blk.ada_w, ggml_silu(ctx0, y));
|
|
mod = ggml_add(ctx0, mod, blk.ada_b);
|
|
|
|
ggml_tensor * shift = ggml_view_1d(ctx0, mod, n_ch, 0);
|
|
ggml_tensor * scale = ggml_view_1d(ctx0, mod, n_ch, (size_t) n_ch * mod->nb[0]);
|
|
ggml_tensor * gate = ggml_view_1d(ctx0, mod, n_ch, (size_t) 2 * n_ch * mod->nb[0]);
|
|
|
|
ggml_tensor * h = build_norm(cur, blk.norm_w, blk.norm_b, NORM_TYPE_NORMAL, 1e-6f, (int) il);
|
|
h = modulate(h, shift, scale);
|
|
h = build_mm(blk.up_w, h);
|
|
h = ggml_add(ctx0, h, blk.up_b);
|
|
h = ggml_silu(ctx0, h);
|
|
h = build_mm(blk.down_w, h);
|
|
h = ggml_add(ctx0, h, blk.down_b);
|
|
|
|
cur = ggml_add(ctx0, cur, ggml_mul(ctx0, gate, h));
|
|
cb(cur, "flow_blk", (int) il);
|
|
}
|
|
|
|
// final layer: the norm has no weights, only the AdaLN modulation
|
|
ggml_tensor * mod = build_mm(flow.final_ada_w, ggml_silu(ctx0, y));
|
|
mod = ggml_add(ctx0, mod, flow.final_ada_b);
|
|
|
|
ggml_tensor * shift = ggml_view_1d(ctx0, mod, n_ch, 0);
|
|
ggml_tensor * scale = ggml_view_1d(ctx0, mod, n_ch, (size_t) n_ch * mod->nb[0]);
|
|
|
|
cur = build_norm(cur, nullptr, nullptr, NORM_TYPE_NORMAL, 1e-6f, -1);
|
|
cur = modulate(cur, shift, scale);
|
|
cur = build_mm(flow.final_proj_w, cur);
|
|
cur = ggml_add(ctx0, cur, flow.final_proj_b);
|
|
|
|
return cur;
|
|
}
|
|
|
|
// state carried between GEN_WAV calls: rope offset, per-layer KV window, conv left context
|
|
// and the transposed-conv overlap tails
|
|
std::vector<c2w_state_slot> list_pockettts_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
|
|
std::vector<c2w_state_slot> slots;
|
|
if (model.gen_upsample_w == nullptr) {
|
|
return slots; // not a pocket-tts decoder
|
|
}
|
|
const auto & seanet = model.seanet;
|
|
|
|
// the slots below are sized from these
|
|
GGML_ASSERT(!model.gen_tfm_layers.empty());
|
|
GGML_ASSERT((int) seanet.stages.size() >= hparams.seanet_n_stage);
|
|
GGML_ASSERT((int) hparams.seanet_ratios.size() >= hparams.seanet_n_stage);
|
|
GGML_ASSERT(hparams.mimi_tfm_context > 1 && hparams.mimi_downsample > 0);
|
|
|
|
slots.push_back({"pos", 1, 1});
|
|
|
|
const int64_t n_embd_a = model.gen_tfm_layers[0].q_w->ne[1];
|
|
const int64_t prefix = hparams.mimi_tfm_context - 1;
|
|
for (size_t il = 0; il < model.gen_tfm_layers.size(); il++) {
|
|
slots.push_back({"tfm_k_" + std::to_string(il), n_embd_a, prefix});
|
|
slots.push_back({"tfm_v_" + std::to_string(il), n_embd_a, prefix});
|
|
}
|
|
|
|
// upsample is depthwise, its output channel count is the input one
|
|
slots.push_back({"up", model.gen_upsample_w->ne[0] - hparams.mimi_downsample, model.gen_upsample_w->ne[2]});
|
|
|
|
slots.push_back({"dec_in", seanet.conv_in_w->ne[0] - 1, seanet.conv_in_w->ne[1]});
|
|
for (int i = 0; i < hparams.seanet_n_stage; i++) {
|
|
const auto & stage = seanet.stages[i];
|
|
const int stride = hparams.seanet_ratios[hparams.seanet_n_stage - 1 - i];
|
|
slots.push_back({"dec_up_" + std::to_string(i), stage.scale_conv_w->ne[0] - stride, stage.scale_conv_w->ne[1]});
|
|
slots.push_back({"dec_res_" + std::to_string(i), stage.res_conv1_w->ne[0] - 1, stage.res_conv1_w->ne[1]});
|
|
}
|
|
slots.push_back({"dec_out", seanet.conv_out_w->ne[0] - 1, seanet.conv_out_w->ne[1]});
|
|
|
|
return slots;
|
|
}
|
|
|
|
ggml_cgraph * mtmd_graph_pockettts_gen::build() {
|
|
if (gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE) {
|
|
// the backbone hidden state arrives as the single batch entry
|
|
ggml_tensor * h_state = build_inp_raw(1);
|
|
h_state = ggml_reshape_2d(ctx0, h_state, n_mmproj_embd, 1);
|
|
|
|
// end-of-speech probe, thresholded on the host side
|
|
ggml_tensor * eos = build_mm(model.gen_out_eos_w, h_state);
|
|
eos = ggml_add(ctx0, eos, model.gen_out_eos_b);
|
|
ggml_set_name(eos, "out_eos_score");
|
|
ggml_set_output(eos);
|
|
ggml_build_forward_expand(gf, eos);
|
|
|
|
const int64_t n_latent = model.gen_input_lin_w->ne[0];
|
|
|
|
ggml_tensor * noise = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_latent, 1);
|
|
ggml_set_name(noise, "inp_noise");
|
|
ggml_set_input(noise);
|
|
|
|
// lsd_decode: integrate the velocity field from the noise sample
|
|
ggml_tensor * cur = noise;
|
|
for (int i = 0; i < n_step; i++) {
|
|
const float s = (float) i / (float) n_step;
|
|
const float t = (float) (i + 1) / (float) n_step;
|
|
ggml_tensor * v = flow_forward(h_state, cur, s, t);
|
|
cur = ggml_add(ctx0, cur, ggml_scale(ctx0, v, 1.0f / (float) n_step));
|
|
}
|
|
cb(cur, "flow_latent", -1);
|
|
|
|
ggml_set_name(cur, "out_feats");
|
|
ggml_set_output(cur);
|
|
ggml_build_forward_expand(gf, cur);
|
|
|
|
// the same latent, projected into the backbone's input space for the next step
|
|
ggml_tensor * embd = build_mm(model.gen_input_lin_w, cur);
|
|
cb(embd, "gen_embd", -1);
|
|
ggml_build_forward_expand(gf, embd);
|
|
|
|
return gf;
|
|
}
|
|
|
|
// GEN_WAV: [32, n_frames] latents -> PCM
|
|
ggml_tensor * feats = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32,
|
|
model.gen_input_lin_w->ne[0], n_frames);
|
|
ggml_set_name(feats, "inp_feats");
|
|
ggml_set_input(feats);
|
|
|
|
// denormalize, then the DummyQuantizer up-projection
|
|
ggml_tensor * cur = ggml_add(ctx0, ggml_mul(ctx0, feats, model.gen_emb_std), model.gen_emb_mean);
|
|
cur = build_mm(model.gen_quant_out_w, cur);
|
|
cb(cur, "quant_out", -1);
|
|
|
|
mtmd_graph_pockettts_seanet seanet(*this);
|
|
seanet.state_inputs(list_pockettts_state_slots(hparams, model));
|
|
|
|
cur = seanet.mimi_decode(cur, n_head, eps);
|
|
|
|
seanet.state_outputs(gf);
|
|
|
|
// [n_samples, 1] -> [n_samples], clamped like the reference output
|
|
cur = ggml_reshape_1d(ctx0, cur, cur->ne[0]);
|
|
cur = ggml_clamp(ctx0, cur, -1.0f, 1.0f);
|
|
ggml_set_name(cur, "out_audio");
|
|
ggml_set_output(cur);
|
|
ggml_build_forward_expand(gf, cur);
|
|
|
|
return gf;
|
|
}
|