Files
llama.cpp/tools/mtmd/models/fish-s2-spkenc.cpp
T
Xuan-Son Nguyen a68c6136bb mtmd: support fish audio s2 (#32330)
* add fisk-audio-s2 support

* add voice cloning

* cont merge master

* nits

* nits

* clean up

* nits

* use new batch api
2026-11-16 17:32:27 +01:00

136 lines
7.1 KiB
C++

#include "models.h"
#include <cmath>
#include <string>
// reference clip -> codec frames -> backbone input rows, see DAC.encode() and DownsampleResidualVectorQuantize.encode()
// activations are [C, T]; a conv is a sum of matmuls over tap-shifted views: exact in F32, no im2col buffer
// causal conv, w is taps-major [stride * IC, OC, K / stride], see conversion/fish_s2.py
ggml_tensor * mtmd_graph_fish_s2_spkenc::conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation) const {
const int64_t n_tap = w->ne[2];
const int pad = (int) (n_tap - 1) * stride * dilation;
if (pad > 0) {
x = ggml_pad_ext(ctx0, x, 0, 0, pad, 0, 0, 0, 0, 0);
}
if (stride > 1) {
// stride frames per column, then each tap group is one matmul
x = ggml_reshape_2d(ctx0, x, x->ne[0] * stride, x->ne[1] / stride);
}
const int64_t n_out = x->ne[1] - (n_tap - 1) * dilation;
ggml_tensor * y = nullptr;
for (int64_t k = 0; k < n_tap; k++) {
ggml_tensor * wk = ggml_view_2d(ctx0, w, w->ne[0], w->ne[1], w->nb[1], (size_t) k * w->nb[2]);
ggml_tensor * xk = ggml_view_2d(ctx0, x, x->ne[0], n_out, x->nb[1], (size_t) k * dilation * x->nb[1]);
ggml_tensor * yk = ggml_mul_mat(ctx0, wk, xk);
y = y ? ggml_add(ctx0, y, yk) : yk;
}
return ggml_add(ctx0, y, b);
}
// x + sin(alpha * x)^2 / alpha, beta = 1 / alpha is folded at conversion
// TODO @ngxson: on Metal, residual codes match the reference less often than on CPU (89-98% vs 99%), maybe the fast-math sin
ggml_tensor * mtmd_graph_fish_s2_spkenc::snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const {
ggml_tensor * s = ggml_sqr(ctx0, ggml_sin(ctx0, ggml_mul(ctx0, x, alpha)));
return ggml_add(ctx0, x, ggml_mul(ctx0, s, beta));
}
// causal depthwise conv (w [C, K]) -> LayerNorm -> pwconv1 -> GELU -> pwconv2 -> layer scale -> residual
ggml_tensor * mtmd_graph_fish_s2_spkenc::convnext(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk) const {
const int64_t n_tap = blk.dwconv_w->ne[1];
const int64_t n_pos = x->ne[1];
ggml_tensor * xp = ggml_pad_ext(ctx0, x, 0, 0, (int) n_tap - 1, 0, 0, 0, 0, 0);
ggml_tensor * h = nullptr;
for (int64_t k = 0; k < n_tap; k++) {
ggml_tensor * wk = ggml_view_1d(ctx0, blk.dwconv_w, blk.dwconv_w->ne[0], (size_t) k * blk.dwconv_w->nb[1]);
ggml_tensor * xk = ggml_view_2d(ctx0, xp, xp->ne[0], n_pos, xp->nb[1], (size_t) k * xp->nb[1]);
ggml_tensor * hk = ggml_mul(ctx0, xk, wk);
h = h ? ggml_add(ctx0, h, hk) : hk;
}
h = ggml_add(ctx0, h, blk.dwconv_b);
h = ggml_add(ctx0, ggml_mul(ctx0, ggml_norm(ctx0, h, 1e-6f), blk.norm_w), blk.norm_b);
h = ggml_gelu_erf(ctx0, ggml_add(ctx0, ggml_mul_mat(ctx0, blk.pw1_w, h), blk.pw1_b));
h = ggml_add(ctx0, ggml_mul_mat(ctx0, blk.pw2_w, h), blk.pw2_b);
return ggml_add(ctx0, x, ggml_mul(ctx0, h, blk.gamma));
}
ggml_cgraph * mtmd_graph_fish_s2_spkenc::build() {
const auto & c2w = model.c2w;
// the preprocessor hands over the waveform as a single-row "mel", [n_samples, 1] is also [1, n_samples]
ggml_tensor * inp_raw = build_inp_raw(1);
ggml_tensor * x = ggml_reshape_2d(ctx0, inp_raw, 1, inp_raw->ne[0]);
// windowed causal transformer on a cold start: zero state, positions from 0
mtmd_graph_qwen3tts_gen::code2wav dec(*this);
auto zero_state = [&](const std::string & name, int64_t n0, int64_t n1) {
dec.state_in[name] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n0, n1), 0.0f);
};
auto transformer = [&](ggml_tensor * cur, int il0, int il1, int64_t window, ggml_tensor * norm_w) {
const int64_t n_kv = model.layers[il0].k_w->ne[1];
zero_state("tfm_pos", 1, 1);
for (int il = il0; il < il1; il++) {
zero_state("tfm_k_" + std::to_string(il), n_kv, window - 1);
zero_state("tfm_v_" + std::to_string(il), n_kv, window - 1);
cur = dec.tfm_layer_forward(cur, model.layers[il], il, window);
}
return ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, hparams.wav_tfm_eps), norm_w);
};
// encoder: entry conv, 4x (3 residual units, Snake, strided conv), transformer, Snake, post conv
x = conv(x, c2w.dac_entry_w, c2w.dac_entry_b, 1, 1);
static constexpr int DAC_DILATIONS[3] = { 1, 3, 9 };
for (const auto & blk : c2w.dac) {
for (size_t ir = 0; ir < blk.res.size(); ir++) {
const auto & res = blk.res[ir];
ggml_tensor * h = snake(x, res.act1_alpha, res.act1_beta);
h = conv(h, res.conv1_w, res.conv1_b, 1, DAC_DILATIONS[ir]);
h = snake(h, res.act2_alpha, res.act2_beta);
h = conv(h, res.conv2_w, res.conv2_b, 1, 1);
x = ggml_add(ctx0, x, h);
}
x = snake(x, blk.snake_alpha, blk.snake_beta);
x = conv(x, blk.conv_w, blk.conv_b, (int) (blk.conv_w->ne[0] / x->ne[0]), 1);
}
x = transformer(x, 0, n_enc_tfm_layer, 512, model.post_ln_w);
x = snake(x, c2w.dac_post_snake_alpha, c2w.dac_post_snake_beta);
x = conv(x, c2w.dac_post_conv_w, c2w.dac_post_conv_b, 1, 1);
cb(x, "codec_enc", -1);
// quantizer: 2x (strided conv + ConvNeXt), transformer
for (const auto & down : c2w.upsample) {
x = conv(x, down.conv_w, down.conv_b, (int) (down.conv_w->ne[0] / x->ne[0]), 1);
x = convnext(x, down);
}
ggml_tensor * z = transformer(x, n_enc_tfm_layer, (int) model.layers.size(), 128, c2w.tfm_output_norm_w);
cb(z, "codec_latent", -1);
// nearest entry by cosine similarity, then the entry through out_proj is removed from the residual
auto vq = [&](ggml_tensor * in_w, ggml_tensor * in_b, ggml_tensor * cb_norm, ggml_tensor * table) {
ggml_tensor * e = ggml_add(ctx0, ggml_mul_mat(ctx0, in_w, z), in_b);
e = ggml_l2_norm(ctx0, e, 1e-12f);
ggml_tensor * idx = ggml_argmax(ctx0, ggml_mul_mat(ctx0, cb_norm, e));
z = ggml_sub(ctx0, z, ggml_get_rows(ctx0, table, idx));
return idx;
};
auto page = [&](ggml_tensor * t, int64_t g) {
return ggml_view_2d(ctx0, t, t->ne[0], t->ne[1], t->nb[1], (size_t) g * t->nb[2]);
};
// backbone row: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1), same as fish_s2_gen
const int64_t n_cb = model.gen_code_embd_w->ne[2];
ggml_tensor * idx = vq(c2w.quant_first_in_w, c2w.quant_first_in_b, c2w.quant_first_cb_norm_w, c2w.quant_first_cb_w);
ggml_tensor * out = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, idx),
ggml_get_rows(ctx0, page(model.gen_code_embd_w, 0), idx));
for (int64_t g = 1; g < n_cb; g++) {
ggml_tensor * in_b = ggml_view_1d(ctx0, c2w.quant_rest_in_b, c2w.quant_rest_in_b->ne[0], (size_t) (g - 1) * c2w.quant_rest_in_b->nb[1]);
idx = vq(page(c2w.quant_rest_in_w, g - 1), in_b, page(c2w.quant_rest_cb_norm_w, g - 1), page(c2w.quant_rest_cb_w, g - 1));
out = ggml_add(ctx0, out, ggml_get_rows(ctx0, page(model.gen_code_embd_w, g), idx));
}
out = ggml_scale(ctx0, out, 1.0f / sqrtf((float) (n_cb + 1)));
cb(out, "spk_rows", -1);
ggml_build_forward_expand(gf, out);
return gf;
}