* add fisk-audio-s2 support * add voice cloning * cont merge master * nits * nits * clean up * nits * use new batch api
136 lines
7.1 KiB
C++
136 lines
7.1 KiB
C++
#include "models.h"
|
|
|
|
#include <cmath>
|
|
#include <string>
|
|
|
|
// reference clip -> codec frames -> backbone input rows, see DAC.encode() and DownsampleResidualVectorQuantize.encode()
|
|
// activations are [C, T]; a conv is a sum of matmuls over tap-shifted views: exact in F32, no im2col buffer
|
|
|
|
// causal conv, w is taps-major [stride * IC, OC, K / stride], see conversion/fish_s2.py
|
|
ggml_tensor * mtmd_graph_fish_s2_spkenc::conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation) const {
|
|
const int64_t n_tap = w->ne[2];
|
|
const int pad = (int) (n_tap - 1) * stride * dilation;
|
|
if (pad > 0) {
|
|
x = ggml_pad_ext(ctx0, x, 0, 0, pad, 0, 0, 0, 0, 0);
|
|
}
|
|
if (stride > 1) {
|
|
// stride frames per column, then each tap group is one matmul
|
|
x = ggml_reshape_2d(ctx0, x, x->ne[0] * stride, x->ne[1] / stride);
|
|
}
|
|
const int64_t n_out = x->ne[1] - (n_tap - 1) * dilation;
|
|
ggml_tensor * y = nullptr;
|
|
for (int64_t k = 0; k < n_tap; k++) {
|
|
ggml_tensor * wk = ggml_view_2d(ctx0, w, w->ne[0], w->ne[1], w->nb[1], (size_t) k * w->nb[2]);
|
|
ggml_tensor * xk = ggml_view_2d(ctx0, x, x->ne[0], n_out, x->nb[1], (size_t) k * dilation * x->nb[1]);
|
|
ggml_tensor * yk = ggml_mul_mat(ctx0, wk, xk);
|
|
y = y ? ggml_add(ctx0, y, yk) : yk;
|
|
}
|
|
return ggml_add(ctx0, y, b);
|
|
}
|
|
|
|
// x + sin(alpha * x)^2 / alpha, beta = 1 / alpha is folded at conversion
|
|
// TODO @ngxson: on Metal, residual codes match the reference less often than on CPU (89-98% vs 99%), maybe the fast-math sin
|
|
ggml_tensor * mtmd_graph_fish_s2_spkenc::snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const {
|
|
ggml_tensor * s = ggml_sqr(ctx0, ggml_sin(ctx0, ggml_mul(ctx0, x, alpha)));
|
|
return ggml_add(ctx0, x, ggml_mul(ctx0, s, beta));
|
|
}
|
|
|
|
// causal depthwise conv (w [C, K]) -> LayerNorm -> pwconv1 -> GELU -> pwconv2 -> layer scale -> residual
|
|
ggml_tensor * mtmd_graph_fish_s2_spkenc::convnext(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk) const {
|
|
const int64_t n_tap = blk.dwconv_w->ne[1];
|
|
const int64_t n_pos = x->ne[1];
|
|
ggml_tensor * xp = ggml_pad_ext(ctx0, x, 0, 0, (int) n_tap - 1, 0, 0, 0, 0, 0);
|
|
ggml_tensor * h = nullptr;
|
|
for (int64_t k = 0; k < n_tap; k++) {
|
|
ggml_tensor * wk = ggml_view_1d(ctx0, blk.dwconv_w, blk.dwconv_w->ne[0], (size_t) k * blk.dwconv_w->nb[1]);
|
|
ggml_tensor * xk = ggml_view_2d(ctx0, xp, xp->ne[0], n_pos, xp->nb[1], (size_t) k * xp->nb[1]);
|
|
ggml_tensor * hk = ggml_mul(ctx0, xk, wk);
|
|
h = h ? ggml_add(ctx0, h, hk) : hk;
|
|
}
|
|
h = ggml_add(ctx0, h, blk.dwconv_b);
|
|
h = ggml_add(ctx0, ggml_mul(ctx0, ggml_norm(ctx0, h, 1e-6f), blk.norm_w), blk.norm_b);
|
|
h = ggml_gelu_erf(ctx0, ggml_add(ctx0, ggml_mul_mat(ctx0, blk.pw1_w, h), blk.pw1_b));
|
|
h = ggml_add(ctx0, ggml_mul_mat(ctx0, blk.pw2_w, h), blk.pw2_b);
|
|
return ggml_add(ctx0, x, ggml_mul(ctx0, h, blk.gamma));
|
|
}
|
|
|
|
ggml_cgraph * mtmd_graph_fish_s2_spkenc::build() {
|
|
const auto & c2w = model.c2w;
|
|
|
|
// the preprocessor hands over the waveform as a single-row "mel", [n_samples, 1] is also [1, n_samples]
|
|
ggml_tensor * inp_raw = build_inp_raw(1);
|
|
ggml_tensor * x = ggml_reshape_2d(ctx0, inp_raw, 1, inp_raw->ne[0]);
|
|
|
|
// windowed causal transformer on a cold start: zero state, positions from 0
|
|
mtmd_graph_qwen3tts_gen::code2wav dec(*this);
|
|
auto zero_state = [&](const std::string & name, int64_t n0, int64_t n1) {
|
|
dec.state_in[name] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n0, n1), 0.0f);
|
|
};
|
|
auto transformer = [&](ggml_tensor * cur, int il0, int il1, int64_t window, ggml_tensor * norm_w) {
|
|
const int64_t n_kv = model.layers[il0].k_w->ne[1];
|
|
zero_state("tfm_pos", 1, 1);
|
|
for (int il = il0; il < il1; il++) {
|
|
zero_state("tfm_k_" + std::to_string(il), n_kv, window - 1);
|
|
zero_state("tfm_v_" + std::to_string(il), n_kv, window - 1);
|
|
cur = dec.tfm_layer_forward(cur, model.layers[il], il, window);
|
|
}
|
|
return ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, hparams.wav_tfm_eps), norm_w);
|
|
};
|
|
|
|
// encoder: entry conv, 4x (3 residual units, Snake, strided conv), transformer, Snake, post conv
|
|
x = conv(x, c2w.dac_entry_w, c2w.dac_entry_b, 1, 1);
|
|
static constexpr int DAC_DILATIONS[3] = { 1, 3, 9 };
|
|
for (const auto & blk : c2w.dac) {
|
|
for (size_t ir = 0; ir < blk.res.size(); ir++) {
|
|
const auto & res = blk.res[ir];
|
|
ggml_tensor * h = snake(x, res.act1_alpha, res.act1_beta);
|
|
h = conv(h, res.conv1_w, res.conv1_b, 1, DAC_DILATIONS[ir]);
|
|
h = snake(h, res.act2_alpha, res.act2_beta);
|
|
h = conv(h, res.conv2_w, res.conv2_b, 1, 1);
|
|
x = ggml_add(ctx0, x, h);
|
|
}
|
|
x = snake(x, blk.snake_alpha, blk.snake_beta);
|
|
x = conv(x, blk.conv_w, blk.conv_b, (int) (blk.conv_w->ne[0] / x->ne[0]), 1);
|
|
}
|
|
x = transformer(x, 0, n_enc_tfm_layer, 512, model.post_ln_w);
|
|
x = snake(x, c2w.dac_post_snake_alpha, c2w.dac_post_snake_beta);
|
|
x = conv(x, c2w.dac_post_conv_w, c2w.dac_post_conv_b, 1, 1);
|
|
cb(x, "codec_enc", -1);
|
|
|
|
// quantizer: 2x (strided conv + ConvNeXt), transformer
|
|
for (const auto & down : c2w.upsample) {
|
|
x = conv(x, down.conv_w, down.conv_b, (int) (down.conv_w->ne[0] / x->ne[0]), 1);
|
|
x = convnext(x, down);
|
|
}
|
|
ggml_tensor * z = transformer(x, n_enc_tfm_layer, (int) model.layers.size(), 128, c2w.tfm_output_norm_w);
|
|
cb(z, "codec_latent", -1);
|
|
|
|
// nearest entry by cosine similarity, then the entry through out_proj is removed from the residual
|
|
auto vq = [&](ggml_tensor * in_w, ggml_tensor * in_b, ggml_tensor * cb_norm, ggml_tensor * table) {
|
|
ggml_tensor * e = ggml_add(ctx0, ggml_mul_mat(ctx0, in_w, z), in_b);
|
|
e = ggml_l2_norm(ctx0, e, 1e-12f);
|
|
ggml_tensor * idx = ggml_argmax(ctx0, ggml_mul_mat(ctx0, cb_norm, e));
|
|
z = ggml_sub(ctx0, z, ggml_get_rows(ctx0, table, idx));
|
|
return idx;
|
|
};
|
|
auto page = [&](ggml_tensor * t, int64_t g) {
|
|
return ggml_view_2d(ctx0, t, t->ne[0], t->ne[1], t->nb[1], (size_t) g * t->nb[2]);
|
|
};
|
|
|
|
// backbone row: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1), same as fish_s2_gen
|
|
const int64_t n_cb = model.gen_code_embd_w->ne[2];
|
|
ggml_tensor * idx = vq(c2w.quant_first_in_w, c2w.quant_first_in_b, c2w.quant_first_cb_norm_w, c2w.quant_first_cb_w);
|
|
ggml_tensor * out = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, idx),
|
|
ggml_get_rows(ctx0, page(model.gen_code_embd_w, 0), idx));
|
|
for (int64_t g = 1; g < n_cb; g++) {
|
|
ggml_tensor * in_b = ggml_view_1d(ctx0, c2w.quant_rest_in_b, c2w.quant_rest_in_b->ne[0], (size_t) (g - 1) * c2w.quant_rest_in_b->nb[1]);
|
|
idx = vq(page(c2w.quant_rest_in_w, g - 1), in_b, page(c2w.quant_rest_cb_norm_w, g - 1), page(c2w.quant_rest_cb_w, g - 1));
|
|
out = ggml_add(ctx0, out, ggml_get_rows(ctx0, page(model.gen_code_embd_w, g), idx));
|
|
}
|
|
out = ggml_scale(ctx0, out, 1.0f / sqrtf((float) (n_cb + 1)));
|
|
cb(out, "spk_rows", -1);
|
|
ggml_build_forward_expand(gf, out);
|
|
|
|
return gf;
|
|
}
|