Files
llama.cpp/tools/mtmd/models/chatterbox-spkenc.cpp
T

157 lines
7.2 KiB
C++

#include "models.h"
// Chatterbox speaker encoder: CAMPPlus x-vector on kaldi fbank features, see s3gen/xvector.py
static constexpr float bn_eps = 1e-5f;
// per-channel batchnorm on x [C, T], w/b are null on the affine=False variant
ggml_tensor * mtmd_graph_chatterbox_spkenc::bn1d(const mtmd_chatterbox::bn & n, ggml_tensor * x) {
ggml_tensor * sd = ggml_sqrt(ctx0, ggml_scale_bias(ctx0, n.var, 1.0f, bn_eps));
if (!n.w) {
return ggml_div(ctx0, ggml_sub(ctx0, x, n.mean), sd);
}
ggml_tensor * a = ggml_div(ctx0, n.w, sd);
ggml_tensor * shift = ggml_sub(ctx0, n.b, ggml_mul(ctx0, n.mean, a));
return ggml_add(ctx0, ggml_mul(ctx0, x, a), shift);
}
// batchnorm on a conv2d activation [W=T, H=F, C, 1], stats on ne2
static ggml_tensor * cbx_bn2d(ggml_context * ctx0, const mtmd_chatterbox::bn & n, ggml_tensor * x) {
const int C = (int) x->ne[2];
ggml_tensor * mean = ggml_reshape_4d(ctx0, n.mean, 1, 1, C, 1);
ggml_tensor * var = ggml_reshape_4d(ctx0, n.var, 1, 1, C, 1);
ggml_tensor * w = ggml_reshape_4d(ctx0, n.w, 1, 1, C, 1);
ggml_tensor * b = ggml_reshape_4d(ctx0, n.b, 1, 1, C, 1);
ggml_tensor * a = ggml_div(ctx0, w, ggml_sqrt(ctx0, ggml_scale_bias(ctx0, var, 1.0f, bn_eps)));
ggml_tensor * shift = ggml_sub(ctx0, b, ggml_mul(ctx0, mean, a));
return ggml_add(ctx0, ggml_mul(ctx0, x, a), shift);
}
ggml_tensor * mtmd_graph_chatterbox_spkenc::bn2d_relu(const mtmd_chatterbox::bn & n, ggml_tensor * x) {
return ggml_relu(ctx0, cbx_bn2d(ctx0, n, x));
}
// fcm residual 2d block, stride on the frequency axis only
ggml_tensor * mtmd_graph_chatterbox_spkenc::res2d(const mtmd_chatterbox::spk_res2d & r, ggml_tensor * x,
int stride) {
ggml_tensor * cur = ggml_conv_2d(ctx0, r.conv1_w, x, 1, stride, 1, 1, 1, 1);
cur = bn2d_relu(r.bn1, cur);
cur = ggml_conv_2d(ctx0, r.conv2_w, cur, 1, 1, 1, 1, 1, 1);
// bn2 without the relu, applied before the residual add
cur = cbx_bn2d(ctx0, r.bn2, cur);
ggml_tensor * res = x;
if (r.shortcut_w) {
res = ggml_conv_2d(ctx0, r.shortcut_w, x, 1, stride, 0, 0, 1, 1);
res = cbx_bn2d(ctx0, r.shortcut_bn, res);
}
return ggml_relu(ctx0, ggml_add(ctx0, cur, res));
}
// cam dense tdnn layer: bottleneck then context-gated conv; x [C_in, T] -> [growth, T]
ggml_tensor * mtmd_graph_chatterbox_spkenc::cam_layer(const mtmd_chatterbox::spk_cam_layer & l, ggml_tensor * x,
int dil, ggml_tensor * segfix) {
ggml_tensor * h = ggml_relu(ctx0, bn1d(l.nl1_bn, x));
h = cbx_conv1d(l.linear1_w, nullptr, h, 1, 0, 0);
h = ggml_relu(ctx0, bn1d(l.nl2_bn, h));
ggml_tensor * k = l.local_w;
const int pad = ((int) k->ne[0] - 1) / 2 * dil;
ggml_tensor * y = cbx_conv1d_dil(k, nullptr, h, pad, dil);
// context: global mean plus ceil-mode segment means of length 100
const int T = (int) h->ne[1];
const int C = (int) h->ne[0];
const int S = (T + 99) / 100;
ggml_tensor * ht = ggml_cont(ctx0, ggml_transpose(ctx0, h)); // [T, C]
ggml_tensor * gmean = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_mean(ctx0, ht))); // [C, 1]
ggml_tensor * seg;
{
ggml_tensor * padded = ht;
if (S * 100 != T) {
ggml_tensor * z = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, S * 100 - T, C);
z = ggml_scale(ctx0, z, 0.0f);
padded = ggml_concat(ctx0, ht, z, 0);
}
ggml_tensor * pooled = ggml_pool_1d(ctx0, padded, GGML_OP_POOL_AVG, 100, 100, 0); // [S, C]
pooled = ggml_mul(ctx0, pooled, segfix);
ggml_tensor * exp = ggml_interpolate(ctx0, pooled, S * 100, C, 1, 1, GGML_SCALE_MODE_NEAREST);
exp = ggml_cont(ctx0, ggml_view_2d(ctx0, exp, T, C, exp->nb[1], 0));
seg = ggml_cont(ctx0, ggml_transpose(ctx0, exp)); // [C, T]
}
ggml_tensor * context = ggml_add(ctx0, seg, gmean);
context = cbx_conv1d(l.ctx1_w, l.ctx1_b, context, 1, 0, 0);
context = ggml_relu(ctx0, context);
context = cbx_conv1d(l.ctx2_w, l.ctx2_b, context, 1, 0, 0);
ggml_tensor * m = ggml_sigmoid(ctx0, context);
return ggml_mul(ctx0, y, m);
}
ggml_cgraph * mtmd_graph_chatterbox_spkenc::build() {
const auto & c = model.cbx;
const int T = img.nx();
const int T1 = (T - 1) / 2 + 1;
const int S = (T1 + 99) / 100;
ggml_tensor * segfix = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, S);
ggml_set_name(segfix, "inp_segfix");
ggml_set_input(segfix);
// fbank features [T, 80] from the preprocessor
ggml_tensor * inp = build_inp_raw(1);
// fcm 2d front: [W=T, H=F=80, C=1] -> [T, 10, 32] -> [320, T]
ggml_tensor * x = ggml_reshape_4d(ctx0, inp, T, 80, 1, 1);
x = ggml_conv_2d(ctx0, c.spk_conv1_w, x, 1, 1, 1, 1, 1, 1);
x = bn2d_relu(c.spk_bn1, x);
x = res2d(c.spk_layer1_0, x, 2);
x = res2d(c.spk_layer1_1, x, 1);
x = res2d(c.spk_layer2_0, x, 2);
x = res2d(c.spk_layer2_1, x, 1);
x = ggml_conv_2d(ctx0, c.spk_conv2_w, x, 1, 2, 1, 1, 1, 1);
x = bn2d_relu(c.spk_bn2, x);
x = ggml_reshape_2d(ctx0, x, T, 320);
x = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [320, T]
cb(x, "spk_fcm", -1);
// tdnn k5 stride 2 over time, then the three cam dense blocks
x = cbx_conv1d(c.spk_tdnn_w, nullptr, x, 2, 2, 2); // [128, T1]
x = ggml_relu(ctx0, bn1d(c.spk_tdnn_bn, x));
cb(x, "spk_tdnn", -1);
static const int block_dil[3] = {1, 2, 2};
const mtmd_chatterbox::spk_cam_block * blocks[3] = { &c.spk_block1, &c.spk_block2, &c.spk_block3 };
for (int bi = 1; bi <= 3; bi++) {
const auto & blk = *blocks[bi - 1];
for (const auto & l : blk.layers) {
ggml_tensor * out = cam_layer(l, x, block_dil[bi - 1], segfix);
x = ggml_concat(ctx0, x, out, 0);
}
x = ggml_relu(ctx0, bn1d(blk.transit_bn, x));
x = cbx_conv1d(blk.transit_w, nullptr, x, 1, 0, 0);
cb(x, "spk_block", bi);
}
x = ggml_relu(ctx0, bn1d(c.spk_out_bn, x)); // [512, T1]
// statistics pooling: mean and unbiased std over time -> [1024, 1]
ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T1, 512]
ggml_tensor * mean = ggml_mean(ctx0, xt); // [1, 512]
ggml_tensor * m2 = ggml_mean(ctx0, ggml_mul(ctx0, xt, xt));
ggml_tensor * var = ggml_sub(ctx0, m2, ggml_mul(ctx0, mean, mean));
var = ggml_scale(ctx0, var, (float) T1 / (float) (T1 - 1));
ggml_tensor * sd = ggml_sqrt(ctx0, ggml_relu(ctx0, var));
ggml_tensor * stats = ggml_concat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, mean)),
ggml_cont(ctx0, ggml_transpose(ctx0, sd)), 0); // [1024, 1]
cb(stats, "spk_stats_pool", -1);
// the x-vector stays raw, the gen graph normalizes it
ggml_tensor * dw = ggml_reshape_2d(ctx0, c.spk_dense_w, 1024, 192);
ggml_tensor * emb = ggml_mul_mat(ctx0, dw, stats); // [192, 1]
emb = bn1d(c.spk_dense_bn, emb);
emb = ggml_cont(ctx0, ggml_reshape_1d(ctx0, emb, 192));
cb(emb, "spk_embd", -1);
ggml_set_name(emb, "out_xvec");
ggml_set_output(emb);
ggml_build_forward_expand(gf, emb);
return gf;
}