60 lines
2.8 KiB
C++
60 lines
2.8 KiB
C++
#include "models.h"
|
|
|
|
ggml_tensor * mtmd_graph_voxcpm2_spkenc::conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation, int pad, bool depthwise) const {
|
|
if (pad > 0) {
|
|
x = ggml_pad_ext(ctx0, x, pad, 0, 0, 0, 0, 0, 0, 0);
|
|
}
|
|
// im2col in F32: the reference AudioVAE runs in FP32
|
|
ggml_tensor * y = nullptr;
|
|
if (depthwise) {
|
|
ggml_tensor * col = ggml_im2col(ctx0, w, ggml_reshape_4d(ctx0, x, x->ne[0], 1, x->ne[1], 1), stride, 0, 0, 0, dilation, 0, false, GGML_TYPE_F32);
|
|
y = ggml_mul_mat(ctx0, col, w); // [T, 1, C]
|
|
y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[2]);
|
|
} else {
|
|
ggml_tensor * col = ggml_im2col(ctx0, w, x, stride, 0, 0, 0, dilation, 0, false, GGML_TYPE_F32); // [IC * K, T]
|
|
y = ggml_mul_mat(ctx0,
|
|
ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[1] * col->ne[2]),
|
|
ggml_reshape_2d(ctx0, w, w->ne[0] * w->ne[1], w->ne[2])); // [T, OC]
|
|
}
|
|
if (b) {
|
|
y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
|
|
}
|
|
return y;
|
|
}
|
|
|
|
// causal AudioVAE encoder: waveform -> mu of the latents, 4 latents per output row
|
|
ggml_cgraph * mtmd_graph_voxcpm2_spkenc::build() {
|
|
using g = mtmd_graph_voxcpm2_gen;
|
|
const auto & m = model.voxcpm2;
|
|
mtmd_graph_qwen3tts_gen::code2wav act(*this);
|
|
|
|
// the preprocessor hands over the waveform as a single-row "mel"
|
|
ggml_tensor * inp_raw = build_inp_raw(1);
|
|
ggml_tensor * x = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], 1); // [T, 1]
|
|
|
|
x = conv(x, m.enc_conv_in_w, m.enc_conv_in_b, 1, 1, (int) m.enc_conv_in_w->ne[0] - 1, false);
|
|
for (size_t ib = 0; ib < m.enc.size(); ib++) {
|
|
const auto & blk = m.enc[ib];
|
|
for (size_t ir = 0; ir < blk.res.size(); ir++) {
|
|
const auto & ru = blk.res[ir];
|
|
const int dil = g::res_dilations[ir];
|
|
ggml_tensor * h = act.snake(x, ru.act1_alpha, ru.act1_beta);
|
|
h = conv(h, ru.conv1_w, ru.conv1_b, 1, dil, (int) (ru.conv1_w->ne[0] - 1) * dil, true);
|
|
h = act.snake(h, ru.act2_alpha, ru.act2_beta);
|
|
h = conv(h, ru.conv2_w, ru.conv2_b, 1, 1, 0, false);
|
|
x = ggml_add(ctx0, x, h);
|
|
}
|
|
x = act.snake(x, blk.snake_alpha, blk.snake_beta);
|
|
// kernel 2 * stride, the causal padding is one stride
|
|
const int stride = (int) (blk.conv_w->ne[0] / 2);
|
|
x = conv(x, blk.conv_w, blk.conv_b, stride, 1, stride, false);
|
|
cb(x, "vae_enc_block", (int) ib);
|
|
}
|
|
x = conv(x, m.enc_mu_w, m.enc_mu_b, 1, 1, (int) m.enc_mu_w->ne[0] - 1, false); // [T / hop, n_latent]
|
|
|
|
x = ggml_cont(ctx0, ggml_transpose(ctx0, x));
|
|
x = ggml_reshape_2d(ctx0, x, g::n_latent * g::n_patch, x->ne[1] / g::n_patch);
|
|
ggml_build_forward_expand(gf, x);
|
|
return gf;
|
|
}
|