#include "models.h" // SnakeBeta, alpha and beta folded at conversion: x + beta * sin(alpha * x)^2 // x: [T, C], alpha/beta: [C] ggml_tensor * mtmd_graph_snake_vae::snake(ggml_tensor * x, const mtmd_snake_vae::snake & s) const { ggml_tensor * a = ggml_reshape_2d(ctx0, s.alpha, 1, s.alpha->ne[0]); ggml_tensor * b = ggml_reshape_2d(ctx0, s.beta, 1, s.beta->ne[0]); // expand reshapes first so mul/sin/sqr/mul/add lands as consecutive nodes, letting backends fuse them ggml_build_forward_expand(gf, a); ggml_build_forward_expand(gf, b); ggml_tensor * y = ggml_sqr(ctx0, ggml_sin(ctx0, ggml_mul(ctx0, x, a))); return ggml_add(ctx0, x, ggml_mul(ctx0, y, b)); } // same-length conv, stride 1; x: [T, IC], w: [K, IC, OC], returns [T, OC] // im2col in F32: the reference decoders run in FP32 ggml_tensor * mtmd_graph_snake_vae::conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const { const int pad = (int) (w->ne[0] - 1) / 2 * dilation; ggml_tensor * col = ggml_im2col(ctx0, w, x, 1, 0, pad, 0, dilation, 0, false, GGML_TYPE_F32); // [1, T, IC * K] ggml_tensor * y = ggml_mul_mat(ctx0, ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[1] * col->ne[2]), ggml_reshape_2d(ctx0, w, w->ne[0] * w->ne[1], w->ne[2])); if (b) { y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0])); } return y; } ggml_tensor * mtmd_graph_snake_vae::build_snake_vae(ggml_tensor * x, const mtmd_snake_vae & vae) const { static constexpr int dilations[3] = { 1, 3, 9 }; if (vae.in_proj_w) { x = conv1d(x, vae.in_proj_w, vae.in_proj_b, 1); } x = conv1d(x, vae.in_w, vae.in_b, 1); for (size_t ib = 0; ib < vae.blocks.size(); ib++) { const auto & blk = vae.blocks[ib]; const int stride = (int) (blk.up_w->ne[0] / 2); const int pad = (stride + 1) / 2; // ceil(stride / 2) of the reference x = snake(x, blk.act); { // ConvTranspose1d as GEMM + col2im, same as qwen3tts; up_w: [K, OC, IC] const int64_t K = blk.up_w->ne[0]; const int64_t OC = blk.up_w->ne[1]; ggml_tensor * w2 = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, blk.up_w, K * OC, blk.up_w->ne[2]))); ggml_tensor * col = ggml_mul_mat(ctx0, w2, ggml_cont(ctx0, ggml_transpose(ctx0, x))); // [K * OC, T] x = ggml_col2im_1d(ctx0, col, stride, (int) OC, pad); } x = ggml_add(ctx0, x, ggml_reshape_2d(ctx0, blk.up_b, 1, blk.up_b->ne[0])); for (size_t ir = 0; ir < blk.res.size(); ir++) { const auto & ru = blk.res[ir]; ggml_tensor * h = snake(x, ru.act1); h = conv1d(h, ru.conv1_w, ru.conv1_b, dilations[ir]); h = snake(h, ru.act2); h = conv1d(h, ru.conv2_w, ru.conv2_b, 1); x = ggml_add(ctx0, x, h); } cb(x, "vae_block", (int) ib); } x = snake(x, vae.post); return conv1d(x, vae.out_w, vae.out_b, 1); }