* add fisk-audio-s2 support * add voice cloning * cont merge master * nits * nits * clean up * nits * use new batch api
75 lines
3.4 KiB
C++
75 lines
3.4 KiB
C++
#include "models.h"
|
|
|
|
#include <cmath>
|
|
#include <vector>
|
|
|
|
// one frame: backbone hidden + semantic code -> the codes of this frame + next backbone input
|
|
// fast AR row 0 is the backbone hidden, row p > 0 is the embedding of code p - 1 and picks code p (greedy, as the reference)
|
|
ggml_cgraph * mtmd_graph_fish_s2_gen::build_fast_ar() {
|
|
const int64_t n_cb = model.gen_code_embd_w->ne[2];
|
|
const int64_t n_embd_cb = model.gen_code_embd_w->ne[0];
|
|
const int64_t n_embd = model.gen_code_tok_embd_w->ne[0];
|
|
const int64_t n_kv = model.layers[0].k_w->ne[1];
|
|
const int n_layer = (int) model.layers.size();
|
|
|
|
ggml_tensor * h = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd);
|
|
ggml_set_name(h, "inp_raw");
|
|
ggml_set_input(h);
|
|
|
|
ggml_tensor * inp_code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1);
|
|
ggml_set_name(inp_code0, "inp_code0");
|
|
ggml_set_input(inp_code0);
|
|
|
|
mtmd_graph_qwen3tts_gen::code_gen cg(*this, top_k, top_p);
|
|
|
|
std::vector<ggml_tensor *> k_cache(n_layer), v_cache(n_layer);
|
|
for (int il = 0; il < n_layer; il++) {
|
|
k_cache[il] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, n_cb), 0.0f);
|
|
v_cache[il] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, n_cb), 0.0f);
|
|
}
|
|
|
|
ggml_tensor * out_codes = cg.cache_set(ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, 1, n_cb), 0, inp_code0);
|
|
|
|
for (int p = 0; p < n_cb; p++) {
|
|
ggml_tensor * cur = h;
|
|
if (p > 0) {
|
|
ggml_tensor * code = ggml_view_1d(ctx0, out_codes, 1, (size_t) (p - 1) * out_codes->nb[1]);
|
|
cur = ggml_reshape_1d(ctx0, ggml_get_rows(ctx0, model.gen_code_tok_embd_w, code), n_embd);
|
|
}
|
|
ggml_tensor * kq_mask = cg.causal_mask_row(n_cb, p);
|
|
ggml_tensor * inp_pos = cg.const_i32(k_cache[0], (float) p);
|
|
for (int il = 0; il < n_layer; il++) {
|
|
cur = cg.layer_forward(cur, model.layers[il], inp_pos, kq_mask, k_cache[il], v_cache[il], n_cb, p, il);
|
|
}
|
|
if (p == 0) {
|
|
continue; // only fills the KV
|
|
}
|
|
cur = ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, hparams.eps), model.gen_code_norm_w);
|
|
ggml_tensor * logits = ggml_mul_mat(ctx0, model.gen_code_head_w, cur);
|
|
out_codes = cg.cache_set(out_codes, p, ggml_argmax(ctx0, logits));
|
|
}
|
|
|
|
out_codes = ggml_cont(ctx0, out_codes);
|
|
ggml_set_name(out_codes, "out_codes");
|
|
ggml_set_output(out_codes);
|
|
ggml_build_forward_expand(gf, out_codes);
|
|
|
|
// backbone input of the next row: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1)
|
|
ggml_tensor * out_embd = ggml_get_rows(ctx0, model.gen_code_out_embd_w, inp_code0);
|
|
for (int64_t g = 0; g < n_cb; g++) {
|
|
ggml_tensor * table = ggml_view_2d(ctx0, model.gen_code_embd_w, n_embd_cb, model.gen_code_embd_w->ne[1],
|
|
model.gen_code_embd_w->nb[1], (size_t) g * model.gen_code_embd_w->nb[2]);
|
|
ggml_tensor * code = ggml_view_1d(ctx0, out_codes, 1, (size_t) g * out_codes->nb[1]);
|
|
out_embd = ggml_add(ctx0, out_embd, ggml_get_rows(ctx0, table, code));
|
|
}
|
|
out_embd = ggml_scale(ctx0, out_embd, 1.0f / sqrtf((float) (n_cb + 1)));
|
|
cb(out_embd, "gen_audio_out", -1);
|
|
ggml_build_forward_expand(gf, out_embd);
|
|
|
|
return gf;
|
|
}
|
|
|
|
ggml_cgraph * mtmd_graph_fish_s2_gen::build() {
|
|
return gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV ? build_codec() : build_fast_ar();
|
|
}
|