mtmd: drama3 voice cloning (#32586)

* mtmd: drama3 voice cloning

The Fish S2 codec encoder (fish_s2_spkenc) now also serves drama3:
- conversion: drama3 writes the codec encoder and the audio encoder KV, shared with fish_s2
- fish_s2_spkenc: outputs the codes of the clip (gen-side reference) and, in a drama3_gen mmproj,
  builds the drama3 backbone rows (E_lm[semantic] + E_cb[1][code 1]) / sqrt(2)
- drama3_gen tower: teacher forcing when the caller gives the codes of the next frame
- drama3 pipeline: reference-bracket-v1 prompt with one reference (vLLM-Omni fish_omni), the
  reference frames go through the backbone as codec rows and through the tower with their codes
- mtmd_gen_state_get_ref_codes(): read-only access to the reference codes of a gen state

Claude-Session: https://claude.ai/code/session_01PpX3DmaxNSmJbxGaxnUBrX

* fix correctness

* clean up

* rm redundant code

* fish_s2: rm unused import

---------

Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
This commit is contained in:
FeitengandXuan Son Nguyen authored and GitHub committed 2026-11-20 18:34:52 +01:00
1 parent 6f5c9ad217
commit 34ff2e047c
12 files changed
+414 -193

No files matched your search

+45 -3
View File
@@ -2617,8 +2617,9 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im
static int32_t cbx_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd);
static int32_t voxcpm2_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd);
static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector<float> & out_embd) {
mtmd_image_f32_batch batch = batch_f32.clone();
// copy of batch_f32, the mel is computed from source when there is one
static bool mtmd_audio_batch_prepare(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, mtmd_image_f32_batch & batch) {
batch = batch_f32.clone();
if (source) {
if (!ctx->audio_preproc->compute(*source)) {
LOG_ERR("%s: unable to compute the mel\n", __func__);
@@ -2627,7 +2628,38 @@ static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_bat
GGML_ASSERT(idx < source->entries.size());
batch.entries[0] = source->entries[idx];
}
return ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd);
return true;
}
static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector<float> & out_embd) {
mtmd_image_f32_batch batch;
return mtmd_audio_batch_prepare(ctx, batch_f32, source, idx, batch) && ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd);
}
// fish_s2_spkenc: one backbone row per reference frame, the codes go to ctx->gen_ref, see mtmd_gen_state_set_ref()
static int32_t mtmd_fish_s2_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd) {
mtmd_image_f32_batch batch;
if (!mtmd_audio_batch_prepare(ctx, audio_tokens->batch_f32, audio_tokens->source.get(), audio_tokens->source_idx, batch)) {
return 1;
}
std::vector<int32_t> codes;
out_embd.resize((size_t) audio_tokens->n_tokens * ctx->n_embd_out());
mtmd_subctx_encode_params params;
params.imgs = &batch;
params.n_threads = ctx->n_threads;
params.out_embd = &out_embd;
params.out_codes = &codes;
if (!ctx->ctx_a->encode(&params)) {
return 1;
}
const size_t n_cb = ctx->ctx_a->model.gen_code_embd_w->ne[2];
if (codes.size() != (size_t) audio_tokens->n_tokens * n_cb) {
LOG_ERR("%s: %zu reference codes for %u frames\n", __func__, codes.size(), audio_tokens->n_tokens);
return 1;
}
ctx->gen_ref.tokens = std::move(codes);
return 0;
}
// qwen3tts: the speaker row, then one talker input row per reference frame
@@ -2732,6 +2764,9 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk
if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_VOXCPM2_SPKENC) {
return voxcpm2_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
}
if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_FISH_S2_SPKENC) {
return mtmd_fish_s2_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
}
if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_QWEN3TTS_SPKENC && chunk->tokens_audio->batch_f32.entries.size() == 2) {
return mtmd_qwen3tts_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
}
@@ -4214,6 +4249,13 @@ int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) {
return 0;
}
const int32_t * mtmd_gen_state_get_ref_codes(const mtmd_gen_state * state, size_t * n_codes) {
if (n_codes) {
*n_codes = state ? state->ref.tokens.size() : 0;
}
return state && !state->ref.tokens.empty() ? state->ref.tokens.data() : nullptr;
}
static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
mtmd_subcontext * sctx = ctx->ctx_gen_a.get();
if (!sctx) {