mtmd: drama3 voice cloning (#32586)
* mtmd: drama3 voice cloning The Fish S2 codec encoder (fish_s2_spkenc) now also serves drama3: - conversion: drama3 writes the codec encoder and the audio encoder KV, shared with fish_s2 - fish_s2_spkenc: outputs the codes of the clip (gen-side reference) and, in a drama3_gen mmproj, builds the drama3 backbone rows (E_lm[semantic] + E_cb[1][code 1]) / sqrt(2) - drama3_gen tower: teacher forcing when the caller gives the codes of the next frame - drama3 pipeline: reference-bracket-v1 prompt with one reference (vLLM-Omni fish_omni), the reference frames go through the backbone as codec rows and through the tower with their codes - mtmd_gen_state_get_ref_codes(): read-only access to the reference codes of a gen state Claude-Session: https://claude.ai/code/session_01PpX3DmaxNSmJbxGaxnUBrX * fix correctness * clean up * rm redundant code * fish_s2: rm unused import --------- Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
This commit is contained in:
1 parent
6f5c9ad217
commit
34ff2e047c
12 files changed
+414
-193
No files matched your search
+45
-3
@@ -2617,8 +2617,9 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im
|
||||
static int32_t cbx_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd);
|
||||
static int32_t voxcpm2_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd);
|
||||
|
||||
static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector<float> & out_embd) {
|
||||
mtmd_image_f32_batch batch = batch_f32.clone();
|
||||
// copy of batch_f32, the mel is computed from source when there is one
|
||||
static bool mtmd_audio_batch_prepare(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, mtmd_image_f32_batch & batch) {
|
||||
batch = batch_f32.clone();
|
||||
if (source) {
|
||||
if (!ctx->audio_preproc->compute(*source)) {
|
||||
LOG_ERR("%s: unable to compute the mel\n", __func__);
|
||||
@@ -2627,7 +2628,38 @@ static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_bat
|
||||
GGML_ASSERT(idx < source->entries.size());
|
||||
batch.entries[0] = source->entries[idx];
|
||||
}
|
||||
return ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector<float> & out_embd) {
|
||||
mtmd_image_f32_batch batch;
|
||||
return mtmd_audio_batch_prepare(ctx, batch_f32, source, idx, batch) && ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd);
|
||||
}
|
||||
|
||||
// fish_s2_spkenc: one backbone row per reference frame, the codes go to ctx->gen_ref, see mtmd_gen_state_set_ref()
|
||||
static int32_t mtmd_fish_s2_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd) {
|
||||
mtmd_image_f32_batch batch;
|
||||
if (!mtmd_audio_batch_prepare(ctx, audio_tokens->batch_f32, audio_tokens->source.get(), audio_tokens->source_idx, batch)) {
|
||||
return 1;
|
||||
}
|
||||
std::vector<int32_t> codes;
|
||||
out_embd.resize((size_t) audio_tokens->n_tokens * ctx->n_embd_out());
|
||||
|
||||
mtmd_subctx_encode_params params;
|
||||
params.imgs = &batch;
|
||||
params.n_threads = ctx->n_threads;
|
||||
params.out_embd = &out_embd;
|
||||
params.out_codes = &codes;
|
||||
if (!ctx->ctx_a->encode(¶ms)) {
|
||||
return 1;
|
||||
}
|
||||
const size_t n_cb = ctx->ctx_a->model.gen_code_embd_w->ne[2];
|
||||
if (codes.size() != (size_t) audio_tokens->n_tokens * n_cb) {
|
||||
LOG_ERR("%s: %zu reference codes for %u frames\n", __func__, codes.size(), audio_tokens->n_tokens);
|
||||
return 1;
|
||||
}
|
||||
ctx->gen_ref.tokens = std::move(codes);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// qwen3tts: the speaker row, then one talker input row per reference frame
|
||||
@@ -2732,6 +2764,9 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk
|
||||
if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_VOXCPM2_SPKENC) {
|
||||
return voxcpm2_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
|
||||
}
|
||||
if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_FISH_S2_SPKENC) {
|
||||
return mtmd_fish_s2_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
|
||||
}
|
||||
if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_QWEN3TTS_SPKENC && chunk->tokens_audio->batch_f32.entries.size() == 2) {
|
||||
return mtmd_qwen3tts_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
|
||||
}
|
||||
@@ -4214,6 +4249,13 @@ int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const int32_t * mtmd_gen_state_get_ref_codes(const mtmd_gen_state * state, size_t * n_codes) {
|
||||
if (n_codes) {
|
||||
*n_codes = state ? state->ref.tokens.size() : 0;
|
||||
}
|
||||
return state && !state->ref.tokens.empty() ? state->ref.tokens.data() : nullptr;
|
||||
}
|
||||
|
||||
static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
|
||||
mtmd_subcontext * sctx = ctx->ctx_gen_a.get();
|
||||
if (!sctx) {
|
||||
|
||||
Reference in new issue
Block a user