mtmd: support higgs v3 STT and vibevoice-ASR (#32329)

* mtmd: support higgs v3 and vibevoice-ASR

* use proper mapping

* address review comment

* nits

* rm hacky get_audio_length, add mtmd_helper_get_media_meta
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-11-15 21:25:34 +01:00
1 parent 18b74fa41d
commit fc504f91d8
24 files changed
+1101 -18

No files matched your search

+15 -2
View File
@@ -155,6 +155,7 @@ struct realtime_conversation {
bool started = false; // something went into the slot, the session config is fixed
bool turn_open = false; // a user turn is being filled
bool has_audio = false; // the open turn has audio not committed yet
int64_t n_turn_samples = 0; // audio of the open turn, at the model rate
bool tool_results = false; // tool results went in after the last response, the next one answers them
int n_turns = 0;
std::string next_header; // text that goes in before the next user turn
@@ -376,8 +377,10 @@ struct realtime_conversation {
if (!resample(samples) || samples.empty() || !ensure_turn()) {
return;
}
const int64_t n = (int64_t) samples.size();
if (control(SERVER_CONTROL_ACTION_AUDIO_APPEND, {}, std::move(samples))) {
has_audio = true;
n_turn_samples += n;
}
}
@@ -410,7 +413,8 @@ struct realtime_conversation {
if (!control(SERVER_CONTROL_ACTION_AUDIO_END)) {
return false;
}
has_audio = false;
has_audio = false;
n_turn_samples = 0;
std::lock_guard<std::mutex> lock(mutex);
const json prev = last_item_id.empty() ? json(nullptr) : json(last_item_id);
@@ -424,6 +428,7 @@ struct realtime_conversation {
}
void on_commit() {
const int64_t n_audio = n_turn_samples;
if (!commit() || type != "transcription") {
return;
}
@@ -433,11 +438,19 @@ struct realtime_conversation {
transcribe_item_id = last_item_id;
transcript.clear();
}
// some models (VibeVoice-ASR) take an instruction after the audio, with its length
const auto * tmpls = deps.meta.chat_params.tmpls.get();
common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language);
if (preset.needs_audio_length) {
preset = common_chat_get_asr_prompt(tmpls, language, (double) n_audio / mtmd_get_audio_sample_rate(deps.mctx));
}
const std::string after_audio = preset.media_first ? preset.user : "";
server_control_response response;
response.n_predict = max_output_tokens;
response.out_mode = MTMD_HELPER_GEN_AUDIO_OUT_TEXT;
response.sampling = response_sampling(nullptr);
if (!control(SERVER_CONTROL_ACTION_RESPONSE_CREATE, user_footer + assistant_header, {}, response)) {
if (!control(SERVER_CONTROL_ACTION_RESPONSE_CREATE, after_audio + user_footer + assistant_header, {}, response)) {
return;
}
control(SERVER_CONTROL_ACTION_CLEAR);