mtmd: support higgs v3 STT and vibevoice-ASR (#32329)

* mtmd: support higgs v3 and vibevoice-ASR

* use proper mapping

* address review comment

* nits

* rm hacky get_audio_length, add mtmd_helper_get_media_meta
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-11-15 21:25:34 +01:00
1 parent 18b74fa41d
commit fc504f91d8
24 files changed
+1101 -18

No files matched your search

+18 -1
View File
@@ -827,6 +827,11 @@ static std::string string_lower(std::string s) {
return s;
}
// VibeVoice-ASR: default system prompt of the template written by the converter
static bool is_vibevoice_asr_template(const std::string & src) {
return src.find("transcribes audio input into text output in JSON format") != std::string::npos;
}
// LFM3-Audio: language code -> name
static std::string lfm3_language_name(const std::string & language) {
static const std::map<std::string, std::string> names = {
@@ -847,7 +852,8 @@ static std::string lfm3_asr_system_prompt(const std::string & language) {
return "Perform ASR in " + lfm3_language_name(language) + ".";
}
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) {
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language,
double audio_length, const std::string & context) {
common_chat_prompt_preset asr_preset;
asr_preset.system = "";
asr_preset.user = "Transcribe audio to text";
@@ -861,6 +867,17 @@ common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates
asr_preset.system = lfm3_asr_system_prompt(language);
asr_preset.user = "";
asr_preset.has_language = true;
} else if (is_vibevoice_asr_template(src)) {
// without the audio length the model does not know where the audio ends and loops
const char * keys = "transcribe it with these keys: Start time, End time, Speaker ID, Content";
if (context.empty()) {
asr_preset.user = string_format("\nThis is a %.2f seconds audio, please %s", audio_length, keys);
} else {
asr_preset.user = string_format("\nThis is a %.2f seconds audio, with extra info: %s\n\nPlease %s", audio_length, context.c_str(), keys);
asr_preset.has_context = true;
}
asr_preset.media_first = true;
asr_preset.needs_audio_length = true;
}
}