From fc504f91d8cfdfcd9b0fedcf14600fd2b0e63420 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Sun, 15 Nov 2026 21:25:34 +0100 Subject: [PATCH] mtmd: support higgs v3 STT and vibevoice-ASR (#32329) * mtmd: support higgs v3 and vibevoice-ASR * use proper mapping * address review comment * nits * rm hacky get_audio_length, add mtmd_helper_get_media_meta --- common/chat.cpp | 19 +- common/chat.h | 10 +- conversion/__init__.py | 4 + conversion/base.py | 2 +- conversion/higgs_audio.py | 74 ++++++++ conversion/vibevoice_asr.py | 157 ++++++++++++++++ gguf-py/gguf/constants.py | 29 +++ gguf-py/gguf/tensor_mapping.py | 2 + tools/mtmd/CMakeLists.txt | 1 + tools/mtmd/models/models.h | 15 ++ tools/mtmd/models/vibevoice-asr.cpp | 138 ++++++++++++++ tools/mtmd/models/whisper-enc.cpp | 15 +- tools/mtmd/mtmd-audio.cpp | 78 +++++++- tools/mtmd/mtmd-audio.h | 18 ++ tools/mtmd/mtmd-graph.cpp | 5 + tools/mtmd/mtmd-helper.cpp | 49 +++++ tools/mtmd/mtmd-helper.h | 26 +++ tools/mtmd/mtmd-impl.h | 17 ++ tools/mtmd/mtmd-model.cpp | 122 ++++++++++++ tools/mtmd/mtmd-model.h | 21 ++- tools/mtmd/mtmd-subcontext.cpp | 10 +- tools/mtmd/mtmd.cpp | 276 +++++++++++++++++++++++++++- tools/server/server-chat.cpp | 14 +- tools/server/server-realtime.cpp | 17 +- 24 files changed, 1101 insertions(+), 18 deletions(-) create mode 100644 conversion/higgs_audio.py create mode 100644 conversion/vibevoice_asr.py create mode 100644 tools/mtmd/models/vibevoice-asr.cpp diff --git a/common/chat.cpp b/common/chat.cpp index e060362d2..b3c930ec4 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -827,6 +827,11 @@ static std::string string_lower(std::string s) { return s; } +// VibeVoice-ASR: default system prompt of the template written by the converter +static bool is_vibevoice_asr_template(const std::string & src) { + return src.find("transcribes audio input into text output in JSON format") != std::string::npos; +} + // LFM3-Audio: language code -> name static std::string lfm3_language_name(const std::string & language) { static const std::map names = { @@ -847,7 +852,8 @@ static std::string lfm3_asr_system_prompt(const std::string & language) { return "Perform ASR in " + lfm3_language_name(language) + "."; } -common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) { +common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language, + double audio_length, const std::string & context) { common_chat_prompt_preset asr_preset; asr_preset.system = ""; asr_preset.user = "Transcribe audio to text"; @@ -861,6 +867,17 @@ common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates asr_preset.system = lfm3_asr_system_prompt(language); asr_preset.user = ""; asr_preset.has_language = true; + } else if (is_vibevoice_asr_template(src)) { + // without the audio length the model does not know where the audio ends and loops + const char * keys = "transcribe it with these keys: Start time, End time, Speaker ID, Content"; + if (context.empty()) { + asr_preset.user = string_format("\nThis is a %.2f seconds audio, please %s", audio_length, keys); + } else { + asr_preset.user = string_format("\nThis is a %.2f seconds audio, with extra info: %s\n\nPlease %s", audio_length, context.c_str(), keys); + asr_preset.has_context = true; + } + asr_preset.media_first = true; + asr_preset.needs_audio_length = true; } } diff --git a/common/chat.h b/common/chat.h index 304ccf9ec..d66afaef7 100644 --- a/common/chat.h +++ b/common/chat.h @@ -491,10 +491,16 @@ std::optional common_chat_try_specialized_template( struct common_chat_prompt_preset { std::string system; std::string user; - bool has_language = false; // the preset already names the requested language + bool has_language = false; // the preset already names the requested language + bool has_context = false; // the user text already holds the context + bool media_first = false; // the media goes before the user text + bool needs_audio_length = false; // the user text depends on the audio length, call again with it }; -common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = ""); +// audio_length: in seconds, 0 if unknown +// context: extra info from the user (e.g. names, hotwords), only used by some presets, see has_context +common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "", + double audio_length = 0.0, const std::string & context = ""); // prompt of the model's "tts" chat template for the text to speak, empty if the model has none // language and voice are optional, the template defines their defaults diff --git a/conversion/__init__.py b/conversion/__init__.py index e915753a8..2ec0551e8 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -138,6 +138,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "Grok1ForCausalLM": "grok", "GrokForCausalLM": "grok", "GroveMoeForCausalLM": "grovemoe", + "HiggsAudio3Model": "higgs_audio", "HunYuanDenseV1ForCausalLM": "hunyuan", "HunYuanMoEV1ForCausalLM": "hunyuan", "HunYuanVLForConditionalGeneration": "hunyuan", @@ -325,6 +326,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "UMT5Model": "t5", "UltravoxModel": "ultravox", "UnlimitedOCRForCausalLM": "deepseek", + "VibeVoiceAsrForConditionalGeneration": "vibevoice_asr", "VLlama3ForCausalLM": "llama", "VllmDecision2Model": "vllm_decision2", "VoxtralForConditionalGeneration": "llama", @@ -372,6 +374,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "Granite4VisionForConditionalGeneration": "granite", "GraniteSpeechForConditionalGeneration": "granite", "GraniteSpeechPlusForConditionalGeneration": "granite", + "HiggsAudio3Model": "higgs_audio", "HunYuanVLForConditionalGeneration": "hunyuan", "Idefics3ForConditionalGeneration": "smolvlm", "Inkling2ForConditionalGeneration": "inkling2", @@ -431,6 +434,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "Step3p7ForConditionalGeneration": "step3", "UltravoxModel": "ultravox", "UnlimitedOCRForCausalLM": "deepseek", + "VibeVoiceAsrForConditionalGeneration": "vibevoice_asr", "VoxtralForConditionalGeneration": "ultravox", "VoxtralRealtimeForConditionalGeneration": "ultravox", "YoutuVLForConditionalGeneration": "youtuvl", diff --git a/conversion/base.py b/conversion/base.py index 7558fdcb4..2abd15c91 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -3475,7 +3475,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st # Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older # Kimi-Linear-48B architecture and cannot load K3 (no attention residuals, # latent MoE, situ, ...). Route on the top-level architecture instead. - if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK31ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration"): + if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK31ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration", "HiggsAudio3Model"): return arch # if "architectures" is found in the sub-config, use that instead diff --git a/conversion/higgs_audio.py b/conversion/higgs_audio.py new file mode 100644 index 000000000..9e4d8b2b8 --- /dev/null +++ b/conversion/higgs_audio.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from typing import Any, Callable, TYPE_CHECKING + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, gguf +from .qwen import Qwen3Model +from .ultravox import WhisperEncoderModel + + +# plain ChatML as built by transcribe.py upstream, the Qwen3 template renders past assistant turns differently from the last one +HIGGS_AUDIO_CHAT_TEMPLATE = ( + "{%- for message in messages -%}" + "{{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' -}}" + "{%- endfor -%}" + "{%- if add_generation_prompt -%}" + "{{- '<|im_start|>assistant\\n' -}}" + "{%- if enable_thinking is defined and enable_thinking is false -%}" + "{{- '\\n\\n\\n\\n' -}}" + "{%- endif -%}" + "{%- endif -%}" +) + + +@ModelBase.register("HiggsAudio3Model") +@ModelBase.example("bosonai/higgs-audio-v3-stt") +class HiggsAudioModel(Qwen3Model): + model_arch = gguf.MODEL_ARCH.QWEN3 + + def set_vocab(self): + super().set_vocab() + self.gguf_writer.add_chat_template(HIGGS_AUDIO_CHAT_TEMPLATE) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + + # the audio_* tensors are dropped by TextModel, rename the text head before that + if name == "audio_decoder_proj.text_lm_head.weight": + name = "lm_head.weight" + else: + name = "model." + name + + return super().filter_tensors((name, gen)) + + +@ModelBase.register("HiggsAudio3Model") +class HiggsAudioWhisperEncoderModel(WhisperEncoderModel): + has_vision_encoder = False + has_audio_encoder = True + + def get_audio_config(self) -> dict[str, Any] | None: + return self.global_config["audio_encoder_config"] + + def tensor_force_quant(self, name, new_name, bid, n_dims): + # depthwise conv kernel, same as the whisper convs + if name == "audio_encoder_proj.temporal.weight": + return gguf.GGMLQuantizationType.F16 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + def set_gguf_parameters(self): + assert self.global_config["projector_type"] == "mlp" and self.global_config["projector_temporal_downsample"] == 2 + assert self.global_config["audio_adapter_type"] == "stack" and not self.global_config.get("duplicate_audio_triplet", False) + super().set_gguf_parameters() + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.HIGGS_AUDIO) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, _ = item + if not name.startswith(("audio_tower.", "audio_encoder_proj.")): + return None + return super().filter_tensors(item) diff --git a/conversion/vibevoice_asr.py b/conversion/vibevoice_asr.py new file mode 100644 index 000000000..f160c0e01 --- /dev/null +++ b/conversion/vibevoice_asr.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +from typing import Any, Callable, Iterable, TYPE_CHECKING + +if TYPE_CHECKING: + from torch import Tensor + +from .base import MmprojModel, ModelBase, gguf +from .qwen import Qwen2Model + + +# the upstream chat_template.jinja is a processor template (list content, audio parts), use plain ChatML with its default system prompt +VIBEVOICE_ASR_CHAT_TEMPLATE = ( + "{%- if messages[0]['role'] == 'system' -%}" + "{{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' -}}" + "{%- set loop_messages = messages[1:] -%}" + "{%- else -%}" + "{{- '<|im_start|>system\\nYou are a helpful assistant that transcribes audio input into text output in JSON format.<|im_end|>\\n' -}}" + "{%- set loop_messages = messages -%}" + "{%- endif -%}" + "{%- for message in loop_messages -%}" + "{{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' -}}" + "{%- endfor -%}" + "{%- if add_generation_prompt -%}" + "{{- '<|im_start|>assistant\\n' -}}" + "{%- endif -%}" +) + + +@ModelBase.register("VibeVoiceAsrForConditionalGeneration") +@ModelBase.example("microsoft/VibeVoice-ASR-HF") +class VibeVoiceAsrModel(Qwen2Model): + model_arch = gguf.MODEL_ARCH.QWEN2 + + def set_vocab(self): + super().set_vocab() + self.gguf_writer.add_chat_template(VIBEVOICE_ASR_CHAT_TEMPLATE) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, _ = item + if not name.startswith("language_model."): + return None + return super().filter_tensors(item) + + +@ModelBase.register("VibeVoiceAsrForConditionalGeneration") +class VibeVoiceAsrAudioModel(MmprojModel): + has_vision_encoder = False + has_audio_encoder = True + + # values hard-coded in tools/mtmd/models/vibevoice-asr.cpp + _encoder_consts = {"channels": 1, "kernel_size": 7, "num_filters": 32, "ffn_expansion": 4, "hidden_act": "gelu", "rms_norm_eps": 1e-5} + _encoders = ("acoustic_tokenizer_encoder", "semantic_tokenizer_encoder") + + def get_audio_config(self) -> dict[str, Any] | None: + c = self.global_config["acoustic_tokenizer_encoder_config"] + return { + **c, + "intermediate_size": c["ffn_expansion"] * c["hidden_size"], + "num_attention_heads": 1, + "num_hidden_layers": sum(c["depths"]), + } + + def set_gguf_parameters(self): + super().set_gguf_parameters() + for enc in self._encoders: + c = self.global_config[f"{enc}_config"] + for k, v in self._encoder_consts.items(): + assert c[k] == v, f"{enc}.{k} = {c[k]}, expected {v}" + assert c["depths"] == self.hparams["depths"] and c["downsampling_ratios"] == self.hparams["downsampling_ratios"] + + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.VIBEVOICE_ASR) + # the input is the raw waveform, one channel takes the place of the mel bins + self.gguf_writer.add_audio_num_mel_bins(1) + self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams["rms_norm_eps"]) + self.gguf_writer.add_audio_conv_kernel_size(self.hparams["kernel_size"]) + # stride of each stage, the stem has none + self.gguf_writer.add_audio_patch_sizes([1] + [int(r) for r in self.hparams["downsampling_ratios"]]) + self.gguf_writer.add_audio_depths([int(d) for d in self.hparams["depths"]]) + + def tensor_force_quant(self, name, new_name, bid, n_dims): + if ".dw." in new_name: + return gguf.GGMLQuantizationType.F32 # ggml_ssm_conv + if ".conv." in new_name and new_name.endswith(".weight"): + return gguf.GGMLQuantizationType.F16 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, _ = item + if not name.startswith(cls._encoders + ("multi_modal_projector.",)): + return None + return super().filter_tensors(item) + + _block_tensors = { + "norm.weight": gguf.MODEL_TENSOR.A_ENC_CNX_NORM, + "mixer.conv.weight": gguf.MODEL_TENSOR.A_ENC_CNX_DW, + "mixer.conv.bias": gguf.MODEL_TENSOR.A_ENC_CNX_DW, + "gamma": gguf.MODEL_TENSOR.A_ENC_CNX_LS1, + "ffn_norm.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_NORM, + "ffn.linear1.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_UP, + "ffn.linear1.bias": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_UP, + "ffn.linear2.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_DOWN, + "ffn.linear2.bias": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_DOWN, + "ffn_gamma": gguf.MODEL_TENSOR.A_ENC_CNX_LS2, + } + + # mm.a.mlp.{0,1} + mm.a.mlp_norm.0 for the acoustic connector, {2,3} + 2 for the semantic one + _proj_tensors = { + "acoustic_linear_1": (gguf.MODEL_TENSOR.A_MMPROJ, 0), + "acoustic_norm": (gguf.MODEL_TENSOR.A_MMPROJ_NORM, 0), + "acoustic_linear_2": (gguf.MODEL_TENSOR.A_MMPROJ, 1), + "semantic_linear_1": (gguf.MODEL_TENSOR.A_MMPROJ, 2), + "semantic_norm": (gguf.MODEL_TENSOR.A_MMPROJ_NORM, 2), + "semantic_linear_2": (gguf.MODEL_TENSOR.A_MMPROJ, 3), + } + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # the layer scales (gamma) have no suffix upstream + suffix = "." + name.rsplit(".", 1)[-1] if name.endswith((".weight", ".bias")) else ".weight" + + if name.startswith("multi_modal_projector."): + key, idx = self._proj_tensors[name.split(".")[1]] + yield (self.format_tensor_name(key, idx, suffix=suffix), data_torch) + return + + enc, _, rest = name.partition(".") + enc_id = self._encoders.index(enc) + parts = rest.split(".") + depths = [int(d) for d in self.hparams["depths"]] + + # causal convs of the stages: 0 = stem, 1..6 = downsample, 7 = head + if rest.startswith("stem.conv."): + conv_id = 0 + elif parts[0] == "conv_layers" and parts[2] == "conv": + conv_id = int(parts[1]) + 1 + elif parts[0] == "head": + conv_id = len(depths) + else: + conv_id = None + if conv_id is not None: + yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_ENC_CNX_CONV].format(bid=enc_id, xid=conv_id) + suffix, data_torch) + return + + # ConvNeXt blocks are numbered per stage upstream and flat here + if parts[0] == "stem": + stage, blk, tail = 0, int(parts[2]), ".".join(parts[3:]) + else: + assert parts[0] == "conv_layers" and parts[2] == "stage", name + stage, blk, tail = int(parts[1]) + 1, int(parts[3]), ".".join(parts[4:]) + xid = sum(depths[:stage]) + blk + + key = self._block_tensors[tail] + if key == gguf.MODEL_TENSOR.A_ENC_CNX_DW and tail.endswith(".weight"): + data_torch = data_torch.squeeze(1) # [C, 1, K] -> [C, K] for ggml_ssm_conv + yield (gguf.TENSOR_NAMES[key].format(bid=enc_id, xid=xid) + suffix, data_torch) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index c9b27a11f..037e1332a 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -1386,6 +1386,14 @@ class MODEL_TENSOR(IntEnum): A_ENC_SEANET_RES_CONV1 = auto() # residual unit, dilated conv A_ENC_SEANET_RES_CONV2 = auto() # residual unit, pointwise conv A_ENC_SEANET_SCALE_CONV = auto() # strided downsample conv + A_ENC_CNX_CONV = auto() # vibevoice: causal conv of a stage (stem, downsample, head), bid = encoder + A_ENC_CNX_NORM = auto() # vibevoice: ConvNeXt block, bid = encoder, xid = block + A_ENC_CNX_DW = auto() + A_ENC_CNX_LS1 = auto() + A_ENC_CNX_FFN_NORM = auto() + A_ENC_CNX_FFN_UP = auto() + A_ENC_CNX_FFN_DOWN = auto() + A_ENC_CNX_LS2 = auto() A_ENC_ATTN_SCALE = auto() # layer scale (gamma) on the attn output A_ENC_FFN_SCALE_LS = auto() # layer scale (gamma) on the FFN output A_ENC_SPEAKER_PROJ = auto() # voice latent -> backbone embd @@ -1419,6 +1427,7 @@ class MODEL_TENSOR(IntEnum): A_GEN_WAV_OUTPUT = auto() A_MMPROJ = auto() A_MMPROJ_FC = auto() + A_MMPROJ_NORM = auto() # vibevoice: norm after mm.a.mlp.{bid} A_MM_NORM_PRE = auto() A_MM_NORM_MID = auto() A_MM_EMBEDDING = auto() # gemma3n @@ -2317,6 +2326,14 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.A_ENC_SEANET_RES_CONV1: "a.seanet.blk.{bid}.res_conv1", MODEL_TENSOR.A_ENC_SEANET_RES_CONV2: "a.seanet.blk.{bid}.res_conv2", MODEL_TENSOR.A_ENC_SEANET_SCALE_CONV: "a.seanet.blk.{bid}.scale_conv", + MODEL_TENSOR.A_ENC_CNX_CONV: "a.cnx.{bid}.conv.{xid}", + MODEL_TENSOR.A_ENC_CNX_NORM: "a.cnx.{bid}.blk.{xid}.norm", + MODEL_TENSOR.A_ENC_CNX_DW: "a.cnx.{bid}.blk.{xid}.dw", + MODEL_TENSOR.A_ENC_CNX_LS1: "a.cnx.{bid}.blk.{xid}.ls1", + MODEL_TENSOR.A_ENC_CNX_FFN_NORM: "a.cnx.{bid}.blk.{xid}.ffn_norm", + MODEL_TENSOR.A_ENC_CNX_FFN_UP: "a.cnx.{bid}.blk.{xid}.ffn_up", + MODEL_TENSOR.A_ENC_CNX_FFN_DOWN: "a.cnx.{bid}.blk.{xid}.ffn_down", + MODEL_TENSOR.A_ENC_CNX_LS2: "a.cnx.{bid}.blk.{xid}.ls2", MODEL_TENSOR.A_ENC_ATTN_SCALE: "a.blk.{bid}.ls1", MODEL_TENSOR.A_ENC_FFN_SCALE_LS: "a.blk.{bid}.ls2", MODEL_TENSOR.A_ENC_SPEAKER_PROJ: "a.speaker_proj", @@ -2349,6 +2366,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.A_GEN_WAV_OUTPUT: "a.gen.wav.output", MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}", MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc", + MODEL_TENSOR.A_MMPROJ_NORM: "mm.a.mlp_norm.{bid}", MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre", MODEL_TENSOR.A_MM_NORM_MID: "mm.a.norm_mid", MODEL_TENSOR.A_MM_INP_PROJ: "mm.a.input_projection", # gemma3n @@ -2641,6 +2659,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.A_ENC_RVQ_CODEBOOK, MODEL_TENSOR.A_MMPROJ, MODEL_TENSOR.A_MMPROJ_FC, + MODEL_TENSOR.A_MMPROJ_NORM, MODEL_TENSOR.A_MM_NORM_PRE, MODEL_TENSOR.A_MM_NORM_MID, MODEL_TENSOR.A_MM_CODE_EMBD, @@ -2741,6 +2760,14 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.A_ENC_SEANET_RES_CONV1, MODEL_TENSOR.A_ENC_SEANET_RES_CONV2, MODEL_TENSOR.A_ENC_SEANET_SCALE_CONV, + MODEL_TENSOR.A_ENC_CNX_CONV, + MODEL_TENSOR.A_ENC_CNX_NORM, + MODEL_TENSOR.A_ENC_CNX_DW, + MODEL_TENSOR.A_ENC_CNX_LS1, + MODEL_TENSOR.A_ENC_CNX_FFN_NORM, + MODEL_TENSOR.A_ENC_CNX_FFN_UP, + MODEL_TENSOR.A_ENC_CNX_FFN_DOWN, + MODEL_TENSOR.A_ENC_CNX_LS2, MODEL_TENSOR.A_ENC_ATTN_SCALE, MODEL_TENSOR.A_ENC_FFN_SCALE_LS, MODEL_TENSOR.A_ENC_SPEAKER_PROJ, @@ -7155,6 +7182,8 @@ class VisionProjectorType: QWEN2A = "qwen2a" # audio QWEN3A = "qwen3a" # audio GLMA = "glma" # audio + HIGGS_AUDIO = "higgs_audio" # audio + VIBEVOICE_ASR = "vibevoice_asr" # audio QWEN25O = "qwen2.5o" # omni VOXTRAL = "voxtral" VOXTRAL_RT = "voxtral_rt" diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 4ee516afa..61e4e7ddc 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -2973,6 +2973,7 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV: ( "encoder.down_sample_layer.0", # mimo-audio-tokenizer + "audio_encoder_proj.temporal", # higgs-audio ), MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM: ( @@ -3026,6 +3027,7 @@ class TensorNameMap: "mlp.model.{bid}", # lfm3a "audio_tower.proj{bid}", # qwen3omni "sound_projection.linear{bid}", # parakeet (linear1, linear2) + "audio_encoder_proj.linear{bid}", # higgs-audio (linear1, linear2) ), MODEL_TENSOR.A_MMPROJ_FC: ( diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 32a17ef61..b23fd5e6b 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -63,6 +63,7 @@ add_library(mtmd models/nemotron-transcribe.cpp models/muse-glimmer.cpp models/muse-spark-audio.cpp + models/vibevoice-asr.cpp models/llama4.cpp models/llava.cpp models/minicpmv.cpp diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 05494c71f..a8e6120da 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -268,6 +268,18 @@ struct mtmd_graph_granite_speech : mtmd_graph { ggml_cgraph * build() override; }; +struct mtmd_graph_vibevoice_asr : mtmd_graph { + mtmd_graph_vibevoice_asr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} + ggml_cgraph * build() override; + + std::map state_in; + mutable std::vector> state_out; + + ggml_tensor * with_left_ctx(ggml_tensor * x, const std::string & name) const; + ggml_tensor * causal_conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & name) const; + ggml_tensor * encode(int e, ggml_tensor * x) const; +}; + struct mtmd_graph_muse_spark_audio : mtmd_graph { mtmd_graph_muse_spark_audio(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; @@ -643,6 +655,9 @@ std::vector list_lfm3a_state_slots(const mtmd_hparams & hparams, // state slots of the streaming Gemma 4 audio encoder (gemma4a) std::vector list_gemma4a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); +// vibevoice-asr: the left context of every causal conv, a zero state is the causal zero padding +std::vector list_vibevoice_asr_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); + struct mtmd_graph_kimik25 : mtmd_graph { mtmd_graph_kimik25(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/models/vibevoice-asr.cpp b/tools/mtmd/models/vibevoice-asr.cpp new file mode 100644 index 000000000..1158988cb --- /dev/null +++ b/tools/mtmd/models/vibevoice-asr.cpp @@ -0,0 +1,138 @@ +#include "models.h" + +// VibeVoice-ASR audio tower: two causal ConvNeXt pyramids (acoustic and semantic) over the raw 24 kHz waveform +// each one is a VAE tokenizer encoder, its latents go through their own connector and the two are summed +// see modeling_vibevoice_acoustic_tokenizer.py and modeling_vibevoice_asr.py in transformers +// +// tensors are T-first [T, C] around the convs and C-first [C, T] in the blocks + +static std::string cnx_slot(int e, const char * kind, int i) { + return "e" + std::to_string(e) + "_" + kind + "_" + std::to_string(i); +} + +std::vector list_vibevoice_asr_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { + std::vector slots; + slots.push_back({"pos", 1, 1}); // unused, set_enc_state_in() expects it + const int n_stage = (int) hparams.audio_patch_sizes.size(); + for (int e = 0; e < 2; ++e) { + const auto & enc = model.cnx_enc[e]; + for (int i = 0; i <= n_stage; ++i) { + const int64_t stride = i < n_stage ? hparams.audio_patch_sizes[i] : 1; + slots.push_back({cnx_slot(e, "conv", i), enc.conv_w[i]->ne[0] - stride, enc.conv_w[i]->ne[1]}); + } + for (int il = 0; il < (int) enc.blocks.size(); ++il) { + const auto * w = enc.blocks[il].conv_w; + slots.push_back({cnx_slot(e, "dw", il), w->ne[0] - 1, w->ne[1]}); + } + } + return slots; +} + +// x: [T, C] -> [left + T, C], the left context comes from the state, its new value is the tail +ggml_tensor * mtmd_graph_vibevoice_asr::with_left_ctx(ggml_tensor * x, const std::string & name) const { + ggml_tensor * left = state_in.at(name); + x = ggml_concat(ctx0, left, x, 0); + state_out.push_back({name, ggml_cont(ctx0, ggml_view_2d(ctx0, x, left->ne[0], x->ne[1], x->nb[1], (x->ne[0] - left->ne[0]) * x->nb[0]))}); + return x; +} + +// x: [T, IC], w: [K, IC, OC] -> [T / stride, OC] +ggml_tensor * mtmd_graph_vibevoice_asr::causal_conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & name) const { + x = with_left_ctx(x, name); + ggml_tensor * y = ggml_conv_1d(ctx0, w, x, stride, 0, 1); + y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[1]); + return ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0])); +} + +// x: [T, 1] -> [C_latent, T / hop] +ggml_tensor * mtmd_graph_vibevoice_asr::encode(int e, ggml_tensor * x) const { + const auto & enc = model.cnx_enc[e]; + const int n_stage = (int) hparams.audio_patch_sizes.size(); + + ggml_tensor * cur = x; + int bid = 0; + for (int stage = 0; stage < n_stage; ++stage) { + cur = causal_conv(cur, enc.conv_w[stage], enc.conv_b[stage], hparams.audio_patch_sizes[stage], cnx_slot(e, "conv", stage)); + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cb(cur, "stage_in", stage); + + const int64_t n_pos = cur->ne[1]; + for (int i = 0; i < hparams.audio_depths[stage]; ++i, ++bid) { + const auto & layer = enc.blocks[bid]; + + // causal depthwise conv mixer + ggml_tensor * h = build_norm(cur, layer.ln_1_w, nullptr, NORM_TYPE_RMS, hparams.eps, bid); + h = ggml_cont(ctx0, ggml_transpose(ctx0, h)); + h = with_left_ctx(h, cnx_slot(e, "dw", bid)); + h = ggml_reshape_3d(ctx0, h, h->ne[0], h->ne[1], 1); + h = ggml_ssm_conv(ctx0, h, layer.conv_w); + h = ggml_reshape_2d(ctx0, h, h->ne[0], n_pos); + h = ggml_add(ctx0, h, layer.conv_b); + cur = ggml_add(ctx0, cur, ggml_mul(ctx0, h, layer.ls_1_w)); + + h = build_norm(cur, layer.ln_2_w, nullptr, NORM_TYPE_RMS, hparams.eps, bid); + h = build_ffn(h, + layer.ff_up_w, layer.ff_up_b, + nullptr, nullptr, + layer.ff_down_w, layer.ff_down_b, + FFN_GELU_ERF, bid); + cur = ggml_add(ctx0, cur, ggml_mul(ctx0, h, layer.ls_2_w)); + cb(cur, "block_out", bid); + } + + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + } + + cur = causal_conv(cur, enc.conv_w[n_stage], enc.conv_b[n_stage], 1, cnx_slot(e, "conv", n_stage)); + return ggml_cont(ctx0, ggml_transpose(ctx0, cur)); +} + +ggml_cgraph * mtmd_graph_vibevoice_asr::build() { + const int64_t hop = hparams.audio_samples_per_token(); + const int64_t n_frames = img.nx() / hop; + const int64_t n_ctx = img.audio_n_ctx / hop; + GGML_ASSERT(img.nx() % hop == 0 && img.audio_n_ctx % hop == 0 && n_ctx < n_frames); + + for (const auto & slot : list_vibevoice_asr_state_slots(hparams, model)) { + ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1); + ggml_set_name(t, ("state_in_" + slot.name).c_str()); + ggml_set_input(t); + state_in[slot.name] = t; + } + state_out.push_back({"pos", ggml_scale_bias(ctx0, state_in.at("pos"), 1.0f, (float) n_frames)}); + + ggml_tensor * inp = build_inp_raw(1); + inp = ggml_reshape_2d(ctx0, inp, img.nx(), 1); + + // the leading context frames only warm up the causal convs + auto drop_ctx = [&](ggml_tensor * t) { + return ggml_view_2d(ctx0, t, t->ne[0], n_frames - n_ctx, t->nb[1], n_ctx * t->nb[1]); + }; + + // the reference adds VAE noise to the acoustic latents, skipped here to keep the output deterministic + ggml_tensor * acoustic = drop_ctx(encode(0, inp)); + ggml_tensor * semantic = drop_ctx(encode(1, inp)); + cb(acoustic, "acoustic_latents", -1); + cb(semantic, "semantic_latents", -1); + + acoustic = ggml_add(ctx0, build_mm(model.mm_0_w, acoustic), model.mm_0_b); + acoustic = build_norm(acoustic, model.mm_norm_pre_w, nullptr, NORM_TYPE_RMS, 1e-6f, -1); + acoustic = ggml_add(ctx0, build_mm(model.mm_1_w, acoustic), model.mm_1_b); + + semantic = ggml_add(ctx0, build_mm(model.mm_2_w, semantic), model.mm_2_b); + semantic = build_norm(semantic, model.mm_norm_mid_w, nullptr, NORM_TYPE_RMS, 1e-6f, -1); + semantic = ggml_add(ctx0, build_mm(model.mm_3_w, semantic), model.mm_3_b); + + ggml_tensor * cur = ggml_add(ctx0, acoustic, semantic); + cb(cur, "projected", -1); + + // the embeddings must stay the last node + for (const auto & s : state_out) { + ggml_set_name(s.second, ("state_out_" + s.first).c_str()); + ggml_set_output(s.second); + ggml_build_forward_expand(gf, s.second); + } + + ggml_build_forward_expand(gf, cur); + return gf; +} diff --git a/tools/mtmd/models/whisper-enc.cpp b/tools/mtmd/models/whisper-enc.cpp index 6225b707c..e755990d5 100644 --- a/tools/mtmd/models/whisper-enc.cpp +++ b/tools/mtmd/models/whisper-enc.cpp @@ -2,7 +2,7 @@ ggml_cgraph * mtmd_graph_whisper_enc::build() { const int n_frames = img.nx(); - const int n_pos = n_frames / 2; + const int n_pos = (n_frames + 1) / 2; // stride-2 conv1d with padding 1 GGML_ASSERT(model.position_embeddings->ne[1] >= n_pos); ggml_tensor * inp = build_inp_raw(1); @@ -77,6 +77,19 @@ ggml_cgraph * mtmd_graph_whisper_enc::build() { cur = build_mm(model.mm_fc_w, cur); cur = ggml_add(ctx0, cur, model.mm_fc_b); + } else if (proj_type == PROJECTOR_TYPE_HIGGS_AUDIO) { + // depthwise conv (k=3, s=2, p=1) over time, then linear-relu-linear + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cur = ggml_conv_1d_dw(ctx0, model.downsample_conv_w, cur, 2, 1, 1); + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cur = ggml_add(ctx0, cur, model.downsample_conv_b); + + cur = build_mm(model.mm_1_w, cur); + cur = ggml_add(ctx0, cur, model.mm_1_b); + cur = ggml_relu(ctx0, cur); + cur = build_mm(model.mm_2_w, cur); + cur = ggml_add(ctx0, cur, model.mm_2_b); + } else if (proj_type == PROJECTOR_TYPE_VOXTRAL) { // projector cur = build_ffn(cur, diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index c6d751b41..dfd3b30e7 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -277,6 +277,7 @@ static void mtmd_audio_fill_entries(std::vector & mels, bool sha GGML_ASSERT(mel.n_mel <= INT32_MAX && mel.n_mel >= 0); mtmd_image_f32 entry; entry.set_size({(int) mel.n_len, (int) mel.n_mel}, shape_only || mel.data.empty(), /* is_audio */ true); + entry.audio_n_ctx = (int32_t) mel.n_ctx; if (!entry.is_placeholder()) { entry.cpy_buf(mel.data); } @@ -688,7 +689,7 @@ bool mtmd_audio_preprocessor_qwen3a::process(const float * sampl // // mtmd_audio_preprocessor_dots3note // -// Matches Dots3NoteFeatureExtractor: the waveform is split into 60s chunks and each chunk gets +// Matches Dots3NoteFeatureExtractor: the waveform is split into audio_chunk_len chunks and each chunk gets // its own whisper-style log-mel (center=True, log10 + (max-8)/4). Only sample_length//hop frames // per chunk are valid; the reference masks everything beyond them, so we emit exactly that many. // @@ -733,7 +734,7 @@ bool mtmd_audio_preprocessor_dots3note::process(const float * sa continue; // sub-hop tail, contributes no frames } - // reflect-pad the start; the reference zero-pads partial chunks to 60s before the STFT, + // reflect-pad the start; the reference zero-pads partial chunks to the chunk length before the STFT, // so a partial chunk sees zeros past its end while a full chunk reflects its own tail std::vector padded(n_chunk + 2 * pad, 0.0f); for (int i = 0; i < pad; i++) { @@ -780,6 +781,20 @@ bool mtmd_audio_preprocessor_dots3note::process(const float * sa return !output.empty(); } +bool mtmd_audio_preprocessor_higgs_audio::process(const float * samples, + size_t n_samples, + std::vector & output, + bool shape_only) const { + if (!mtmd_audio_preprocessor_dots3note::process(samples, n_samples, output, shape_only)) { + return false; + } + // a tail under 3 frames gives no token, the reference drops it too + if (output.size() > 1 && output.back().n_len < 3) { + output.pop_back(); + } + return !output.empty() && output.back().n_len >= 3; +} + // // mtmd_audio_preprocessor_mimo_audio // @@ -2227,6 +2242,65 @@ bool mtmd_audio_preprocessor_pockettts::process(const float * sa return true; } +// +// mtmd_audio_preprocessor_vibevoice_asr +// +// Matches VibeVoiceAcousticTokenizerFeatureExtractor: the clip is scaled to -25 dBFS RMS, then zero-padded to a whole number of frames. +// The reference encodes it with a streaming conv cache, here each chunk re-reads audio_n_ctx samples before its start instead, which gives the same output. +// + +double mtmd_audio_vibevoice_gain(double sum_sq, int64_t n_samples, double peak) { + const double rms = std::max(std::sqrt(sum_sq / std::max(n_samples, 1)), std::pow(10.0, -50.0 / 20.0)); + double gain = std::pow(10.0, -25.0 / 20.0) / (rms + 1e-6); + if (peak * gain > 1.0) { + gain /= peak * gain + 1e-6; + } + return gain; +} + +bool mtmd_audio_preprocessor_vibevoice_asr::process(const float * samples, + size_t n_samples, + std::vector & output, + bool /*shape_only*/) const { + if (n_samples == 0) { + return false; + } + + const int64_t hop = hparams.audio_samples_per_token(); + const int64_t n_ctx = hparams.audio_n_ctx; + const int64_t n_chunk = (int64_t) hparams.audio_chunk_len * hparams.audio_sample_rate / hop * hop; + const int64_t n_padded = ((int64_t) n_samples + hop - 1) / hop * hop; + GGML_ASSERT(n_chunk > 0 && n_ctx % hop == 0); + + double sum_sq = 0.0; + double peak = 0.0; + for (size_t i = 0; i < n_samples; i++) { + sum_sq += (double) samples[i] * samples[i]; + peak = std::max(peak, (double) std::fabs(samples[i])); + } + const double scale = mtmd_audio_vibevoice_gain(sum_sq, (int64_t) n_samples, peak); + + std::vector audio((size_t) n_padded, 0.0f); + for (size_t i = 0; i < n_samples; i++) { + audio[i] = (float) (samples[i] * scale); + } + + output.clear(); + for (int64_t start = 0; start < n_padded; start += n_chunk) { + const int64_t beg = std::max(0, start - n_ctx); + const int64_t end = std::min(n_padded, start + n_chunk); + + mtmd_audio_mel mel; + mel.n_len = end - beg; + mel.n_len_org = end - beg; + mel.n_mel = 1; + mel.n_ctx = start - beg; + mel.data.assign(audio.begin() + beg, audio.begin() + end); + output.push_back(std::move(mel)); + } + return true; +} + // Muse Spark reads the raw waveform: pass the samples through, zero padded to a whole number of embeddings bool mtmd_audio_preprocessor_muse_spark_audio::process( const float * samples, diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index 937451c8a..b4b1ce692 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -14,6 +14,7 @@ struct mtmd_audio_mel { int64_t n_len; int64_t n_len_org; int64_t n_mel; + int64_t n_ctx = 0; // see mtmd_image_f32::audio_n_ctx std::vector data; }; @@ -128,6 +129,17 @@ struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor { bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; }; +// vibevoice: gain to -25 dBFS RMS, without clipping the peak +// the RMS is floored at -50 dBFS, so that silence or noise alone is not boosted to full scale +double mtmd_audio_vibevoice_gain(double sum_sq, int64_t n_samples, double peak); + +// raw 24 kHz waveform, cut in chunks that carry their causal context, see mtmd_hparams::audio_n_ctx +struct mtmd_audio_preprocessor_vibevoice_asr : mtmd_audio_preprocessor { + mtmd_audio_preprocessor_vibevoice_asr(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} + void initialize() override {} + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; +}; + struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor { mtmd_audio_preprocessor_conformer(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} mtmd_audio_preprocessor_conformer(const mtmd_hparams & hparams, ggml_backend_sched_t sched) : mtmd_audio_preprocessor(hparams, sched) {} @@ -198,6 +210,12 @@ struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor { mtmd_audio_cache cache; }; +// same per-chunk log-mel as dots3note, with 4s chunks +struct mtmd_audio_preprocessor_higgs_audio : mtmd_audio_preprocessor_dots3note { + mtmd_audio_preprocessor_higgs_audio(const mtmd_model * model) : mtmd_audio_preprocessor_dots3note(model) {} + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; +}; + struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor { mtmd_audio_preprocessor_mimo_audio(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp index ac656032d..100bc3c58 100644 --- a/tools/mtmd/mtmd-graph.cpp +++ b/tools/mtmd/mtmd-graph.cpp @@ -870,6 +870,7 @@ std::unique_ptr mtmd_get_graph_builder(mtmd_subcontext * ctx, const case PROJECTOR_TYPE_ULTRAVOX: case PROJECTOR_TYPE_VOXTRAL: case PROJECTOR_TYPE_QWEN2A: + case PROJECTOR_TYPE_HIGGS_AUDIO: case PROJECTOR_TYPE_GLMA: case PROJECTOR_TYPE_MERALION: case PROJECTOR_TYPE_MUSIC_FLAMINGO: @@ -880,6 +881,10 @@ std::unique_ptr mtmd_get_graph_builder(mtmd_subcontext * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_VIBEVOICE_ASR: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_KIMIVL: { builder = std::make_unique(ctx, img); diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index e14476788..bce73a74e 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include //#define MTMD_AUDIO_DEBUG @@ -939,6 +940,54 @@ static void mtmd_helper_video_set_id(mtmd_helper_video * vctx, const std::string vctx->id = id; } +int32_t mtmd_helper_get_media_meta(const unsigned char * buf, size_t len, mtmd_helper_init_opt opt, mtmd_helper_media_meta * out) { + *out = {MTMD_HELPER_MEDIA_TYPE_UNKNOWN, 0, 0, 0, -1.0f, -1}; + + if (audio_helpers::is_audio_file((const char *) buf, len)) { + // zero channels and rate: the source format, so the length comes from the header (or an mp3 frame scan) + ma_decoder_config cfg = ma_decoder_config_init(ma_format_f32, 0, 0); + ma_decoder dec; + if (ma_decoder_init_memory(buf, len, &cfg, &dec) != MA_SUCCESS) { + return 1; + } + ma_uint64 n = 0; + if (ma_decoder_get_length_in_pcm_frames(&dec, &n) == MA_SUCCESS && dec.outputSampleRate > 0) { + out->duration = (float) ((double) n / dec.outputSampleRate); + } + out->type = MTMD_HELPER_MEDIA_TYPE_AUDIO; + out->sample_rate = (int32_t) dec.outputSampleRate; + ma_decoder_uninit(&dec); + return 0; + } + + int nx, ny, nc; + if (len <= INT_MAX && stbi_info_from_memory(buf, (int) len, &nx, &ny, &nc)) { + out->type = MTMD_HELPER_MEDIA_TYPE_IMAGE; + out->width = (uint32_t) nx; + out->height = (uint32_t) ny; + return 0; + } + + // webp (not read by stb_image) and video go through the video decoder + mtmd_helper_video_init_params params = opt.video_params; + const bool is_webp = is_webp_file(buf, len); + if (is_webp && video_builtin_decoder()) { + params.decoder = nullptr; // see decode_still_image() + } + mtmd_helper_video_stream_info src; + if (!video_open(nullptr, buf, len, params, src)) { + return 1; + } + out->type = is_webp ? MTMD_HELPER_MEDIA_TYPE_IMAGE : MTMD_HELPER_MEDIA_TYPE_VIDEO; + out->width = src.width; + out->height = src.height; + if (!is_webp) { + out->duration = src.duration; + out->n_frames = src.n_frames; + } + return 0; +} + mtmd_helper_video_init_params mtmd_helper_video_init_params_default() { return { /* fps_target */ 4.0f, diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index 5ca414906..b33ed65ec 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -107,6 +107,32 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf( bool placeholder, struct mtmd_helper_init_opt opt); +enum mtmd_helper_media_type { + MTMD_HELPER_MEDIA_TYPE_UNKNOWN, + MTMD_HELPER_MEDIA_TYPE_IMAGE, + MTMD_HELPER_MEDIA_TYPE_AUDIO, + MTMD_HELPER_MEDIA_TYPE_VIDEO, +}; + +struct mtmd_helper_media_meta { + enum mtmd_helper_media_type type; + uint32_t width; // image, video (frame size) + uint32_t height; + int32_t sample_rate; // audio, at the source rate + float duration; // audio, video: in seconds, -1 if unknown + int32_t n_frames; // video: -1 if unknown +}; +typedef struct mtmd_helper_media_meta mtmd_helper_media_meta; + +// metadata of a media file in a buffer, the same formats as mtmd_helper_bitmap_init_from_buf() +// only the headers are read (ffprobe or opt.video_params.decoder for video), nothing is decoded +// returns 0 on success, 1 if the format is not recognized +// this function is thread-safe +MTMD_API int32_t mtmd_helper_get_media_meta( + const unsigned char * buf, size_t len, + struct mtmd_helper_init_opt opt, + struct mtmd_helper_media_meta * out); + // streaming resampler for raw mono F32 PCM (e.g. realtime input), to match mtmd_get_audio_sample_rate() // it keeps state between calls, use one per stream struct mtmd_helper_resampler; diff --git a/tools/mtmd/mtmd-impl.h b/tools/mtmd/mtmd-impl.h index 0d287dcdd..70db37654 100644 --- a/tools/mtmd/mtmd-impl.h +++ b/tools/mtmd/mtmd-impl.h @@ -415,6 +415,17 @@ #define TN_A_CONV1D_NORM "a.conv1d.%d.norm.%s" #define TN_A_PATCH "a.patch.%d.%s" #define TN_A_PATCH_NORM "a.patch.%d.norm.%s" + +// vibevoice-asr: causal ConvNeXt encoders, first index = encoder (0 = acoustic, 1 = semantic) +#define TN_A_CNX_CONV "a.cnx.%d.conv.%d.%s" +#define TN_A_CNX_NORM "a.cnx.%d.blk.%d.norm.%s" +#define TN_A_CNX_DW "a.cnx.%d.blk.%d.dw.%s" +#define TN_A_CNX_LS1 "a.cnx.%d.blk.%d.ls1.%s" +#define TN_A_CNX_FFN_NORM "a.cnx.%d.blk.%d.ffn_norm.%s" +#define TN_A_CNX_FFN_UP "a.cnx.%d.blk.%d.ffn_up.%s" +#define TN_A_CNX_FFN_DOWN "a.cnx.%d.blk.%d.ffn_down.%s" +#define TN_A_CNX_LS2 "a.cnx.%d.blk.%d.ls2.%s" +#define TN_MM_AUDIO_MLP_NORM "mm.a.mlp_norm.%d.%s" #define TN_A_OUT_PROJ "a.pre_encode.out.%s" #define TN_A_ATTN_PRE_NORM "%s.blk.%d.attn_pre_norm.%s" #define TN_A_ATTN_POST_NORM "%s.blk.%d.attn_post_norm.%s" @@ -522,6 +533,8 @@ enum projector_type { PROJECTOR_TYPE_QWEN2A, PROJECTOR_TYPE_QWEN3A, PROJECTOR_TYPE_GLMA, + PROJECTOR_TYPE_HIGGS_AUDIO, + PROJECTOR_TYPE_VIBEVOICE_ASR, PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on mtmd_subcontext PROJECTOR_TYPE_VOXTRAL, PROJECTOR_TYPE_VOXTRAL_RT, @@ -612,6 +625,8 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_QWEN2A, "qwen2a"}, { PROJECTOR_TYPE_QWEN3A, "qwen3a"}, { PROJECTOR_TYPE_GLMA, "glma"}, + { PROJECTOR_TYPE_HIGGS_AUDIO, "higgs_audio"}, + { PROJECTOR_TYPE_VIBEVOICE_ASR, "vibevoice_asr"}, { PROJECTOR_TYPE_QWEN25O, "qwen2.5o"}, { PROJECTOR_TYPE_VOXTRAL, "voxtral"}, { PROJECTOR_TYPE_VOXTRAL_RT, "voxtral_rt"}, @@ -815,6 +830,8 @@ struct mtmd_image_f32 { // deepseek4v: number of leading IMAGE_PAD embeddings, aligns IMAGE_START to the LLM compressor ratio // depends on the chunk position, set at tokenize time (see mtmd_tokenizer::add_media) int32_t lead_pad = 0; + // audio (vibevoice): leading samples that only feed the causal convs, they produce no embedding + int32_t audio_n_ctx = 0; // separators appended after the image tokens mtmd_suffix_type suffix_type = MTMD_SUFFIX_NONE; diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp index c4c52a4d5..86bb127fb 100644 --- a/tools/mtmd/mtmd-model.cpp +++ b/tools/mtmd/mtmd-model.cpp @@ -785,6 +785,46 @@ struct mtmd_model_loader { } hparams.warmup_audio_size = hparams.audio_samples_per_token(); } break; + case PROJECTOR_TYPE_VIBEVOICE_ASR: + { + hparams.audio_sample_rate = 24000; // mono waveform, no frontend + hparams.audio_chunk_len = 30; // in seconds, any length works as the convs are causal + get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size, true); + get_arr_int(KEY_A_PATCH_SIZES, hparams.audio_patch_sizes, true); + get_arr_int(KEY_A_DEPTHS, hparams.audio_depths, true); + + // bounds keep the receptive field below from overflowing + const size_t n_stage = hparams.audio_patch_sizes.size(); + if (n_stage == 0 || n_stage > 16 || hparams.audio_depths.size() != n_stage || hparams.n_mel_bins != 1 || + hparams.audio_conv_kernel_size < 1 || hparams.audio_conv_kernel_size > 64) { + throw std::runtime_error(mtmd_format("%s: invalid vibevoice audio hparams\n", __func__)); + } + + // receptive field: stem conv (kernel), downsample convs (2 * stride), blocks (kernel), head conv (kernel) + const int64_t k = hparams.audio_conv_kernel_size; + int64_t hop = 1; + int64_t rf = 0; + int64_t n_blocks = 0; + for (size_t i = 0; i < n_stage; ++i) { + const int64_t stride = hparams.audio_patch_sizes[i]; + if (stride < 1 || stride > 64 || hop * stride > (1 << 20) || hparams.audio_depths[i] < 0 || hparams.audio_depths[i] > 64) { + throw std::runtime_error(mtmd_format("%s: invalid vibevoice audio hparams\n", __func__)); + } + rf += ((i == 0 ? k : 2 * stride) - 1) * hop; + hop *= stride; + rf += hparams.audio_depths[i] * (k - 1) * hop; + n_blocks += hparams.audio_depths[i]; + } + rf += (k - 1) * hop; + if (rf > 60 * hparams.audio_sample_rate) { + throw std::runtime_error(mtmd_format("%s: vibevoice receptive field of %" PRId64 " samples is too long\n", __func__, rf)); + } + if (n_blocks != hparams.n_layer) { + throw std::runtime_error(mtmd_format("%s: sum of audio depths (%" PRId64 ") does not match n_layer (%d)\n", __func__, n_blocks, hparams.n_layer)); + } + hparams.audio_n_ctx = (int32_t) ((rf + hop - 1) / hop * hop); + hparams.warmup_audio_size = (int32_t) ((int64_t) hparams.audio_chunk_len * hparams.audio_sample_rate / hop * hop) + hparams.audio_n_ctx; + } break; case PROJECTOR_TYPE_INKLING2V: { hparams.ffn_op = FFN_GELU_ERF; @@ -892,6 +932,7 @@ struct mtmd_model_loader { } break; case PROJECTOR_TYPE_ULTRAVOX: case PROJECTOR_TYPE_QWEN2A: + case PROJECTOR_TYPE_HIGGS_AUDIO: case PROJECTOR_TYPE_QWEN3A: case PROJECTOR_TYPE_GLMA: case PROJECTOR_TYPE_VOXTRAL: @@ -912,6 +953,12 @@ struct mtmd_model_loader { hparams.audio_n_fft = 400; hparams.audio_window_len = 400; hparams.audio_hop_len = 160; + + if (model.proj_type == PROJECTOR_TYPE_HIGGS_AUDIO) { + // each 4s chunk is encoded on its own + hparams.audio_chunk_len = 4; + hparams.warmup_audio_size = 400; + } } break; case PROJECTOR_TYPE_VOXTRAL_RT: { @@ -1324,6 +1371,7 @@ struct mtmd_model_loader { // Muse Spark is encoder-free in the same sense: its conv pyramid reads the raw waveform, one channel takes the place of the mel bins const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA && model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO && + model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR && model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC; // Validate audio hparams loaded from GGUF metadata @@ -1497,6 +1545,7 @@ struct mtmd_model_loader { model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN && model.proj_type != PROJECTOR_TYPE_SOPRANO && model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO && + model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR && model.proj_type != PROJECTOR_TYPE_CHATTERBOX && model.proj_type != PROJECTOR_TYPE_CHATTERBOX_SPKENC); @@ -1928,6 +1977,52 @@ struct mtmd_model_loader { model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight")); model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias")); } break; + case PROJECTOR_TYPE_VIBEVOICE_ASR: + { + const int n_stage = (int) hparams.audio_patch_sizes.size(); + for (int e = 0; e < 2; ++e) { + auto & enc = model.cnx_enc[e]; + enc.conv_w.resize(n_stage + 1); + enc.conv_b.resize(n_stage + 1); + for (int i = 0; i <= n_stage; ++i) { + enc.conv_w[i] = get_tensor(mtmd_format(TN_A_CNX_CONV, e, i, "weight")); + enc.conv_b[i] = get_tensor(mtmd_format(TN_A_CNX_CONV, e, i, "bias")); + // the receptive field in load_hparams assumes these kernel sizes + const int64_t k_expected = (i == 0 || i == n_stage) ? hparams.audio_conv_kernel_size : 2 * hparams.audio_patch_sizes[i]; + if (enc.conv_w[i]->ne[0] != k_expected) { + throw std::runtime_error(mtmd_format("%s: %s has kernel %" PRId64 ", expected %" PRId64 "\n", __func__, ggml_get_name(enc.conv_w[i]), enc.conv_w[i]->ne[0], k_expected)); + } + } + enc.blocks.resize(hparams.n_layer); + for (int il = 0; il < hparams.n_layer; ++il) { + auto & layer = enc.blocks[il]; + layer.ln_1_w = get_tensor(mtmd_format(TN_A_CNX_NORM, e, il, "weight")); + layer.conv_w = get_tensor(mtmd_format(TN_A_CNX_DW, e, il, "weight")); + layer.conv_b = get_tensor(mtmd_format(TN_A_CNX_DW, e, il, "bias")); + layer.ls_1_w = get_tensor(mtmd_format(TN_A_CNX_LS1, e, il, "weight")); + layer.ln_2_w = get_tensor(mtmd_format(TN_A_CNX_FFN_NORM, e, il, "weight")); + layer.ff_up_w = get_tensor(mtmd_format(TN_A_CNX_FFN_UP, e, il, "weight")); + layer.ff_up_b = get_tensor(mtmd_format(TN_A_CNX_FFN_UP, e, il, "bias")); + layer.ff_down_w = get_tensor(mtmd_format(TN_A_CNX_FFN_DOWN, e, il, "weight")); + layer.ff_down_b = get_tensor(mtmd_format(TN_A_CNX_FFN_DOWN, e, il, "bias")); + layer.ls_2_w = get_tensor(mtmd_format(TN_A_CNX_LS2, e, il, "weight")); + if (layer.conv_w->ne[0] != hparams.audio_conv_kernel_size || layer.conv_w->type != GGML_TYPE_F32) { + throw std::runtime_error(mtmd_format("%s: %s must be F32 with kernel %d\n", __func__, ggml_get_name(layer.conv_w), hparams.audio_conv_kernel_size)); + } + } + } + // acoustic connector: mlp 0 -> norm -> mlp 1, semantic connector: mlp 2 -> norm -> mlp 3 + model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias")); + model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP_NORM, 0, "weight")); + model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); + model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias")); + model.mm_norm_mid_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP_NORM, 2, "weight")); + model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight")); + model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias")); + } break; case PROJECTOR_TYPE_INKLING2V: { // these come from the generic path above, where they are optional @@ -2315,6 +2410,19 @@ struct mtmd_model_loader { model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight")); model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias")); } break; + case PROJECTOR_TYPE_HIGGS_AUDIO: + { + model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); + model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); + model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); + model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); + model.downsample_conv_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight")); + model.downsample_conv_b = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); + model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias")); + } break; case PROJECTOR_TYPE_QWEN3A: { model.conv2d_1_w = get_tensor(mtmd_format(TN_CONV2D, 1, "weight")); @@ -4187,6 +4295,10 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const { const int64_t fold = hparams.audio_samples_per_token(); n_patches = (int) ((img->nx() + fold - 1) / fold); } break; + case PROJECTOR_TYPE_VIBEVOICE_ASR: + { + n_patches = (img->nx() - img->audio_n_ctx) / hparams.audio_samples_per_token(); + } break; case PROJECTOR_TYPE_VOXTRAL: case PROJECTOR_TYPE_ULTRAVOX: case PROJECTOR_TYPE_QWEN2A: @@ -4210,6 +4322,13 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const { n_patches /= 2; } } break; + case PROJECTOR_TYPE_HIGGS_AUDIO: + { + // stride-2 conv1d, avgpool(2), stride-2 depthwise conv1d + n_patches = (img->nx() + 1) / 2; + n_patches = n_patches / 2; + n_patches = (n_patches + 1) / 2; + } break; case PROJECTOR_TYPE_QWEN3A: { // chunk_size=100 frames --> 3x stride-2 conv2d --> 13 tokens per chunk @@ -4449,6 +4568,8 @@ int mtmd_model::n_mmproj_embd() const { case PROJECTOR_TYPE_IDEFICS3: case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: return mm_fc_w->ne[1]; + case PROJECTOR_TYPE_VIBEVOICE_ASR: + return mm_1_w->ne[1]; case PROJECTOR_TYPE_ULTRAVOX: case PROJECTOR_TYPE_VOXTRAL: case PROJECTOR_TYPE_VOXTRAL_RT: @@ -4466,6 +4587,7 @@ int mtmd_model::n_mmproj_embd() const { case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: return mm_fc_w->ne[1]; case PROJECTOR_TYPE_QWEN3A: + case PROJECTOR_TYPE_HIGGS_AUDIO: return mm_2_w->ne[1]; case PROJECTOR_TYPE_GLMA: case PROJECTOR_TYPE_LFM2: diff --git a/tools/mtmd/mtmd-model.h b/tools/mtmd/mtmd-model.h index d04314d93..44863636b 100644 --- a/tools/mtmd/mtmd-model.h +++ b/tools/mtmd/mtmd-model.h @@ -160,7 +160,7 @@ struct mtmd_hparams { int32_t audio_proj_downsample_rate = 0; int32_t audio_proj_head_count = 0; - // audio pyramid (muse-spark): widths, folded samples and blocks per stage + // audio pyramid (muse-spark, vibevoice): widths, folded samples (vibevoice: conv strides) and blocks per stage std::vector audio_hidden_sizes; std::vector audio_patch_sizes; std::vector audio_depths; @@ -174,6 +174,9 @@ struct mtmd_hparams { return (int32_t) n; } + // vibevoice: leading samples of a chunk that only feed the causal convs, at least their receptive field + int32_t audio_n_ctx = 0; + // audio-to-mel preprocessor params int32_t audio_chunk_len = -1; // in seconds int32_t audio_sample_rate = -1; @@ -528,6 +531,14 @@ struct mtmd_seanet { std::vector stages; }; +// vibevoice-asr: causal conv pyramid over the raw waveform +// stage = strided conv -> ConvNeXt blocks (RMS norm, depthwise conv, layer scale, then RMS norm, MLP, layer scale) +struct mtmd_cnx_encoder { + std::vector conv_w; // stem, one downsample conv per stage, head + std::vector conv_b; + std::vector blocks; // ln_1, conv (depthwise), ls_1, ln_2, ffn_up, ffn_down, ls_2 +}; + // pocket-tts flow-matching decoder (SimpleMLPAdaLN) struct mtmd_flow_net { // AdaLN res block: in_ln -> modulate -> Linear -> SiLU -> Linear, gated residual @@ -1160,7 +1171,9 @@ struct mtmd_model { ggml_tensor * mm_norm_mid_w = nullptr; // mimo-audio-tokenizer: post-transformer downsample + RVQ codebook - ggml_tensor * downsample_conv_w = nullptr; // no bias + // higgs-audio: depthwise downsample conv of the projector + ggml_tensor * downsample_conv_w = nullptr; + ggml_tensor * downsample_conv_b = nullptr; // higgs-audio only ggml_tensor * downsample_norm_w = nullptr; ggml_tensor * downsample_norm_b = nullptr; ggml_tensor * rvq_codebook = nullptr; // merged 3D [n_q, max_bins, dim] @@ -1208,6 +1221,9 @@ struct mtmd_model { // pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path) mtmd_seanet seanet; + // vibevoice-asr: acoustic and semantic encoders + mtmd_cnx_encoder cnx_enc[2]; + // pocket-tts: voice latent -> backbone embd (speaker path) ggml_tensor * spk_proj_w = nullptr; ggml_tensor * downsample_w = nullptr; @@ -1332,6 +1348,7 @@ struct mtmd_model { bool audio_has_avgpool() const { return proj_type == PROJECTOR_TYPE_QWEN2A + || proj_type == PROJECTOR_TYPE_HIGGS_AUDIO || proj_type == PROJECTOR_TYPE_VOXTRAL || proj_type == PROJECTOR_TYPE_MUSIC_FLAMINGO; } diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp index ed8800fce..db54ec9c9 100644 --- a/tools/mtmd/mtmd-subcontext.cpp +++ b/tools/mtmd/mtmd-subcontext.cpp @@ -402,6 +402,7 @@ static std::vector list_enc_state_slots(const mtmd_hparams & hpa switch (model.proj_type) { case PROJECTOR_TYPE_LFM3A: return list_lfm3a_state_slots(hparams, model); case PROJECTOR_TYPE_GEMMA4A: return list_gemma4a_state_slots(hparams, model); + case PROJECTOR_TYPE_VIBEVOICE_ASR: return list_vibevoice_asr_state_slots(hparams, model); default: return {}; } } @@ -451,7 +452,7 @@ mtmd_subctx_gen_state * mtmd_subcontext::gen_state_init(uint32_t seed) { mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) { const auto & hparams = model.hparams; - if (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A) { + if (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A || model.proj_type == PROJECTOR_TYPE_VIBEVOICE_ASR) { // see list_enc_state_slots() const auto slots = list_enc_state_slots(hparams, model); auto st = std::make_unique(); @@ -1662,6 +1663,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { case PROJECTOR_TYPE_NEMOTRON_V2_VL: case PROJECTOR_TYPE_NEMOTRON_OMNI: case PROJECTOR_TYPE_QWEN2A: + case PROJECTOR_TYPE_HIGGS_AUDIO: case PROJECTOR_TYPE_QWEN3A: case PROJECTOR_TYPE_GLMA: case PROJECTOR_TYPE_ULTRAVOX: @@ -1982,6 +1984,10 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { } set_input_i32("positions", positions); } break; + case PROJECTOR_TYPE_VIBEVOICE_ASR: + { + set_enc_state_in(); + } break; case PROJECTOR_TYPE_GEMMA4A: { GGML_ASSERT(imgs.entries.size() == 1); @@ -2627,7 +2633,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { } } } - if (params->enc_state != nullptr && (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A)) { + if (params->enc_state != nullptr && (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A || model.proj_type == PROJECTOR_TYPE_VIBEVOICE_ASR)) { const auto & slots = params->enc_state->slots; size_t i = 0; for (const auto & slot : list_enc_state_slots(hparams, model)) { diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 0042aeb42..f8f7ea048 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -48,7 +48,7 @@ std::string mtmd_format(const char * fmt, ...) { } // remember to bump this if the serialization format changes -#define MTMD_SERIALIZATION_VERSION 4 +#define MTMD_SERIALIZATION_VERSION 5 // oldest compat version that can be loaded #define MTMD_SERIALIZATION_VERSION_MIN 2 @@ -137,6 +137,7 @@ void mtmd_image_f32::serialize(mtmd_serialization & ser) const { ser.write((int32_t)anyres.orig_ny); ser.write((int32_t)nx_); ser.write((int32_t)ny_); + ser.write(audio_n_ctx); } void mtmd_image_f32::deserialize(mtmd_serialization & ser) { add_viewsep = ser.read(); @@ -155,6 +156,12 @@ void mtmd_image_f32::deserialize(mtmd_serialization & ser) { } nx_ = ser.read(); ny_ = ser.read(); + if (ser.version >= 5) { + audio_n_ctx = ser.read(); + if (audio_n_ctx < 0 || audio_n_ctx > nx_) { + throw std::runtime_error("invalid audio_n_ctx"); + } + } buf.clear(); // always a placeholder after loading } @@ -1194,6 +1201,19 @@ struct mtmd_context { aud_end = "<|audio_eos|>"; audio_preproc = std::make_unique(models.a.get()); } break; + case PROJECTOR_TYPE_HIGGS_AUDIO: + { + // the 4s chunks are joined in one <|audio_bos|> ... <|audio_eos|> span + aud_beg = "<|audio_bos|>"; + aud_end = "<|audio_eos|>"; + audio_preproc = std::make_unique(models.a.get()); + } break; + case PROJECTOR_TYPE_VIBEVOICE_ASR: + { + aud_beg = "<|object_ref_start|>"; + aud_end = "<|object_ref_end|>"; + audio_preproc = std::make_unique(models.a.get()); + } break; case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: { aud_beg = "<|soft_audio_start|>"; @@ -4270,12 +4290,58 @@ struct mtmd_audio_stream_gemma4a : mtmd_audio_stream { bool encode(int n); }; +// Higgs Audio: each 4 s chunk is encoded on its own, so a chunk goes out once it is complete +// a flush encodes the partial chunk as a chunk of its own, the reference also cuts chunks at VAD boundaries +struct mtmd_audio_stream_higgs_audio : mtmd_audio_stream { + int64_t chunk_len = 0; // samples per complete chunk + + std::vector beg_rows; + std::vector end_rows; + bool beg_done = false; + bool end_done = false; + + bool init(const mtmd_audio_stream_params & params) override; + int32_t flush(bool end) override; + int32_t step(llama_token last_token) override; + + int64_t n_chunk_rows(int64_t n) const; + bool encode(int64_t n); +}; + +// VibeVoice-ASR: the encoder is causal, its convs keep their left context in the state +// the reference scales the whole clip to -25 dBFS, here the gain comes from the audio pushed so far (see mtmd_audio_vibevoice_gain) +struct mtmd_audio_stream_vibevoice_asr : mtmd_audio_stream { + mtmd_subctx_enc_state * enc_state = nullptr; + + int64_t n_step = 0; // samples per encoder call + int64_t n_step_max = 0; // largest call, the warmup size + int64_t n_stat = 0; // pushed samples counted in sum_sq and peak + double sum_sq = 0.0; + double peak = 0.0; + + std::vector beg_rows; + std::vector end_rows; + bool beg_done = false; + bool end_done = false; + + ~mtmd_audio_stream_vibevoice_asr() override { + delete enc_state; + } + + bool init(const mtmd_audio_stream_params & params) override; + int32_t flush(bool end) override; + int32_t step(llama_token last_token) override; + + bool encode(int64_t n); +}; + bool mtmd_support_audio_stream(const mtmd_context * ctx) { if (!ctx->ctx_a || !ctx->text_model) { return false; } const projector_type proj = ctx->ctx_a->model.proj_type; - return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A || proj == PROJECTOR_TYPE_GEMMA4A; + return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A || proj == PROJECTOR_TYPE_GEMMA4A || + proj == PROJECTOR_TYPE_HIGGS_AUDIO || proj == PROJECTOR_TYPE_VIBEVOICE_ASR; } mtmd_audio_stream_params mtmd_audio_stream_params_default(void) { @@ -4298,6 +4364,8 @@ mtmd_audio_stream * mtmd_audio_stream_init(mtmd_context * ctx, mtmd_audio_stream case PROJECTOR_TYPE_VOXTRAL_RT: s = std::make_unique(); break; case PROJECTOR_TYPE_LFM3A: s = std::make_unique(); break; case PROJECTOR_TYPE_GEMMA4A: s = std::make_unique(); break; + case PROJECTOR_TYPE_HIGGS_AUDIO: s = std::make_unique(); break; + case PROJECTOR_TYPE_VIBEVOICE_ASR: s = std::make_unique(); break; default: GGML_ABORT("unsupported projector"); } s->ctx = ctx; @@ -5036,6 +5104,210 @@ int32_t mtmd_audio_stream_gemma4a::step(llama_token /* last_token */) { return (int32_t) n_out; } +// encodes one audio entry, the rows are appended to embd +static bool mtmd_audio_stream_encode(mtmd_audio_stream & s, mtmd_image_f32 && img, mtmd_subctx_enc_state * enc_state) { + const int n_out = s.sctx->model.n_output_tokens(&img); + + mtmd_image_f32_batch batch; + batch.is_audio = true; + batch.entries.push_back(std::move(img)); + + std::vector res((size_t) s.n_embd * n_out); + mtmd_subctx_encode_params params; + params.n_threads = s.ctx->n_threads; + params.imgs = &batch; + params.out_embd = &res; + params.enc_state = enc_state; + if (!s.sctx->encode(¶ms)) { + LOG_ERR("%s: failed to encode audio\n", __func__); + return false; + } + s.embd.insert(s.embd.end(), res.begin(), res.end()); + return true; +} + +// rows carry no text: all of them go out at once between the audio markers, last_token is not used +static int32_t mtmd_audio_stream_emit_rows(mtmd_audio_stream & s, const std::vector & beg_rows, const std::vector & end_rows, + bool & beg_done, bool & end_done) { + const int64_t n_avail = (int64_t) (s.embd.size() / s.n_embd); + s.out.clear(); + if (n_avail > 0 || s.ended) { + if (!beg_done) { + beg_done = true; + s.out = beg_rows; + } + s.out.insert(s.out.end(), s.embd.begin(), s.embd.end()); + s.embd.clear(); + if (s.ended && !end_done) { + end_done = true; + s.out.insert(s.out.end(), end_rows.begin(), end_rows.end()); + } + } + const int64_t n_out = (int64_t) (s.out.size() / s.n_embd); + if (n_out == 0) { + return s.ended ? -1 : 0; + } + s.n_rows += n_out; + s.n_rows_done += n_avail; // the markers are not audio rows + return (int32_t) n_out; +} + +// +// Higgs Audio +// + +bool mtmd_audio_stream_higgs_audio::init(const mtmd_audio_stream_params & params) { + const mtmd_hparams * hparams = &sctx->model.hparams; + if (params.n_prompt > 0) { + LOG_ERR("%s: Higgs Audio takes no prompt, its audio is a chat turn\n", __func__); + return false; + } + if (!ctx->audio_preproc) { + LOG_ERR("%s: the audio preprocessor is not initialized\n", __func__); + return false; + } + chunk_len = (int64_t) hparams->audio_chunk_len * hparams->audio_sample_rate; + frames_per_tok = 8; // stride-2 conv, avgpool, stride-2 projector conv + samples_per_tok = frames_per_tok * hop; + return mtmd_audio_stream_marker_rows(ctx, ctx->aud_beg, beg_rows) && mtmd_audio_stream_marker_rows(ctx, ctx->aud_end, end_rows); +} + +// rows of a chunk with n samples, see mtmd_audio_preprocessor_higgs_audio +int64_t mtmd_audio_stream_higgs_audio::n_chunk_rows(int64_t n) const { + const int64_t n_frames = n / hop; + if (n_frames < 3) { + return 0; + } + return ((n_frames + 1) / 2 / 2 + 1) / 2; +} + +bool mtmd_audio_stream_higgs_audio::encode(int64_t n) { + mtmd_audio_preproc_out pre = ctx->audio_preproc->preprocess(buf.data(), (size_t) n); + if (pre.entries.empty() || !ctx->audio_preproc->compute(pre)) { + LOG_ERR("%s: unable to preprocess the audio\n", __func__); + return false; + } + for (auto & entry : pre.entries) { + if (!mtmd_audio_stream_encode(*this, std::move(entry), nullptr)) { + return false; + } + } + buf.erase(buf.begin(), buf.begin() + n); + buf_pos += n; + return true; +} + +int32_t mtmd_audio_stream_higgs_audio::flush(bool end) { + const int64_t n_pending = n_samples - buf_pos; + n_rows_flushed = n_rows_done + (int64_t) (embd.size() / n_embd) + (n_pending / chunk_len) * n_chunk_rows(chunk_len) + n_chunk_rows(n_pending % chunk_len); + pending_flush = true; + ended = end; + return 0; +} + +int32_t mtmd_audio_stream_higgs_audio::step(llama_token /* last_token */) { + const bool flush = pending_flush; + pending_flush = false; + + while ((int64_t) buf.size() >= chunk_len) { + if (!encode(chunk_len)) { + return -2; + } + } + // a tail too short for a row is kept until the next flush, or dropped at the end + if ((flush || ended) && !buf.empty()) { + if (n_chunk_rows((int64_t) buf.size()) > 0) { + if (!encode((int64_t) buf.size())) { + return -2; + } + } else if (ended) { + buf_pos += (int64_t) buf.size(); + buf.clear(); + } + } + return mtmd_audio_stream_emit_rows(*this, beg_rows, end_rows, beg_done, end_done); +} + +// +// VibeVoice-ASR +// + +bool mtmd_audio_stream_vibevoice_asr::init(const mtmd_audio_stream_params & params) { + const mtmd_hparams * hparams = &sctx->model.hparams; + if (params.n_prompt > 0) { + LOG_ERR("%s: VibeVoice-ASR takes no prompt, its audio is a chat turn\n", __func__); + return false; + } + hop = hparams->audio_samples_per_token(); + frames_per_tok = 1; + samples_per_tok = hop; + n_step = std::max(1, ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 + hop - 1) / hop) * hop; + n_step_max = std::max(n_step, (int64_t) hparams->audio_chunk_len * hparams->audio_sample_rate / hop * hop); + + enc_state = sctx->enc_state_init(0); + if (!enc_state) { + return false; + } + return mtmd_audio_stream_marker_rows(ctx, ctx->aud_beg, beg_rows) && mtmd_audio_stream_marker_rows(ctx, ctx->aud_end, end_rows); +} + +// n is a multiple of hop, see mtmd_audio_preprocessor_vibevoice_asr for the gain +bool mtmd_audio_stream_vibevoice_asr::encode(int64_t n) { + const double gain = mtmd_audio_vibevoice_gain(sum_sq, n_stat, peak); + + std::vector data((size_t) n); + for (int64_t i = 0; i < n; i++) { + data[i] = (float) (buf[i] * gain); + } + mtmd_image_f32 img; + img.set_size({(int) n, 1}, false, /* is_audio */ true); + img.cpy_buf(data); + if (!mtmd_audio_stream_encode(*this, std::move(img), enc_state)) { + return false; + } + buf.erase(buf.begin(), buf.begin() + n); + buf_pos += n; + return true; +} + +int32_t mtmd_audio_stream_vibevoice_asr::flush(bool end) { + n_rows_flushed = end ? (n_samples + hop - 1) / hop : n_samples / hop; + pending_flush = true; + ended = end; + return 0; +} + +int32_t mtmd_audio_stream_vibevoice_asr::step(llama_token /* last_token */) { + const bool flush = pending_flush; + pending_flush = false; + + // the padding added at the end does not count in the gain + for (int64_t i = n_stat - buf_pos; i < (int64_t) buf.size(); i++) { + sum_sq += (double) buf[i] * buf[i]; + peak = std::max(peak, (double) std::fabs(buf[i])); + } + n_stat = n_samples; + if (ended && n_samples % hop != 0) { + add_silence(hop - n_samples % hop); + n_stat = n_samples; + } + + // whole rows only: a partial row would put its zero padding into the state + int64_t n = (int64_t) buf.size(); + n -= n % hop; + if (!flush && !ended) { + n -= n % n_step; + } + while (n > 0) { + const int64_t m = std::min(n, n_step_max); + if (!encode(m)) { + return -2; + } + n -= m; + } + return mtmd_audio_stream_emit_rows(*this, beg_rows, end_rows, beg_done, end_done); +} + int64_t mtmd_audio_stream_get_n_rows(const mtmd_audio_stream * s) { return s->n_rows_done; } diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index af1e42dbd..1724a6702 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -739,14 +739,22 @@ json convert_transcriptions_to_chatcmpl( if (response_format != "json") { throw std::invalid_argument("Only 'json' response_format is supported for transcription"); } - const common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language); - if (prompt.empty()) { + common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language, 0.0, prompt); + if (preset.needs_audio_length) { + mtmd_helper_media_meta meta; + if (mtmd_helper_get_media_meta(out_files[0].data(), out_files[0].size(), mtmd_helper_init_opt_default(), &meta) != 0 || + meta.type != MTMD_HELPER_MEDIA_TYPE_AUDIO || meta.duration < 0) { + throw std::invalid_argument("the input file is not a supported audio file"); + } + preset = common_chat_get_asr_prompt(tmpls, language, meta.duration, prompt); + } + if (prompt.empty() || preset.has_context) { prompt = preset.user; } if (!language.empty() && !preset.has_language) { prompt += string_format(" (language: %s)", language.c_str()); } - prompt += get_media_marker(); + prompt = preset.media_first ? get_media_marker() + prompt : prompt + get_media_marker(); json messages = json::array(); if (!preset.system.empty()) { diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp index 8ed3dd535..e747d32d7 100644 --- a/tools/server/server-realtime.cpp +++ b/tools/server/server-realtime.cpp @@ -155,6 +155,7 @@ struct realtime_conversation { bool started = false; // something went into the slot, the session config is fixed bool turn_open = false; // a user turn is being filled bool has_audio = false; // the open turn has audio not committed yet + int64_t n_turn_samples = 0; // audio of the open turn, at the model rate bool tool_results = false; // tool results went in after the last response, the next one answers them int n_turns = 0; std::string next_header; // text that goes in before the next user turn @@ -376,8 +377,10 @@ struct realtime_conversation { if (!resample(samples) || samples.empty() || !ensure_turn()) { return; } + const int64_t n = (int64_t) samples.size(); if (control(SERVER_CONTROL_ACTION_AUDIO_APPEND, {}, std::move(samples))) { has_audio = true; + n_turn_samples += n; } } @@ -410,7 +413,8 @@ struct realtime_conversation { if (!control(SERVER_CONTROL_ACTION_AUDIO_END)) { return false; } - has_audio = false; + has_audio = false; + n_turn_samples = 0; std::lock_guard lock(mutex); const json prev = last_item_id.empty() ? json(nullptr) : json(last_item_id); @@ -424,6 +428,7 @@ struct realtime_conversation { } void on_commit() { + const int64_t n_audio = n_turn_samples; if (!commit() || type != "transcription") { return; } @@ -433,11 +438,19 @@ struct realtime_conversation { transcribe_item_id = last_item_id; transcript.clear(); } + // some models (VibeVoice-ASR) take an instruction after the audio, with its length + const auto * tmpls = deps.meta.chat_params.tmpls.get(); + common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language); + if (preset.needs_audio_length) { + preset = common_chat_get_asr_prompt(tmpls, language, (double) n_audio / mtmd_get_audio_sample_rate(deps.mctx)); + } + const std::string after_audio = preset.media_first ? preset.user : ""; + server_control_response response; response.n_predict = max_output_tokens; response.out_mode = MTMD_HELPER_GEN_AUDIO_OUT_TEXT; response.sampling = response_sampling(nullptr); - if (!control(SERVER_CONTROL_ACTION_RESPONSE_CREATE, user_footer + assistant_header, {}, response)) { + if (!control(SERVER_CONTROL_ACTION_RESPONSE_CREATE, after_audio + user_footer + assistant_header, {}, response)) { return; } control(SERVER_CONTROL_ACTION_CLEAR);