mtmd: support higgs v3 STT and vibevoice-ASR (#32329)

* mtmd: support higgs v3 and vibevoice-ASR

* use proper mapping

* address review comment

* nits

* rm hacky get_audio_length, add mtmd_helper_get_media_meta
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-11-15 21:25:34 +01:00
1 parent 18b74fa41d
commit fc504f91d8
24 files changed
+1101 -18

No files matched your search

+18 -1
View File
@@ -827,6 +827,11 @@ static std::string string_lower(std::string s) {
return s;
}
// VibeVoice-ASR: default system prompt of the template written by the converter
static bool is_vibevoice_asr_template(const std::string & src) {
return src.find("transcribes audio input into text output in JSON format") != std::string::npos;
}
// LFM3-Audio: language code -> name
static std::string lfm3_language_name(const std::string & language) {
static const std::map<std::string, std::string> names = {
@@ -847,7 +852,8 @@ static std::string lfm3_asr_system_prompt(const std::string & language) {
return "Perform ASR in " + lfm3_language_name(language) + ".";
}
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) {
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language,
double audio_length, const std::string & context) {
common_chat_prompt_preset asr_preset;
asr_preset.system = "";
asr_preset.user = "Transcribe audio to text";
@@ -861,6 +867,17 @@ common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates
asr_preset.system = lfm3_asr_system_prompt(language);
asr_preset.user = "";
asr_preset.has_language = true;
} else if (is_vibevoice_asr_template(src)) {
// without the audio length the model does not know where the audio ends and loops
const char * keys = "transcribe it with these keys: Start time, End time, Speaker ID, Content";
if (context.empty()) {
asr_preset.user = string_format("\nThis is a %.2f seconds audio, please %s", audio_length, keys);
} else {
asr_preset.user = string_format("\nThis is a %.2f seconds audio, with extra info: %s\n\nPlease %s", audio_length, context.c_str(), keys);
asr_preset.has_context = true;
}
asr_preset.media_first = true;
asr_preset.needs_audio_length = true;
}
}
+8 -2
View File
@@ -491,10 +491,16 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
struct common_chat_prompt_preset {
std::string system;
std::string user;
bool has_language = false; // the preset already names the requested language
bool has_language = false; // the preset already names the requested language
bool has_context = false; // the user text already holds the context
bool media_first = false; // the media goes before the user text
bool needs_audio_length = false; // the user text depends on the audio length, call again with it
};
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "");
// audio_length: in seconds, 0 if unknown
// context: extra info from the user (e.g. names, hotwords), only used by some presets, see has_context
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "",
double audio_length = 0.0, const std::string & context = "");
// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
// language and voice are optional, the template defines their defaults
+4
View File
@@ -138,6 +138,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"Grok1ForCausalLM": "grok",
"GrokForCausalLM": "grok",
"GroveMoeForCausalLM": "grovemoe",
"HiggsAudio3Model": "higgs_audio",
"HunYuanDenseV1ForCausalLM": "hunyuan",
"HunYuanMoEV1ForCausalLM": "hunyuan",
"HunYuanVLForConditionalGeneration": "hunyuan",
@@ -325,6 +326,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"UMT5Model": "t5",
"UltravoxModel": "ultravox",
"UnlimitedOCRForCausalLM": "deepseek",
"VibeVoiceAsrForConditionalGeneration": "vibevoice_asr",
"VLlama3ForCausalLM": "llama",
"VllmDecision2Model": "vllm_decision2",
"VoxtralForConditionalGeneration": "llama",
@@ -372,6 +374,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"Granite4VisionForConditionalGeneration": "granite",
"GraniteSpeechForConditionalGeneration": "granite",
"GraniteSpeechPlusForConditionalGeneration": "granite",
"HiggsAudio3Model": "higgs_audio",
"HunYuanVLForConditionalGeneration": "hunyuan",
"Idefics3ForConditionalGeneration": "smolvlm",
"Inkling2ForConditionalGeneration": "inkling2",
@@ -431,6 +434,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"Step3p7ForConditionalGeneration": "step3",
"UltravoxModel": "ultravox",
"UnlimitedOCRForCausalLM": "deepseek",
"VibeVoiceAsrForConditionalGeneration": "vibevoice_asr",
"VoxtralForConditionalGeneration": "ultravox",
"VoxtralRealtimeForConditionalGeneration": "ultravox",
"YoutuVLForConditionalGeneration": "youtuvl",
+1 -1
View File
@@ -3475,7 +3475,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
# Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older
# Kimi-Linear-48B architecture and cannot load K3 (no attention residuals,
# latent MoE, situ, ...). Route on the top-level architecture instead.
if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK31ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration"):
if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK31ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration", "HiggsAudio3Model"):
return arch
# if "architectures" is found in the sub-config, use that instead
+74
View File
@@ -0,0 +1,74 @@
from __future__ import annotations
from typing import Any, Callable, TYPE_CHECKING
if TYPE_CHECKING:
from torch import Tensor
from .base import ModelBase, gguf
from .qwen import Qwen3Model
from .ultravox import WhisperEncoderModel
# plain ChatML as built by transcribe.py upstream, the Qwen3 template renders past assistant turns differently from the last one
HIGGS_AUDIO_CHAT_TEMPLATE = (
"{%- for message in messages -%}"
"{{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' -}}"
"{%- endfor -%}"
"{%- if add_generation_prompt -%}"
"{{- '<|im_start|>assistant\\n' -}}"
"{%- if enable_thinking is defined and enable_thinking is false -%}"
"{{- '<think>\\n\\n</think>\\n\\n' -}}"
"{%- endif -%}"
"{%- endif -%}"
)
@ModelBase.register("HiggsAudio3Model")
@ModelBase.example("bosonai/higgs-audio-v3-stt")
class HiggsAudioModel(Qwen3Model):
model_arch = gguf.MODEL_ARCH.QWEN3
def set_vocab(self):
super().set_vocab()
self.gguf_writer.add_chat_template(HIGGS_AUDIO_CHAT_TEMPLATE)
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, gen = item
# the audio_* tensors are dropped by TextModel, rename the text head before that
if name == "audio_decoder_proj.text_lm_head.weight":
name = "lm_head.weight"
else:
name = "model." + name
return super().filter_tensors((name, gen))
@ModelBase.register("HiggsAudio3Model")
class HiggsAudioWhisperEncoderModel(WhisperEncoderModel):
has_vision_encoder = False
has_audio_encoder = True
def get_audio_config(self) -> dict[str, Any] | None:
return self.global_config["audio_encoder_config"]
def tensor_force_quant(self, name, new_name, bid, n_dims):
# depthwise conv kernel, same as the whisper convs
if name == "audio_encoder_proj.temporal.weight":
return gguf.GGMLQuantizationType.F16
return super().tensor_force_quant(name, new_name, bid, n_dims)
def set_gguf_parameters(self):
assert self.global_config["projector_type"] == "mlp" and self.global_config["projector_temporal_downsample"] == 2
assert self.global_config["audio_adapter_type"] == "stack" and not self.global_config.get("duplicate_audio_triplet", False)
super().set_gguf_parameters()
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.HIGGS_AUDIO)
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, _ = item
if not name.startswith(("audio_tower.", "audio_encoder_proj.")):
return None
return super().filter_tensors(item)
+157
View File
@@ -0,0 +1,157 @@
from __future__ import annotations
from typing import Any, Callable, Iterable, TYPE_CHECKING
if TYPE_CHECKING:
from torch import Tensor
from .base import MmprojModel, ModelBase, gguf
from .qwen import Qwen2Model
# the upstream chat_template.jinja is a processor template (list content, audio parts), use plain ChatML with its default system prompt
VIBEVOICE_ASR_CHAT_TEMPLATE = (
"{%- if messages[0]['role'] == 'system' -%}"
"{{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' -}}"
"{%- set loop_messages = messages[1:] -%}"
"{%- else -%}"
"{{- '<|im_start|>system\\nYou are a helpful assistant that transcribes audio input into text output in JSON format.<|im_end|>\\n' -}}"
"{%- set loop_messages = messages -%}"
"{%- endif -%}"
"{%- for message in loop_messages -%}"
"{{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' -}}"
"{%- endfor -%}"
"{%- if add_generation_prompt -%}"
"{{- '<|im_start|>assistant\\n' -}}"
"{%- endif -%}"
)
@ModelBase.register("VibeVoiceAsrForConditionalGeneration")
@ModelBase.example("microsoft/VibeVoice-ASR-HF")
class VibeVoiceAsrModel(Qwen2Model):
model_arch = gguf.MODEL_ARCH.QWEN2
def set_vocab(self):
super().set_vocab()
self.gguf_writer.add_chat_template(VIBEVOICE_ASR_CHAT_TEMPLATE)
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, _ = item
if not name.startswith("language_model."):
return None
return super().filter_tensors(item)
@ModelBase.register("VibeVoiceAsrForConditionalGeneration")
class VibeVoiceAsrAudioModel(MmprojModel):
has_vision_encoder = False
has_audio_encoder = True
# values hard-coded in tools/mtmd/models/vibevoice-asr.cpp
_encoder_consts = {"channels": 1, "kernel_size": 7, "num_filters": 32, "ffn_expansion": 4, "hidden_act": "gelu", "rms_norm_eps": 1e-5}
_encoders = ("acoustic_tokenizer_encoder", "semantic_tokenizer_encoder")
def get_audio_config(self) -> dict[str, Any] | None:
c = self.global_config["acoustic_tokenizer_encoder_config"]
return {
**c,
"intermediate_size": c["ffn_expansion"] * c["hidden_size"],
"num_attention_heads": 1,
"num_hidden_layers": sum(c["depths"]),
}
def set_gguf_parameters(self):
super().set_gguf_parameters()
for enc in self._encoders:
c = self.global_config[f"{enc}_config"]
for k, v in self._encoder_consts.items():
assert c[k] == v, f"{enc}.{k} = {c[k]}, expected {v}"
assert c["depths"] == self.hparams["depths"] and c["downsampling_ratios"] == self.hparams["downsampling_ratios"]
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.VIBEVOICE_ASR)
# the input is the raw waveform, one channel takes the place of the mel bins
self.gguf_writer.add_audio_num_mel_bins(1)
self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams["rms_norm_eps"])
self.gguf_writer.add_audio_conv_kernel_size(self.hparams["kernel_size"])
# stride of each stage, the stem has none
self.gguf_writer.add_audio_patch_sizes([1] + [int(r) for r in self.hparams["downsampling_ratios"]])
self.gguf_writer.add_audio_depths([int(d) for d in self.hparams["depths"]])
def tensor_force_quant(self, name, new_name, bid, n_dims):
if ".dw." in new_name:
return gguf.GGMLQuantizationType.F32 # ggml_ssm_conv
if ".conv." in new_name and new_name.endswith(".weight"):
return gguf.GGMLQuantizationType.F16
return super().tensor_force_quant(name, new_name, bid, n_dims)
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, _ = item
if not name.startswith(cls._encoders + ("multi_modal_projector.",)):
return None
return super().filter_tensors(item)
_block_tensors = {
"norm.weight": gguf.MODEL_TENSOR.A_ENC_CNX_NORM,
"mixer.conv.weight": gguf.MODEL_TENSOR.A_ENC_CNX_DW,
"mixer.conv.bias": gguf.MODEL_TENSOR.A_ENC_CNX_DW,
"gamma": gguf.MODEL_TENSOR.A_ENC_CNX_LS1,
"ffn_norm.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_NORM,
"ffn.linear1.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_UP,
"ffn.linear1.bias": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_UP,
"ffn.linear2.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
"ffn.linear2.bias": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
"ffn_gamma": gguf.MODEL_TENSOR.A_ENC_CNX_LS2,
}
# mm.a.mlp.{0,1} + mm.a.mlp_norm.0 for the acoustic connector, {2,3} + 2 for the semantic one
_proj_tensors = {
"acoustic_linear_1": (gguf.MODEL_TENSOR.A_MMPROJ, 0),
"acoustic_norm": (gguf.MODEL_TENSOR.A_MMPROJ_NORM, 0),
"acoustic_linear_2": (gguf.MODEL_TENSOR.A_MMPROJ, 1),
"semantic_linear_1": (gguf.MODEL_TENSOR.A_MMPROJ, 2),
"semantic_norm": (gguf.MODEL_TENSOR.A_MMPROJ_NORM, 2),
"semantic_linear_2": (gguf.MODEL_TENSOR.A_MMPROJ, 3),
}
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# the layer scales (gamma) have no suffix upstream
suffix = "." + name.rsplit(".", 1)[-1] if name.endswith((".weight", ".bias")) else ".weight"
if name.startswith("multi_modal_projector."):
key, idx = self._proj_tensors[name.split(".")[1]]
yield (self.format_tensor_name(key, idx, suffix=suffix), data_torch)
return
enc, _, rest = name.partition(".")
enc_id = self._encoders.index(enc)
parts = rest.split(".")
depths = [int(d) for d in self.hparams["depths"]]
# causal convs of the stages: 0 = stem, 1..6 = downsample, 7 = head
if rest.startswith("stem.conv."):
conv_id = 0
elif parts[0] == "conv_layers" and parts[2] == "conv":
conv_id = int(parts[1]) + 1
elif parts[0] == "head":
conv_id = len(depths)
else:
conv_id = None
if conv_id is not None:
yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_ENC_CNX_CONV].format(bid=enc_id, xid=conv_id) + suffix, data_torch)
return
# ConvNeXt blocks are numbered per stage upstream and flat here
if parts[0] == "stem":
stage, blk, tail = 0, int(parts[2]), ".".join(parts[3:])
else:
assert parts[0] == "conv_layers" and parts[2] == "stage", name
stage, blk, tail = int(parts[1]) + 1, int(parts[3]), ".".join(parts[4:])
xid = sum(depths[:stage]) + blk
key = self._block_tensors[tail]
if key == gguf.MODEL_TENSOR.A_ENC_CNX_DW and tail.endswith(".weight"):
data_torch = data_torch.squeeze(1) # [C, 1, K] -> [C, K] for ggml_ssm_conv
yield (gguf.TENSOR_NAMES[key].format(bid=enc_id, xid=xid) + suffix, data_torch)
+29
View File
@@ -1386,6 +1386,14 @@ class MODEL_TENSOR(IntEnum):
A_ENC_SEANET_RES_CONV1 = auto() # residual unit, dilated conv
A_ENC_SEANET_RES_CONV2 = auto() # residual unit, pointwise conv
A_ENC_SEANET_SCALE_CONV = auto() # strided downsample conv
A_ENC_CNX_CONV = auto() # vibevoice: causal conv of a stage (stem, downsample, head), bid = encoder
A_ENC_CNX_NORM = auto() # vibevoice: ConvNeXt block, bid = encoder, xid = block
A_ENC_CNX_DW = auto()
A_ENC_CNX_LS1 = auto()
A_ENC_CNX_FFN_NORM = auto()
A_ENC_CNX_FFN_UP = auto()
A_ENC_CNX_FFN_DOWN = auto()
A_ENC_CNX_LS2 = auto()
A_ENC_ATTN_SCALE = auto() # layer scale (gamma) on the attn output
A_ENC_FFN_SCALE_LS = auto() # layer scale (gamma) on the FFN output
A_ENC_SPEAKER_PROJ = auto() # voice latent -> backbone embd
@@ -1419,6 +1427,7 @@ class MODEL_TENSOR(IntEnum):
A_GEN_WAV_OUTPUT = auto()
A_MMPROJ = auto()
A_MMPROJ_FC = auto()
A_MMPROJ_NORM = auto() # vibevoice: norm after mm.a.mlp.{bid}
A_MM_NORM_PRE = auto()
A_MM_NORM_MID = auto()
A_MM_EMBEDDING = auto() # gemma3n
@@ -2317,6 +2326,14 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_ENC_SEANET_RES_CONV1: "a.seanet.blk.{bid}.res_conv1",
MODEL_TENSOR.A_ENC_SEANET_RES_CONV2: "a.seanet.blk.{bid}.res_conv2",
MODEL_TENSOR.A_ENC_SEANET_SCALE_CONV: "a.seanet.blk.{bid}.scale_conv",
MODEL_TENSOR.A_ENC_CNX_CONV: "a.cnx.{bid}.conv.{xid}",
MODEL_TENSOR.A_ENC_CNX_NORM: "a.cnx.{bid}.blk.{xid}.norm",
MODEL_TENSOR.A_ENC_CNX_DW: "a.cnx.{bid}.blk.{xid}.dw",
MODEL_TENSOR.A_ENC_CNX_LS1: "a.cnx.{bid}.blk.{xid}.ls1",
MODEL_TENSOR.A_ENC_CNX_FFN_NORM: "a.cnx.{bid}.blk.{xid}.ffn_norm",
MODEL_TENSOR.A_ENC_CNX_FFN_UP: "a.cnx.{bid}.blk.{xid}.ffn_up",
MODEL_TENSOR.A_ENC_CNX_FFN_DOWN: "a.cnx.{bid}.blk.{xid}.ffn_down",
MODEL_TENSOR.A_ENC_CNX_LS2: "a.cnx.{bid}.blk.{xid}.ls2",
MODEL_TENSOR.A_ENC_ATTN_SCALE: "a.blk.{bid}.ls1",
MODEL_TENSOR.A_ENC_FFN_SCALE_LS: "a.blk.{bid}.ls2",
MODEL_TENSOR.A_ENC_SPEAKER_PROJ: "a.speaker_proj",
@@ -2349,6 +2366,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_GEN_WAV_OUTPUT: "a.gen.wav.output",
MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}",
MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc",
MODEL_TENSOR.A_MMPROJ_NORM: "mm.a.mlp_norm.{bid}",
MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre",
MODEL_TENSOR.A_MM_NORM_MID: "mm.a.norm_mid",
MODEL_TENSOR.A_MM_INP_PROJ: "mm.a.input_projection", # gemma3n
@@ -2641,6 +2659,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.A_ENC_RVQ_CODEBOOK,
MODEL_TENSOR.A_MMPROJ,
MODEL_TENSOR.A_MMPROJ_FC,
MODEL_TENSOR.A_MMPROJ_NORM,
MODEL_TENSOR.A_MM_NORM_PRE,
MODEL_TENSOR.A_MM_NORM_MID,
MODEL_TENSOR.A_MM_CODE_EMBD,
@@ -2741,6 +2760,14 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.A_ENC_SEANET_RES_CONV1,
MODEL_TENSOR.A_ENC_SEANET_RES_CONV2,
MODEL_TENSOR.A_ENC_SEANET_SCALE_CONV,
MODEL_TENSOR.A_ENC_CNX_CONV,
MODEL_TENSOR.A_ENC_CNX_NORM,
MODEL_TENSOR.A_ENC_CNX_DW,
MODEL_TENSOR.A_ENC_CNX_LS1,
MODEL_TENSOR.A_ENC_CNX_FFN_NORM,
MODEL_TENSOR.A_ENC_CNX_FFN_UP,
MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
MODEL_TENSOR.A_ENC_CNX_LS2,
MODEL_TENSOR.A_ENC_ATTN_SCALE,
MODEL_TENSOR.A_ENC_FFN_SCALE_LS,
MODEL_TENSOR.A_ENC_SPEAKER_PROJ,
@@ -7155,6 +7182,8 @@ class VisionProjectorType:
QWEN2A = "qwen2a" # audio
QWEN3A = "qwen3a" # audio
GLMA = "glma" # audio
HIGGS_AUDIO = "higgs_audio" # audio
VIBEVOICE_ASR = "vibevoice_asr" # audio
QWEN25O = "qwen2.5o" # omni
VOXTRAL = "voxtral"
VOXTRAL_RT = "voxtral_rt"
+2
View File
@@ -2973,6 +2973,7 @@ class TensorNameMap:
MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV: (
"encoder.down_sample_layer.0", # mimo-audio-tokenizer
"audio_encoder_proj.temporal", # higgs-audio
),
MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM: (
@@ -3026,6 +3027,7 @@ class TensorNameMap:
"mlp.model.{bid}", # lfm3a
"audio_tower.proj{bid}", # qwen3omni
"sound_projection.linear{bid}", # parakeet (linear1, linear2)
"audio_encoder_proj.linear{bid}", # higgs-audio (linear1, linear2)
),
MODEL_TENSOR.A_MMPROJ_FC: (
+1
View File
@@ -63,6 +63,7 @@ add_library(mtmd
models/nemotron-transcribe.cpp
models/muse-glimmer.cpp
models/muse-spark-audio.cpp
models/vibevoice-asr.cpp
models/llama4.cpp
models/llava.cpp
models/minicpmv.cpp
+15
View File
@@ -268,6 +268,18 @@ struct mtmd_graph_granite_speech : mtmd_graph {
ggml_cgraph * build() override;
};
struct mtmd_graph_vibevoice_asr : mtmd_graph {
mtmd_graph_vibevoice_asr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
std::map<std::string, ggml_tensor *> state_in;
mutable std::vector<std::pair<std::string, ggml_tensor *>> state_out;
ggml_tensor * with_left_ctx(ggml_tensor * x, const std::string & name) const;
ggml_tensor * causal_conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & name) const;
ggml_tensor * encode(int e, ggml_tensor * x) const;
};
struct mtmd_graph_muse_spark_audio : mtmd_graph {
mtmd_graph_muse_spark_audio(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
@@ -643,6 +655,9 @@ std::vector<c2w_state_slot> list_lfm3a_state_slots(const mtmd_hparams & hparams,
// state slots of the streaming Gemma 4 audio encoder (gemma4a)
std::vector<c2w_state_slot> list_gemma4a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
// vibevoice-asr: the left context of every causal conv, a zero state is the causal zero padding
std::vector<c2w_state_slot> list_vibevoice_asr_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
struct mtmd_graph_kimik25 : mtmd_graph {
mtmd_graph_kimik25(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
+138
View File
@@ -0,0 +1,138 @@
#include "models.h"
// VibeVoice-ASR audio tower: two causal ConvNeXt pyramids (acoustic and semantic) over the raw 24 kHz waveform
// each one is a VAE tokenizer encoder, its latents go through their own connector and the two are summed
// see modeling_vibevoice_acoustic_tokenizer.py and modeling_vibevoice_asr.py in transformers
//
// tensors are T-first [T, C] around the convs and C-first [C, T] in the blocks
static std::string cnx_slot(int e, const char * kind, int i) {
return "e" + std::to_string(e) + "_" + kind + "_" + std::to_string(i);
}
std::vector<c2w_state_slot> list_vibevoice_asr_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
std::vector<c2w_state_slot> slots;
slots.push_back({"pos", 1, 1}); // unused, set_enc_state_in() expects it
const int n_stage = (int) hparams.audio_patch_sizes.size();
for (int e = 0; e < 2; ++e) {
const auto & enc = model.cnx_enc[e];
for (int i = 0; i <= n_stage; ++i) {
const int64_t stride = i < n_stage ? hparams.audio_patch_sizes[i] : 1;
slots.push_back({cnx_slot(e, "conv", i), enc.conv_w[i]->ne[0] - stride, enc.conv_w[i]->ne[1]});
}
for (int il = 0; il < (int) enc.blocks.size(); ++il) {
const auto * w = enc.blocks[il].conv_w;
slots.push_back({cnx_slot(e, "dw", il), w->ne[0] - 1, w->ne[1]});
}
}
return slots;
}
// x: [T, C] -> [left + T, C], the left context comes from the state, its new value is the tail
ggml_tensor * mtmd_graph_vibevoice_asr::with_left_ctx(ggml_tensor * x, const std::string & name) const {
ggml_tensor * left = state_in.at(name);
x = ggml_concat(ctx0, left, x, 0);
state_out.push_back({name, ggml_cont(ctx0, ggml_view_2d(ctx0, x, left->ne[0], x->ne[1], x->nb[1], (x->ne[0] - left->ne[0]) * x->nb[0]))});
return x;
}
// x: [T, IC], w: [K, IC, OC] -> [T / stride, OC]
ggml_tensor * mtmd_graph_vibevoice_asr::causal_conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & name) const {
x = with_left_ctx(x, name);
ggml_tensor * y = ggml_conv_1d(ctx0, w, x, stride, 0, 1);
y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[1]);
return ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
}
// x: [T, 1] -> [C_latent, T / hop]
ggml_tensor * mtmd_graph_vibevoice_asr::encode(int e, ggml_tensor * x) const {
const auto & enc = model.cnx_enc[e];
const int n_stage = (int) hparams.audio_patch_sizes.size();
ggml_tensor * cur = x;
int bid = 0;
for (int stage = 0; stage < n_stage; ++stage) {
cur = causal_conv(cur, enc.conv_w[stage], enc.conv_b[stage], hparams.audio_patch_sizes[stage], cnx_slot(e, "conv", stage));
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
cb(cur, "stage_in", stage);
const int64_t n_pos = cur->ne[1];
for (int i = 0; i < hparams.audio_depths[stage]; ++i, ++bid) {
const auto & layer = enc.blocks[bid];
// causal depthwise conv mixer
ggml_tensor * h = build_norm(cur, layer.ln_1_w, nullptr, NORM_TYPE_RMS, hparams.eps, bid);
h = ggml_cont(ctx0, ggml_transpose(ctx0, h));
h = with_left_ctx(h, cnx_slot(e, "dw", bid));
h = ggml_reshape_3d(ctx0, h, h->ne[0], h->ne[1], 1);
h = ggml_ssm_conv(ctx0, h, layer.conv_w);
h = ggml_reshape_2d(ctx0, h, h->ne[0], n_pos);
h = ggml_add(ctx0, h, layer.conv_b);
cur = ggml_add(ctx0, cur, ggml_mul(ctx0, h, layer.ls_1_w));
h = build_norm(cur, layer.ln_2_w, nullptr, NORM_TYPE_RMS, hparams.eps, bid);
h = build_ffn(h,
layer.ff_up_w, layer.ff_up_b,
nullptr, nullptr,
layer.ff_down_w, layer.ff_down_b,
FFN_GELU_ERF, bid);
cur = ggml_add(ctx0, cur, ggml_mul(ctx0, h, layer.ls_2_w));
cb(cur, "block_out", bid);
}
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
}
cur = causal_conv(cur, enc.conv_w[n_stage], enc.conv_b[n_stage], 1, cnx_slot(e, "conv", n_stage));
return ggml_cont(ctx0, ggml_transpose(ctx0, cur));
}
ggml_cgraph * mtmd_graph_vibevoice_asr::build() {
const int64_t hop = hparams.audio_samples_per_token();
const int64_t n_frames = img.nx() / hop;
const int64_t n_ctx = img.audio_n_ctx / hop;
GGML_ASSERT(img.nx() % hop == 0 && img.audio_n_ctx % hop == 0 && n_ctx < n_frames);
for (const auto & slot : list_vibevoice_asr_state_slots(hparams, model)) {
ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
ggml_set_name(t, ("state_in_" + slot.name).c_str());
ggml_set_input(t);
state_in[slot.name] = t;
}
state_out.push_back({"pos", ggml_scale_bias(ctx0, state_in.at("pos"), 1.0f, (float) n_frames)});
ggml_tensor * inp = build_inp_raw(1);
inp = ggml_reshape_2d(ctx0, inp, img.nx(), 1);
// the leading context frames only warm up the causal convs
auto drop_ctx = [&](ggml_tensor * t) {
return ggml_view_2d(ctx0, t, t->ne[0], n_frames - n_ctx, t->nb[1], n_ctx * t->nb[1]);
};
// the reference adds VAE noise to the acoustic latents, skipped here to keep the output deterministic
ggml_tensor * acoustic = drop_ctx(encode(0, inp));
ggml_tensor * semantic = drop_ctx(encode(1, inp));
cb(acoustic, "acoustic_latents", -1);
cb(semantic, "semantic_latents", -1);
acoustic = ggml_add(ctx0, build_mm(model.mm_0_w, acoustic), model.mm_0_b);
acoustic = build_norm(acoustic, model.mm_norm_pre_w, nullptr, NORM_TYPE_RMS, 1e-6f, -1);
acoustic = ggml_add(ctx0, build_mm(model.mm_1_w, acoustic), model.mm_1_b);
semantic = ggml_add(ctx0, build_mm(model.mm_2_w, semantic), model.mm_2_b);
semantic = build_norm(semantic, model.mm_norm_mid_w, nullptr, NORM_TYPE_RMS, 1e-6f, -1);
semantic = ggml_add(ctx0, build_mm(model.mm_3_w, semantic), model.mm_3_b);
ggml_tensor * cur = ggml_add(ctx0, acoustic, semantic);
cb(cur, "projected", -1);
// the embeddings must stay the last node
for (const auto & s : state_out) {
ggml_set_name(s.second, ("state_out_" + s.first).c_str());
ggml_set_output(s.second);
ggml_build_forward_expand(gf, s.second);
}
ggml_build_forward_expand(gf, cur);
return gf;
}
+14 -1
View File
@@ -2,7 +2,7 @@
ggml_cgraph * mtmd_graph_whisper_enc::build() {
const int n_frames = img.nx();
const int n_pos = n_frames / 2;
const int n_pos = (n_frames + 1) / 2; // stride-2 conv1d with padding 1
GGML_ASSERT(model.position_embeddings->ne[1] >= n_pos);
ggml_tensor * inp = build_inp_raw(1);
@@ -77,6 +77,19 @@ ggml_cgraph * mtmd_graph_whisper_enc::build() {
cur = build_mm(model.mm_fc_w, cur);
cur = ggml_add(ctx0, cur, model.mm_fc_b);
} else if (proj_type == PROJECTOR_TYPE_HIGGS_AUDIO) {
// depthwise conv (k=3, s=2, p=1) over time, then linear-relu-linear
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
cur = ggml_conv_1d_dw(ctx0, model.downsample_conv_w, cur, 2, 1, 1);
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
cur = ggml_add(ctx0, cur, model.downsample_conv_b);
cur = build_mm(model.mm_1_w, cur);
cur = ggml_add(ctx0, cur, model.mm_1_b);
cur = ggml_relu(ctx0, cur);
cur = build_mm(model.mm_2_w, cur);
cur = ggml_add(ctx0, cur, model.mm_2_b);
} else if (proj_type == PROJECTOR_TYPE_VOXTRAL) {
// projector
cur = build_ffn(cur,
+76 -2
View File
@@ -277,6 +277,7 @@ static void mtmd_audio_fill_entries(std::vector<mtmd_audio_mel> & mels, bool sha
GGML_ASSERT(mel.n_mel <= INT32_MAX && mel.n_mel >= 0);
mtmd_image_f32 entry;
entry.set_size({(int) mel.n_len, (int) mel.n_mel}, shape_only || mel.data.empty(), /* is_audio */ true);
entry.audio_n_ctx = (int32_t) mel.n_ctx;
if (!entry.is_placeholder()) {
entry.cpy_buf(mel.data);
}
@@ -688,7 +689,7 @@ bool mtmd_audio_preprocessor_qwen3a::process(const float * sampl
//
// mtmd_audio_preprocessor_dots3note
//
// Matches Dots3NoteFeatureExtractor: the waveform is split into 60s chunks and each chunk gets
// Matches Dots3NoteFeatureExtractor: the waveform is split into audio_chunk_len chunks and each chunk gets
// its own whisper-style log-mel (center=True, log10 + (max-8)/4). Only sample_length//hop frames
// per chunk are valid; the reference masks everything beyond them, so we emit exactly that many.
//
@@ -733,7 +734,7 @@ bool mtmd_audio_preprocessor_dots3note::process(const float * sa
continue; // sub-hop tail, contributes no frames
}
// reflect-pad the start; the reference zero-pads partial chunks to 60s before the STFT,
// reflect-pad the start; the reference zero-pads partial chunks to the chunk length before the STFT,
// so a partial chunk sees zeros past its end while a full chunk reflects its own tail
std::vector<float> padded(n_chunk + 2 * pad, 0.0f);
for (int i = 0; i < pad; i++) {
@@ -780,6 +781,20 @@ bool mtmd_audio_preprocessor_dots3note::process(const float * sa
return !output.empty();
}
bool mtmd_audio_preprocessor_higgs_audio::process(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output,
bool shape_only) const {
if (!mtmd_audio_preprocessor_dots3note::process(samples, n_samples, output, shape_only)) {
return false;
}
// a tail under 3 frames gives no token, the reference drops it too
if (output.size() > 1 && output.back().n_len < 3) {
output.pop_back();
}
return !output.empty() && output.back().n_len >= 3;
}
//
// mtmd_audio_preprocessor_mimo_audio
//
@@ -2227,6 +2242,65 @@ bool mtmd_audio_preprocessor_pockettts::process(const float * sa
return true;
}
//
// mtmd_audio_preprocessor_vibevoice_asr
//
// Matches VibeVoiceAcousticTokenizerFeatureExtractor: the clip is scaled to -25 dBFS RMS, then zero-padded to a whole number of frames.
// The reference encodes it with a streaming conv cache, here each chunk re-reads audio_n_ctx samples before its start instead, which gives the same output.
//
double mtmd_audio_vibevoice_gain(double sum_sq, int64_t n_samples, double peak) {
const double rms = std::max(std::sqrt(sum_sq / std::max<int64_t>(n_samples, 1)), std::pow(10.0, -50.0 / 20.0));
double gain = std::pow(10.0, -25.0 / 20.0) / (rms + 1e-6);
if (peak * gain > 1.0) {
gain /= peak * gain + 1e-6;
}
return gain;
}
bool mtmd_audio_preprocessor_vibevoice_asr::process(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output,
bool /*shape_only*/) const {
if (n_samples == 0) {
return false;
}
const int64_t hop = hparams.audio_samples_per_token();
const int64_t n_ctx = hparams.audio_n_ctx;
const int64_t n_chunk = (int64_t) hparams.audio_chunk_len * hparams.audio_sample_rate / hop * hop;
const int64_t n_padded = ((int64_t) n_samples + hop - 1) / hop * hop;
GGML_ASSERT(n_chunk > 0 && n_ctx % hop == 0);
double sum_sq = 0.0;
double peak = 0.0;
for (size_t i = 0; i < n_samples; i++) {
sum_sq += (double) samples[i] * samples[i];
peak = std::max(peak, (double) std::fabs(samples[i]));
}
const double scale = mtmd_audio_vibevoice_gain(sum_sq, (int64_t) n_samples, peak);
std::vector<float> audio((size_t) n_padded, 0.0f);
for (size_t i = 0; i < n_samples; i++) {
audio[i] = (float) (samples[i] * scale);
}
output.clear();
for (int64_t start = 0; start < n_padded; start += n_chunk) {
const int64_t beg = std::max<int64_t>(0, start - n_ctx);
const int64_t end = std::min(n_padded, start + n_chunk);
mtmd_audio_mel mel;
mel.n_len = end - beg;
mel.n_len_org = end - beg;
mel.n_mel = 1;
mel.n_ctx = start - beg;
mel.data.assign(audio.begin() + beg, audio.begin() + end);
output.push_back(std::move(mel));
}
return true;
}
// Muse Spark reads the raw waveform: pass the samples through, zero padded to a whole number of embeddings
bool mtmd_audio_preprocessor_muse_spark_audio::process(
const float * samples,
+18
View File
@@ -14,6 +14,7 @@ struct mtmd_audio_mel {
int64_t n_len;
int64_t n_len_org;
int64_t n_mel;
int64_t n_ctx = 0; // see mtmd_image_f32::audio_n_ctx
std::vector<float> data;
};
@@ -128,6 +129,17 @@ struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor {
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
// vibevoice: gain to -25 dBFS RMS, without clipping the peak
// the RMS is floored at -50 dBFS, so that silence or noise alone is not boosted to full scale
double mtmd_audio_vibevoice_gain(double sum_sq, int64_t n_samples, double peak);
// raw 24 kHz waveform, cut in chunks that carry their causal context, see mtmd_hparams::audio_n_ctx
struct mtmd_audio_preprocessor_vibevoice_asr : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_vibevoice_asr(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override {}
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_conformer(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
mtmd_audio_preprocessor_conformer(const mtmd_hparams & hparams, ggml_backend_sched_t sched) : mtmd_audio_preprocessor(hparams, sched) {}
@@ -198,6 +210,12 @@ struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor {
mtmd_audio_cache cache;
};
// same per-chunk log-mel as dots3note, with 4s chunks
struct mtmd_audio_preprocessor_higgs_audio : mtmd_audio_preprocessor_dots3note {
mtmd_audio_preprocessor_higgs_audio(const mtmd_model * model) : mtmd_audio_preprocessor_dots3note(model) {}
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_mimo_audio(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
+5
View File
@@ -870,6 +870,7 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_VOXTRAL:
case PROJECTOR_TYPE_QWEN2A:
case PROJECTOR_TYPE_HIGGS_AUDIO:
case PROJECTOR_TYPE_GLMA:
case PROJECTOR_TYPE_MERALION:
case PROJECTOR_TYPE_MUSIC_FLAMINGO:
@@ -880,6 +881,10 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
{
builder = std::make_unique<mtmd_graph_muse_spark_audio>(ctx, img);
} break;
case PROJECTOR_TYPE_VIBEVOICE_ASR:
{
builder = std::make_unique<mtmd_graph_vibevoice_asr>(ctx, img);
} break;
case PROJECTOR_TYPE_KIMIVL:
{
builder = std::make_unique<mtmd_graph_kimivl>(ctx, img);
+49
View File
@@ -18,6 +18,7 @@
#include <algorithm>
#include <cmath>
#include <cinttypes>
#include <climits>
#include <vector>
//#define MTMD_AUDIO_DEBUG
@@ -939,6 +940,54 @@ static void mtmd_helper_video_set_id(mtmd_helper_video * vctx, const std::string
vctx->id = id;
}
int32_t mtmd_helper_get_media_meta(const unsigned char * buf, size_t len, mtmd_helper_init_opt opt, mtmd_helper_media_meta * out) {
*out = {MTMD_HELPER_MEDIA_TYPE_UNKNOWN, 0, 0, 0, -1.0f, -1};
if (audio_helpers::is_audio_file((const char *) buf, len)) {
// zero channels and rate: the source format, so the length comes from the header (or an mp3 frame scan)
ma_decoder_config cfg = ma_decoder_config_init(ma_format_f32, 0, 0);
ma_decoder dec;
if (ma_decoder_init_memory(buf, len, &cfg, &dec) != MA_SUCCESS) {
return 1;
}
ma_uint64 n = 0;
if (ma_decoder_get_length_in_pcm_frames(&dec, &n) == MA_SUCCESS && dec.outputSampleRate > 0) {
out->duration = (float) ((double) n / dec.outputSampleRate);
}
out->type = MTMD_HELPER_MEDIA_TYPE_AUDIO;
out->sample_rate = (int32_t) dec.outputSampleRate;
ma_decoder_uninit(&dec);
return 0;
}
int nx, ny, nc;
if (len <= INT_MAX && stbi_info_from_memory(buf, (int) len, &nx, &ny, &nc)) {
out->type = MTMD_HELPER_MEDIA_TYPE_IMAGE;
out->width = (uint32_t) nx;
out->height = (uint32_t) ny;
return 0;
}
// webp (not read by stb_image) and video go through the video decoder
mtmd_helper_video_init_params params = opt.video_params;
const bool is_webp = is_webp_file(buf, len);
if (is_webp && video_builtin_decoder()) {
params.decoder = nullptr; // see decode_still_image()
}
mtmd_helper_video_stream_info src;
if (!video_open(nullptr, buf, len, params, src)) {
return 1;
}
out->type = is_webp ? MTMD_HELPER_MEDIA_TYPE_IMAGE : MTMD_HELPER_MEDIA_TYPE_VIDEO;
out->width = src.width;
out->height = src.height;
if (!is_webp) {
out->duration = src.duration;
out->n_frames = src.n_frames;
}
return 0;
}
mtmd_helper_video_init_params mtmd_helper_video_init_params_default() {
return {
/* fps_target */ 4.0f,
+26
View File
@@ -107,6 +107,32 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(
bool placeholder,
struct mtmd_helper_init_opt opt);
enum mtmd_helper_media_type {
MTMD_HELPER_MEDIA_TYPE_UNKNOWN,
MTMD_HELPER_MEDIA_TYPE_IMAGE,
MTMD_HELPER_MEDIA_TYPE_AUDIO,
MTMD_HELPER_MEDIA_TYPE_VIDEO,
};
struct mtmd_helper_media_meta {
enum mtmd_helper_media_type type;
uint32_t width; // image, video (frame size)
uint32_t height;
int32_t sample_rate; // audio, at the source rate
float duration; // audio, video: in seconds, -1 if unknown
int32_t n_frames; // video: -1 if unknown
};
typedef struct mtmd_helper_media_meta mtmd_helper_media_meta;
// metadata of a media file in a buffer, the same formats as mtmd_helper_bitmap_init_from_buf()
// only the headers are read (ffprobe or opt.video_params.decoder for video), nothing is decoded
// returns 0 on success, 1 if the format is not recognized
// this function is thread-safe
MTMD_API int32_t mtmd_helper_get_media_meta(
const unsigned char * buf, size_t len,
struct mtmd_helper_init_opt opt,
struct mtmd_helper_media_meta * out);
// streaming resampler for raw mono F32 PCM (e.g. realtime input), to match mtmd_get_audio_sample_rate()
// it keeps state between calls, use one per stream
struct mtmd_helper_resampler;
+17
View File
@@ -415,6 +415,17 @@
#define TN_A_CONV1D_NORM "a.conv1d.%d.norm.%s"
#define TN_A_PATCH "a.patch.%d.%s"
#define TN_A_PATCH_NORM "a.patch.%d.norm.%s"
// vibevoice-asr: causal ConvNeXt encoders, first index = encoder (0 = acoustic, 1 = semantic)
#define TN_A_CNX_CONV "a.cnx.%d.conv.%d.%s"
#define TN_A_CNX_NORM "a.cnx.%d.blk.%d.norm.%s"
#define TN_A_CNX_DW "a.cnx.%d.blk.%d.dw.%s"
#define TN_A_CNX_LS1 "a.cnx.%d.blk.%d.ls1.%s"
#define TN_A_CNX_FFN_NORM "a.cnx.%d.blk.%d.ffn_norm.%s"
#define TN_A_CNX_FFN_UP "a.cnx.%d.blk.%d.ffn_up.%s"
#define TN_A_CNX_FFN_DOWN "a.cnx.%d.blk.%d.ffn_down.%s"
#define TN_A_CNX_LS2 "a.cnx.%d.blk.%d.ls2.%s"
#define TN_MM_AUDIO_MLP_NORM "mm.a.mlp_norm.%d.%s"
#define TN_A_OUT_PROJ "a.pre_encode.out.%s"
#define TN_A_ATTN_PRE_NORM "%s.blk.%d.attn_pre_norm.%s"
#define TN_A_ATTN_POST_NORM "%s.blk.%d.attn_post_norm.%s"
@@ -522,6 +533,8 @@ enum projector_type {
PROJECTOR_TYPE_QWEN2A,
PROJECTOR_TYPE_QWEN3A,
PROJECTOR_TYPE_GLMA,
PROJECTOR_TYPE_HIGGS_AUDIO,
PROJECTOR_TYPE_VIBEVOICE_ASR,
PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on mtmd_subcontext
PROJECTOR_TYPE_VOXTRAL,
PROJECTOR_TYPE_VOXTRAL_RT,
@@ -612,6 +625,8 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_QWEN2A, "qwen2a"},
{ PROJECTOR_TYPE_QWEN3A, "qwen3a"},
{ PROJECTOR_TYPE_GLMA, "glma"},
{ PROJECTOR_TYPE_HIGGS_AUDIO, "higgs_audio"},
{ PROJECTOR_TYPE_VIBEVOICE_ASR, "vibevoice_asr"},
{ PROJECTOR_TYPE_QWEN25O, "qwen2.5o"},
{ PROJECTOR_TYPE_VOXTRAL, "voxtral"},
{ PROJECTOR_TYPE_VOXTRAL_RT, "voxtral_rt"},
@@ -815,6 +830,8 @@ struct mtmd_image_f32 {
// deepseek4v: number of leading IMAGE_PAD embeddings, aligns IMAGE_START to the LLM compressor ratio
// depends on the chunk position, set at tokenize time (see mtmd_tokenizer::add_media)
int32_t lead_pad = 0;
// audio (vibevoice): leading samples that only feed the causal convs, they produce no embedding
int32_t audio_n_ctx = 0;
// separators appended after the image tokens
mtmd_suffix_type suffix_type = MTMD_SUFFIX_NONE;
+122
View File
@@ -785,6 +785,46 @@ struct mtmd_model_loader {
}
hparams.warmup_audio_size = hparams.audio_samples_per_token();
} break;
case PROJECTOR_TYPE_VIBEVOICE_ASR:
{
hparams.audio_sample_rate = 24000; // mono waveform, no frontend
hparams.audio_chunk_len = 30; // in seconds, any length works as the convs are causal
get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size, true);
get_arr_int(KEY_A_PATCH_SIZES, hparams.audio_patch_sizes, true);
get_arr_int(KEY_A_DEPTHS, hparams.audio_depths, true);
// bounds keep the receptive field below from overflowing
const size_t n_stage = hparams.audio_patch_sizes.size();
if (n_stage == 0 || n_stage > 16 || hparams.audio_depths.size() != n_stage || hparams.n_mel_bins != 1 ||
hparams.audio_conv_kernel_size < 1 || hparams.audio_conv_kernel_size > 64) {
throw std::runtime_error(mtmd_format("%s: invalid vibevoice audio hparams\n", __func__));
}
// receptive field: stem conv (kernel), downsample convs (2 * stride), blocks (kernel), head conv (kernel)
const int64_t k = hparams.audio_conv_kernel_size;
int64_t hop = 1;
int64_t rf = 0;
int64_t n_blocks = 0;
for (size_t i = 0; i < n_stage; ++i) {
const int64_t stride = hparams.audio_patch_sizes[i];
if (stride < 1 || stride > 64 || hop * stride > (1 << 20) || hparams.audio_depths[i] < 0 || hparams.audio_depths[i] > 64) {
throw std::runtime_error(mtmd_format("%s: invalid vibevoice audio hparams\n", __func__));
}
rf += ((i == 0 ? k : 2 * stride) - 1) * hop;
hop *= stride;
rf += hparams.audio_depths[i] * (k - 1) * hop;
n_blocks += hparams.audio_depths[i];
}
rf += (k - 1) * hop;
if (rf > 60 * hparams.audio_sample_rate) {
throw std::runtime_error(mtmd_format("%s: vibevoice receptive field of %" PRId64 " samples is too long\n", __func__, rf));
}
if (n_blocks != hparams.n_layer) {
throw std::runtime_error(mtmd_format("%s: sum of audio depths (%" PRId64 ") does not match n_layer (%d)\n", __func__, n_blocks, hparams.n_layer));
}
hparams.audio_n_ctx = (int32_t) ((rf + hop - 1) / hop * hop);
hparams.warmup_audio_size = (int32_t) ((int64_t) hparams.audio_chunk_len * hparams.audio_sample_rate / hop * hop) + hparams.audio_n_ctx;
} break;
case PROJECTOR_TYPE_INKLING2V:
{
hparams.ffn_op = FFN_GELU_ERF;
@@ -892,6 +932,7 @@ struct mtmd_model_loader {
} break;
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_QWEN2A:
case PROJECTOR_TYPE_HIGGS_AUDIO:
case PROJECTOR_TYPE_QWEN3A:
case PROJECTOR_TYPE_GLMA:
case PROJECTOR_TYPE_VOXTRAL:
@@ -912,6 +953,12 @@ struct mtmd_model_loader {
hparams.audio_n_fft = 400;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
if (model.proj_type == PROJECTOR_TYPE_HIGGS_AUDIO) {
// each 4s chunk is encoded on its own
hparams.audio_chunk_len = 4;
hparams.warmup_audio_size = 400;
}
} break;
case PROJECTOR_TYPE_VOXTRAL_RT:
{
@@ -1324,6 +1371,7 @@ struct mtmd_model_loader {
// Muse Spark is encoder-free in the same sense: its conv pyramid reads the raw waveform, one channel takes the place of the mel bins
const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA &&
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC;
// Validate audio hparams loaded from GGUF metadata
@@ -1497,6 +1545,7 @@ struct mtmd_model_loader {
model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN &&
model.proj_type != PROJECTOR_TYPE_SOPRANO &&
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
model.proj_type != PROJECTOR_TYPE_CHATTERBOX &&
model.proj_type != PROJECTOR_TYPE_CHATTERBOX_SPKENC);
@@ -1928,6 +1977,52 @@ struct mtmd_model_loader {
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight"));
model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias"));
} break;
case PROJECTOR_TYPE_VIBEVOICE_ASR:
{
const int n_stage = (int) hparams.audio_patch_sizes.size();
for (int e = 0; e < 2; ++e) {
auto & enc = model.cnx_enc[e];
enc.conv_w.resize(n_stage + 1);
enc.conv_b.resize(n_stage + 1);
for (int i = 0; i <= n_stage; ++i) {
enc.conv_w[i] = get_tensor(mtmd_format(TN_A_CNX_CONV, e, i, "weight"));
enc.conv_b[i] = get_tensor(mtmd_format(TN_A_CNX_CONV, e, i, "bias"));
// the receptive field in load_hparams assumes these kernel sizes
const int64_t k_expected = (i == 0 || i == n_stage) ? hparams.audio_conv_kernel_size : 2 * hparams.audio_patch_sizes[i];
if (enc.conv_w[i]->ne[0] != k_expected) {
throw std::runtime_error(mtmd_format("%s: %s has kernel %" PRId64 ", expected %" PRId64 "\n", __func__, ggml_get_name(enc.conv_w[i]), enc.conv_w[i]->ne[0], k_expected));
}
}
enc.blocks.resize(hparams.n_layer);
for (int il = 0; il < hparams.n_layer; ++il) {
auto & layer = enc.blocks[il];
layer.ln_1_w = get_tensor(mtmd_format(TN_A_CNX_NORM, e, il, "weight"));
layer.conv_w = get_tensor(mtmd_format(TN_A_CNX_DW, e, il, "weight"));
layer.conv_b = get_tensor(mtmd_format(TN_A_CNX_DW, e, il, "bias"));
layer.ls_1_w = get_tensor(mtmd_format(TN_A_CNX_LS1, e, il, "weight"));
layer.ln_2_w = get_tensor(mtmd_format(TN_A_CNX_FFN_NORM, e, il, "weight"));
layer.ff_up_w = get_tensor(mtmd_format(TN_A_CNX_FFN_UP, e, il, "weight"));
layer.ff_up_b = get_tensor(mtmd_format(TN_A_CNX_FFN_UP, e, il, "bias"));
layer.ff_down_w = get_tensor(mtmd_format(TN_A_CNX_FFN_DOWN, e, il, "weight"));
layer.ff_down_b = get_tensor(mtmd_format(TN_A_CNX_FFN_DOWN, e, il, "bias"));
layer.ls_2_w = get_tensor(mtmd_format(TN_A_CNX_LS2, e, il, "weight"));
if (layer.conv_w->ne[0] != hparams.audio_conv_kernel_size || layer.conv_w->type != GGML_TYPE_F32) {
throw std::runtime_error(mtmd_format("%s: %s must be F32 with kernel %d\n", __func__, ggml_get_name(layer.conv_w), hparams.audio_conv_kernel_size));
}
}
}
// acoustic connector: mlp 0 -> norm -> mlp 1, semantic connector: mlp 2 -> norm -> mlp 3
model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias"));
model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP_NORM, 0, "weight"));
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias"));
model.mm_norm_mid_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP_NORM, 2, "weight"));
model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight"));
model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias"));
} break;
case PROJECTOR_TYPE_INKLING2V:
{
// these come from the generic path above, where they are optional
@@ -2315,6 +2410,19 @@ struct mtmd_model_loader {
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight"));
model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias"));
} break;
case PROJECTOR_TYPE_HIGGS_AUDIO:
{
model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight"));
model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias"));
model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight"));
model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias"));
model.downsample_conv_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight"));
model.downsample_conv_b = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias"));
} break;
case PROJECTOR_TYPE_QWEN3A:
{
model.conv2d_1_w = get_tensor(mtmd_format(TN_CONV2D, 1, "weight"));
@@ -4187,6 +4295,10 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
const int64_t fold = hparams.audio_samples_per_token();
n_patches = (int) ((img->nx() + fold - 1) / fold);
} break;
case PROJECTOR_TYPE_VIBEVOICE_ASR:
{
n_patches = (img->nx() - img->audio_n_ctx) / hparams.audio_samples_per_token();
} break;
case PROJECTOR_TYPE_VOXTRAL:
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_QWEN2A:
@@ -4210,6 +4322,13 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
n_patches /= 2;
}
} break;
case PROJECTOR_TYPE_HIGGS_AUDIO:
{
// stride-2 conv1d, avgpool(2), stride-2 depthwise conv1d
n_patches = (img->nx() + 1) / 2;
n_patches = n_patches / 2;
n_patches = (n_patches + 1) / 2;
} break;
case PROJECTOR_TYPE_QWEN3A:
{
// chunk_size=100 frames --> 3x stride-2 conv2d --> 13 tokens per chunk
@@ -4449,6 +4568,8 @@ int mtmd_model::n_mmproj_embd() const {
case PROJECTOR_TYPE_IDEFICS3:
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
return mm_fc_w->ne[1];
case PROJECTOR_TYPE_VIBEVOICE_ASR:
return mm_1_w->ne[1];
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_VOXTRAL:
case PROJECTOR_TYPE_VOXTRAL_RT:
@@ -4466,6 +4587,7 @@ int mtmd_model::n_mmproj_embd() const {
case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
return mm_fc_w->ne[1];
case PROJECTOR_TYPE_QWEN3A:
case PROJECTOR_TYPE_HIGGS_AUDIO:
return mm_2_w->ne[1];
case PROJECTOR_TYPE_GLMA:
case PROJECTOR_TYPE_LFM2:
+19 -2
View File
@@ -160,7 +160,7 @@ struct mtmd_hparams {
int32_t audio_proj_downsample_rate = 0;
int32_t audio_proj_head_count = 0;
// audio pyramid (muse-spark): widths, folded samples and blocks per stage
// audio pyramid (muse-spark, vibevoice): widths, folded samples (vibevoice: conv strides) and blocks per stage
std::vector<int32_t> audio_hidden_sizes;
std::vector<int32_t> audio_patch_sizes;
std::vector<int32_t> audio_depths;
@@ -174,6 +174,9 @@ struct mtmd_hparams {
return (int32_t) n;
}
// vibevoice: leading samples of a chunk that only feed the causal convs, at least their receptive field
int32_t audio_n_ctx = 0;
// audio-to-mel preprocessor params
int32_t audio_chunk_len = -1; // in seconds
int32_t audio_sample_rate = -1;
@@ -528,6 +531,14 @@ struct mtmd_seanet {
std::vector<stage> stages;
};
// vibevoice-asr: causal conv pyramid over the raw waveform
// stage = strided conv -> ConvNeXt blocks (RMS norm, depthwise conv, layer scale, then RMS norm, MLP, layer scale)
struct mtmd_cnx_encoder {
std::vector<ggml_tensor *> conv_w; // stem, one downsample conv per stage, head
std::vector<ggml_tensor *> conv_b;
std::vector<mtmd_layer> blocks; // ln_1, conv (depthwise), ls_1, ln_2, ffn_up, ffn_down, ls_2
};
// pocket-tts flow-matching decoder (SimpleMLPAdaLN)
struct mtmd_flow_net {
// AdaLN res block: in_ln -> modulate -> Linear -> SiLU -> Linear, gated residual
@@ -1160,7 +1171,9 @@ struct mtmd_model {
ggml_tensor * mm_norm_mid_w = nullptr;
// mimo-audio-tokenizer: post-transformer downsample + RVQ codebook
ggml_tensor * downsample_conv_w = nullptr; // no bias
// higgs-audio: depthwise downsample conv of the projector
ggml_tensor * downsample_conv_w = nullptr;
ggml_tensor * downsample_conv_b = nullptr; // higgs-audio only
ggml_tensor * downsample_norm_w = nullptr;
ggml_tensor * downsample_norm_b = nullptr;
ggml_tensor * rvq_codebook = nullptr; // merged 3D [n_q, max_bins, dim]
@@ -1208,6 +1221,9 @@ struct mtmd_model {
// pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path)
mtmd_seanet seanet;
// vibevoice-asr: acoustic and semantic encoders
mtmd_cnx_encoder cnx_enc[2];
// pocket-tts: voice latent -> backbone embd (speaker path)
ggml_tensor * spk_proj_w = nullptr;
ggml_tensor * downsample_w = nullptr;
@@ -1332,6 +1348,7 @@ struct mtmd_model {
bool audio_has_avgpool() const {
return proj_type == PROJECTOR_TYPE_QWEN2A
|| proj_type == PROJECTOR_TYPE_HIGGS_AUDIO
|| proj_type == PROJECTOR_TYPE_VOXTRAL
|| proj_type == PROJECTOR_TYPE_MUSIC_FLAMINGO;
}
+8 -2
View File
@@ -402,6 +402,7 @@ static std::vector<c2w_state_slot> list_enc_state_slots(const mtmd_hparams & hpa
switch (model.proj_type) {
case PROJECTOR_TYPE_LFM3A: return list_lfm3a_state_slots(hparams, model);
case PROJECTOR_TYPE_GEMMA4A: return list_gemma4a_state_slots(hparams, model);
case PROJECTOR_TYPE_VIBEVOICE_ASR: return list_vibevoice_asr_state_slots(hparams, model);
default: return {};
}
}
@@ -451,7 +452,7 @@ mtmd_subctx_gen_state * mtmd_subcontext::gen_state_init(uint32_t seed) {
mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) {
const auto & hparams = model.hparams;
if (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A) {
if (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A || model.proj_type == PROJECTOR_TYPE_VIBEVOICE_ASR) {
// see list_enc_state_slots()
const auto slots = list_enc_state_slots(hparams, model);
auto st = std::make_unique<mtmd_subctx_enc_state>();
@@ -1662,6 +1663,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
case PROJECTOR_TYPE_NEMOTRON_OMNI:
case PROJECTOR_TYPE_QWEN2A:
case PROJECTOR_TYPE_HIGGS_AUDIO:
case PROJECTOR_TYPE_QWEN3A:
case PROJECTOR_TYPE_GLMA:
case PROJECTOR_TYPE_ULTRAVOX:
@@ -1982,6 +1984,10 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
}
set_input_i32("positions", positions);
} break;
case PROJECTOR_TYPE_VIBEVOICE_ASR:
{
set_enc_state_in();
} break;
case PROJECTOR_TYPE_GEMMA4A:
{
GGML_ASSERT(imgs.entries.size() == 1);
@@ -2627,7 +2633,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
}
}
}
if (params->enc_state != nullptr && (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A)) {
if (params->enc_state != nullptr && (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A || model.proj_type == PROJECTOR_TYPE_VIBEVOICE_ASR)) {
const auto & slots = params->enc_state->slots;
size_t i = 0;
for (const auto & slot : list_enc_state_slots(hparams, model)) {
+274 -2
View File
@@ -48,7 +48,7 @@ std::string mtmd_format(const char * fmt, ...) {
}
// remember to bump this if the serialization format changes
#define MTMD_SERIALIZATION_VERSION 4
#define MTMD_SERIALIZATION_VERSION 5
// oldest compat version that can be loaded
#define MTMD_SERIALIZATION_VERSION_MIN 2
@@ -137,6 +137,7 @@ void mtmd_image_f32::serialize(mtmd_serialization & ser) const {
ser.write((int32_t)anyres.orig_ny);
ser.write((int32_t)nx_);
ser.write((int32_t)ny_);
ser.write(audio_n_ctx);
}
void mtmd_image_f32::deserialize(mtmd_serialization & ser) {
add_viewsep = ser.read<bool>();
@@ -155,6 +156,12 @@ void mtmd_image_f32::deserialize(mtmd_serialization & ser) {
}
nx_ = ser.read<int32_t>();
ny_ = ser.read<int32_t>();
if (ser.version >= 5) {
audio_n_ctx = ser.read<int32_t>();
if (audio_n_ctx < 0 || audio_n_ctx > nx_) {
throw std::runtime_error("invalid audio_n_ctx");
}
}
buf.clear(); // always a placeholder after loading
}
@@ -1194,6 +1201,19 @@ struct mtmd_context {
aud_end = "<|audio_eos|>";
audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(models.a.get());
} break;
case PROJECTOR_TYPE_HIGGS_AUDIO:
{
// the 4s chunks are joined in one <|audio_bos|> ... <|audio_eos|> span
aud_beg = "<|audio_bos|>";
aud_end = "<|audio_eos|>";
audio_preproc = std::make_unique<mtmd_audio_preprocessor_higgs_audio>(models.a.get());
} break;
case PROJECTOR_TYPE_VIBEVOICE_ASR:
{
aud_beg = "<|object_ref_start|>";
aud_end = "<|object_ref_end|>";
audio_preproc = std::make_unique<mtmd_audio_preprocessor_vibevoice_asr>(models.a.get());
} break;
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
{
aud_beg = "<|soft_audio_start|>";
@@ -4270,12 +4290,58 @@ struct mtmd_audio_stream_gemma4a : mtmd_audio_stream {
bool encode(int n);
};
// Higgs Audio: each 4 s chunk is encoded on its own, so a chunk goes out once it is complete
// a flush encodes the partial chunk as a chunk of its own, the reference also cuts chunks at VAD boundaries
struct mtmd_audio_stream_higgs_audio : mtmd_audio_stream {
int64_t chunk_len = 0; // samples per complete chunk
std::vector<float> beg_rows;
std::vector<float> end_rows;
bool beg_done = false;
bool end_done = false;
bool init(const mtmd_audio_stream_params & params) override;
int32_t flush(bool end) override;
int32_t step(llama_token last_token) override;
int64_t n_chunk_rows(int64_t n) const;
bool encode(int64_t n);
};
// VibeVoice-ASR: the encoder is causal, its convs keep their left context in the state
// the reference scales the whole clip to -25 dBFS, here the gain comes from the audio pushed so far (see mtmd_audio_vibevoice_gain)
struct mtmd_audio_stream_vibevoice_asr : mtmd_audio_stream {
mtmd_subctx_enc_state * enc_state = nullptr;
int64_t n_step = 0; // samples per encoder call
int64_t n_step_max = 0; // largest call, the warmup size
int64_t n_stat = 0; // pushed samples counted in sum_sq and peak
double sum_sq = 0.0;
double peak = 0.0;
std::vector<float> beg_rows;
std::vector<float> end_rows;
bool beg_done = false;
bool end_done = false;
~mtmd_audio_stream_vibevoice_asr() override {
delete enc_state;
}
bool init(const mtmd_audio_stream_params & params) override;
int32_t flush(bool end) override;
int32_t step(llama_token last_token) override;
bool encode(int64_t n);
};
bool mtmd_support_audio_stream(const mtmd_context * ctx) {
if (!ctx->ctx_a || !ctx->text_model) {
return false;
}
const projector_type proj = ctx->ctx_a->model.proj_type;
return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A || proj == PROJECTOR_TYPE_GEMMA4A;
return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A || proj == PROJECTOR_TYPE_GEMMA4A ||
proj == PROJECTOR_TYPE_HIGGS_AUDIO || proj == PROJECTOR_TYPE_VIBEVOICE_ASR;
}
mtmd_audio_stream_params mtmd_audio_stream_params_default(void) {
@@ -4298,6 +4364,8 @@ mtmd_audio_stream * mtmd_audio_stream_init(mtmd_context * ctx, mtmd_audio_stream
case PROJECTOR_TYPE_VOXTRAL_RT: s = std::make_unique<mtmd_audio_stream_voxtral_rt>(); break;
case PROJECTOR_TYPE_LFM3A: s = std::make_unique<mtmd_audio_stream_lfm3a>(); break;
case PROJECTOR_TYPE_GEMMA4A: s = std::make_unique<mtmd_audio_stream_gemma4a>(); break;
case PROJECTOR_TYPE_HIGGS_AUDIO: s = std::make_unique<mtmd_audio_stream_higgs_audio>(); break;
case PROJECTOR_TYPE_VIBEVOICE_ASR: s = std::make_unique<mtmd_audio_stream_vibevoice_asr>(); break;
default: GGML_ABORT("unsupported projector");
}
s->ctx = ctx;
@@ -5036,6 +5104,210 @@ int32_t mtmd_audio_stream_gemma4a::step(llama_token /* last_token */) {
return (int32_t) n_out;
}
// encodes one audio entry, the rows are appended to embd
static bool mtmd_audio_stream_encode(mtmd_audio_stream & s, mtmd_image_f32 && img, mtmd_subctx_enc_state * enc_state) {
const int n_out = s.sctx->model.n_output_tokens(&img);
mtmd_image_f32_batch batch;
batch.is_audio = true;
batch.entries.push_back(std::move(img));
std::vector<float> res((size_t) s.n_embd * n_out);
mtmd_subctx_encode_params params;
params.n_threads = s.ctx->n_threads;
params.imgs = &batch;
params.out_embd = &res;
params.enc_state = enc_state;
if (!s.sctx->encode(&params)) {
LOG_ERR("%s: failed to encode audio\n", __func__);
return false;
}
s.embd.insert(s.embd.end(), res.begin(), res.end());
return true;
}
// rows carry no text: all of them go out at once between the audio markers, last_token is not used
static int32_t mtmd_audio_stream_emit_rows(mtmd_audio_stream & s, const std::vector<float> & beg_rows, const std::vector<float> & end_rows,
bool & beg_done, bool & end_done) {
const int64_t n_avail = (int64_t) (s.embd.size() / s.n_embd);
s.out.clear();
if (n_avail > 0 || s.ended) {
if (!beg_done) {
beg_done = true;
s.out = beg_rows;
}
s.out.insert(s.out.end(), s.embd.begin(), s.embd.end());
s.embd.clear();
if (s.ended && !end_done) {
end_done = true;
s.out.insert(s.out.end(), end_rows.begin(), end_rows.end());
}
}
const int64_t n_out = (int64_t) (s.out.size() / s.n_embd);
if (n_out == 0) {
return s.ended ? -1 : 0;
}
s.n_rows += n_out;
s.n_rows_done += n_avail; // the markers are not audio rows
return (int32_t) n_out;
}
//
// Higgs Audio
//
bool mtmd_audio_stream_higgs_audio::init(const mtmd_audio_stream_params & params) {
const mtmd_hparams * hparams = &sctx->model.hparams;
if (params.n_prompt > 0) {
LOG_ERR("%s: Higgs Audio takes no prompt, its audio is a chat turn\n", __func__);
return false;
}
if (!ctx->audio_preproc) {
LOG_ERR("%s: the audio preprocessor is not initialized\n", __func__);
return false;
}
chunk_len = (int64_t) hparams->audio_chunk_len * hparams->audio_sample_rate;
frames_per_tok = 8; // stride-2 conv, avgpool, stride-2 projector conv
samples_per_tok = frames_per_tok * hop;
return mtmd_audio_stream_marker_rows(ctx, ctx->aud_beg, beg_rows) && mtmd_audio_stream_marker_rows(ctx, ctx->aud_end, end_rows);
}
// rows of a chunk with n samples, see mtmd_audio_preprocessor_higgs_audio
int64_t mtmd_audio_stream_higgs_audio::n_chunk_rows(int64_t n) const {
const int64_t n_frames = n / hop;
if (n_frames < 3) {
return 0;
}
return ((n_frames + 1) / 2 / 2 + 1) / 2;
}
bool mtmd_audio_stream_higgs_audio::encode(int64_t n) {
mtmd_audio_preproc_out pre = ctx->audio_preproc->preprocess(buf.data(), (size_t) n);
if (pre.entries.empty() || !ctx->audio_preproc->compute(pre)) {
LOG_ERR("%s: unable to preprocess the audio\n", __func__);
return false;
}
for (auto & entry : pre.entries) {
if (!mtmd_audio_stream_encode(*this, std::move(entry), nullptr)) {
return false;
}
}
buf.erase(buf.begin(), buf.begin() + n);
buf_pos += n;
return true;
}
int32_t mtmd_audio_stream_higgs_audio::flush(bool end) {
const int64_t n_pending = n_samples - buf_pos;
n_rows_flushed = n_rows_done + (int64_t) (embd.size() / n_embd) + (n_pending / chunk_len) * n_chunk_rows(chunk_len) + n_chunk_rows(n_pending % chunk_len);
pending_flush = true;
ended = end;
return 0;
}
int32_t mtmd_audio_stream_higgs_audio::step(llama_token /* last_token */) {
const bool flush = pending_flush;
pending_flush = false;
while ((int64_t) buf.size() >= chunk_len) {
if (!encode(chunk_len)) {
return -2;
}
}
// a tail too short for a row is kept until the next flush, or dropped at the end
if ((flush || ended) && !buf.empty()) {
if (n_chunk_rows((int64_t) buf.size()) > 0) {
if (!encode((int64_t) buf.size())) {
return -2;
}
} else if (ended) {
buf_pos += (int64_t) buf.size();
buf.clear();
}
}
return mtmd_audio_stream_emit_rows(*this, beg_rows, end_rows, beg_done, end_done);
}
//
// VibeVoice-ASR
//
bool mtmd_audio_stream_vibevoice_asr::init(const mtmd_audio_stream_params & params) {
const mtmd_hparams * hparams = &sctx->model.hparams;
if (params.n_prompt > 0) {
LOG_ERR("%s: VibeVoice-ASR takes no prompt, its audio is a chat turn\n", __func__);
return false;
}
hop = hparams->audio_samples_per_token();
frames_per_tok = 1;
samples_per_tok = hop;
n_step = std::max<int64_t>(1, ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 + hop - 1) / hop) * hop;
n_step_max = std::max(n_step, (int64_t) hparams->audio_chunk_len * hparams->audio_sample_rate / hop * hop);
enc_state = sctx->enc_state_init(0);
if (!enc_state) {
return false;
}
return mtmd_audio_stream_marker_rows(ctx, ctx->aud_beg, beg_rows) && mtmd_audio_stream_marker_rows(ctx, ctx->aud_end, end_rows);
}
// n is a multiple of hop, see mtmd_audio_preprocessor_vibevoice_asr for the gain
bool mtmd_audio_stream_vibevoice_asr::encode(int64_t n) {
const double gain = mtmd_audio_vibevoice_gain(sum_sq, n_stat, peak);
std::vector<float> data((size_t) n);
for (int64_t i = 0; i < n; i++) {
data[i] = (float) (buf[i] * gain);
}
mtmd_image_f32 img;
img.set_size({(int) n, 1}, false, /* is_audio */ true);
img.cpy_buf(data);
if (!mtmd_audio_stream_encode(*this, std::move(img), enc_state)) {
return false;
}
buf.erase(buf.begin(), buf.begin() + n);
buf_pos += n;
return true;
}
int32_t mtmd_audio_stream_vibevoice_asr::flush(bool end) {
n_rows_flushed = end ? (n_samples + hop - 1) / hop : n_samples / hop;
pending_flush = true;
ended = end;
return 0;
}
int32_t mtmd_audio_stream_vibevoice_asr::step(llama_token /* last_token */) {
const bool flush = pending_flush;
pending_flush = false;
// the padding added at the end does not count in the gain
for (int64_t i = n_stat - buf_pos; i < (int64_t) buf.size(); i++) {
sum_sq += (double) buf[i] * buf[i];
peak = std::max(peak, (double) std::fabs(buf[i]));
}
n_stat = n_samples;
if (ended && n_samples % hop != 0) {
add_silence(hop - n_samples % hop);
n_stat = n_samples;
}
// whole rows only: a partial row would put its zero padding into the state
int64_t n = (int64_t) buf.size();
n -= n % hop;
if (!flush && !ended) {
n -= n % n_step;
}
while (n > 0) {
const int64_t m = std::min(n, n_step_max);
if (!encode(m)) {
return -2;
}
n -= m;
}
return mtmd_audio_stream_emit_rows(*this, beg_rows, end_rows, beg_done, end_done);
}
int64_t mtmd_audio_stream_get_n_rows(const mtmd_audio_stream * s) {
return s->n_rows_done;
}
+11 -3
View File
@@ -739,14 +739,22 @@ json convert_transcriptions_to_chatcmpl(
if (response_format != "json") {
throw std::invalid_argument("Only 'json' response_format is supported for transcription");
}
const common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language);
if (prompt.empty()) {
common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language, 0.0, prompt);
if (preset.needs_audio_length) {
mtmd_helper_media_meta meta;
if (mtmd_helper_get_media_meta(out_files[0].data(), out_files[0].size(), mtmd_helper_init_opt_default(), &meta) != 0 ||
meta.type != MTMD_HELPER_MEDIA_TYPE_AUDIO || meta.duration < 0) {
throw std::invalid_argument("the input file is not a supported audio file");
}
preset = common_chat_get_asr_prompt(tmpls, language, meta.duration, prompt);
}
if (prompt.empty() || preset.has_context) {
prompt = preset.user;
}
if (!language.empty() && !preset.has_language) {
prompt += string_format(" (language: %s)", language.c_str());
}
prompt += get_media_marker();
prompt = preset.media_first ? get_media_marker() + prompt : prompt + get_media_marker();
json messages = json::array();
if (!preset.system.empty()) {
+15 -2
View File
@@ -155,6 +155,7 @@ struct realtime_conversation {
bool started = false; // something went into the slot, the session config is fixed
bool turn_open = false; // a user turn is being filled
bool has_audio = false; // the open turn has audio not committed yet
int64_t n_turn_samples = 0; // audio of the open turn, at the model rate
bool tool_results = false; // tool results went in after the last response, the next one answers them
int n_turns = 0;
std::string next_header; // text that goes in before the next user turn
@@ -376,8 +377,10 @@ struct realtime_conversation {
if (!resample(samples) || samples.empty() || !ensure_turn()) {
return;
}
const int64_t n = (int64_t) samples.size();
if (control(SERVER_CONTROL_ACTION_AUDIO_APPEND, {}, std::move(samples))) {
has_audio = true;
n_turn_samples += n;
}
}
@@ -410,7 +413,8 @@ struct realtime_conversation {
if (!control(SERVER_CONTROL_ACTION_AUDIO_END)) {
return false;
}
has_audio = false;
has_audio = false;
n_turn_samples = 0;
std::lock_guard<std::mutex> lock(mutex);
const json prev = last_item_id.empty() ? json(nullptr) : json(last_item_id);
@@ -424,6 +428,7 @@ struct realtime_conversation {
}
void on_commit() {
const int64_t n_audio = n_turn_samples;
if (!commit() || type != "transcription") {
return;
}
@@ -433,11 +438,19 @@ struct realtime_conversation {
transcribe_item_id = last_item_id;
transcript.clear();
}
// some models (VibeVoice-ASR) take an instruction after the audio, with its length
const auto * tmpls = deps.meta.chat_params.tmpls.get();
common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language);
if (preset.needs_audio_length) {
preset = common_chat_get_asr_prompt(tmpls, language, (double) n_audio / mtmd_get_audio_sample_rate(deps.mctx));
}
const std::string after_audio = preset.media_first ? preset.user : "";
server_control_response response;
response.n_predict = max_output_tokens;
response.out_mode = MTMD_HELPER_GEN_AUDIO_OUT_TEXT;
response.sampling = response_sampling(nullptr);
if (!control(SERVER_CONTROL_ACTION_RESPONSE_CREATE, user_footer + assistant_header, {}, response)) {
if (!control(SERVER_CONTROL_ACTION_RESPONSE_CREATE, after_audio + user_footer + assistant_header, {}, response)) {
return;
}
control(SERVER_CONTROL_ACTION_CLEAR);