mtmd: support higgs v3 STT and vibevoice-ASR (#32329)
* mtmd: support higgs v3 and vibevoice-ASR * use proper mapping * address review comment * nits * rm hacky get_audio_length, add mtmd_helper_get_media_meta
This commit is contained in:
1 parent
18b74fa41d
commit
fc504f91d8
24 files changed
+1101
-18
No files matched your search
+18
-1
@@ -827,6 +827,11 @@ static std::string string_lower(std::string s) {
|
||||
return s;
|
||||
}
|
||||
|
||||
// VibeVoice-ASR: default system prompt of the template written by the converter
|
||||
static bool is_vibevoice_asr_template(const std::string & src) {
|
||||
return src.find("transcribes audio input into text output in JSON format") != std::string::npos;
|
||||
}
|
||||
|
||||
// LFM3-Audio: language code -> name
|
||||
static std::string lfm3_language_name(const std::string & language) {
|
||||
static const std::map<std::string, std::string> names = {
|
||||
@@ -847,7 +852,8 @@ static std::string lfm3_asr_system_prompt(const std::string & language) {
|
||||
return "Perform ASR in " + lfm3_language_name(language) + ".";
|
||||
}
|
||||
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) {
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language,
|
||||
double audio_length, const std::string & context) {
|
||||
common_chat_prompt_preset asr_preset;
|
||||
asr_preset.system = "";
|
||||
asr_preset.user = "Transcribe audio to text";
|
||||
@@ -861,6 +867,17 @@ common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates
|
||||
asr_preset.system = lfm3_asr_system_prompt(language);
|
||||
asr_preset.user = "";
|
||||
asr_preset.has_language = true;
|
||||
} else if (is_vibevoice_asr_template(src)) {
|
||||
// without the audio length the model does not know where the audio ends and loops
|
||||
const char * keys = "transcribe it with these keys: Start time, End time, Speaker ID, Content";
|
||||
if (context.empty()) {
|
||||
asr_preset.user = string_format("\nThis is a %.2f seconds audio, please %s", audio_length, keys);
|
||||
} else {
|
||||
asr_preset.user = string_format("\nThis is a %.2f seconds audio, with extra info: %s\n\nPlease %s", audio_length, context.c_str(), keys);
|
||||
asr_preset.has_context = true;
|
||||
}
|
||||
asr_preset.media_first = true;
|
||||
asr_preset.needs_audio_length = true;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+8
-2
@@ -491,10 +491,16 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
|
||||
struct common_chat_prompt_preset {
|
||||
std::string system;
|
||||
std::string user;
|
||||
bool has_language = false; // the preset already names the requested language
|
||||
bool has_language = false; // the preset already names the requested language
|
||||
bool has_context = false; // the user text already holds the context
|
||||
bool media_first = false; // the media goes before the user text
|
||||
bool needs_audio_length = false; // the user text depends on the audio length, call again with it
|
||||
};
|
||||
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "");
|
||||
// audio_length: in seconds, 0 if unknown
|
||||
// context: extra info from the user (e.g. names, hotwords), only used by some presets, see has_context
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "",
|
||||
double audio_length = 0.0, const std::string & context = "");
|
||||
|
||||
// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
|
||||
// language and voice are optional, the template defines their defaults
|
||||
|
||||
@@ -138,6 +138,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"Grok1ForCausalLM": "grok",
|
||||
"GrokForCausalLM": "grok",
|
||||
"GroveMoeForCausalLM": "grovemoe",
|
||||
"HiggsAudio3Model": "higgs_audio",
|
||||
"HunYuanDenseV1ForCausalLM": "hunyuan",
|
||||
"HunYuanMoEV1ForCausalLM": "hunyuan",
|
||||
"HunYuanVLForConditionalGeneration": "hunyuan",
|
||||
@@ -325,6 +326,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"UMT5Model": "t5",
|
||||
"UltravoxModel": "ultravox",
|
||||
"UnlimitedOCRForCausalLM": "deepseek",
|
||||
"VibeVoiceAsrForConditionalGeneration": "vibevoice_asr",
|
||||
"VLlama3ForCausalLM": "llama",
|
||||
"VllmDecision2Model": "vllm_decision2",
|
||||
"VoxtralForConditionalGeneration": "llama",
|
||||
@@ -372,6 +374,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"Granite4VisionForConditionalGeneration": "granite",
|
||||
"GraniteSpeechForConditionalGeneration": "granite",
|
||||
"GraniteSpeechPlusForConditionalGeneration": "granite",
|
||||
"HiggsAudio3Model": "higgs_audio",
|
||||
"HunYuanVLForConditionalGeneration": "hunyuan",
|
||||
"Idefics3ForConditionalGeneration": "smolvlm",
|
||||
"Inkling2ForConditionalGeneration": "inkling2",
|
||||
@@ -431,6 +434,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"Step3p7ForConditionalGeneration": "step3",
|
||||
"UltravoxModel": "ultravox",
|
||||
"UnlimitedOCRForCausalLM": "deepseek",
|
||||
"VibeVoiceAsrForConditionalGeneration": "vibevoice_asr",
|
||||
"VoxtralForConditionalGeneration": "ultravox",
|
||||
"VoxtralRealtimeForConditionalGeneration": "ultravox",
|
||||
"YoutuVLForConditionalGeneration": "youtuvl",
|
||||
|
||||
+1
-1
@@ -3475,7 +3475,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
|
||||
# Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older
|
||||
# Kimi-Linear-48B architecture and cannot load K3 (no attention residuals,
|
||||
# latent MoE, situ, ...). Route on the top-level architecture instead.
|
||||
if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK31ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration"):
|
||||
if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK31ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration", "HiggsAudio3Model"):
|
||||
return arch
|
||||
|
||||
# if "architectures" is found in the sub-config, use that instead
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, gguf
|
||||
from .qwen import Qwen3Model
|
||||
from .ultravox import WhisperEncoderModel
|
||||
|
||||
|
||||
# plain ChatML as built by transcribe.py upstream, the Qwen3 template renders past assistant turns differently from the last one
|
||||
HIGGS_AUDIO_CHAT_TEMPLATE = (
|
||||
"{%- for message in messages -%}"
|
||||
"{{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' -}}"
|
||||
"{%- endfor -%}"
|
||||
"{%- if add_generation_prompt -%}"
|
||||
"{{- '<|im_start|>assistant\\n' -}}"
|
||||
"{%- if enable_thinking is defined and enable_thinking is false -%}"
|
||||
"{{- '<think>\\n\\n</think>\\n\\n' -}}"
|
||||
"{%- endif -%}"
|
||||
"{%- endif -%}"
|
||||
)
|
||||
|
||||
|
||||
@ModelBase.register("HiggsAudio3Model")
|
||||
@ModelBase.example("bosonai/higgs-audio-v3-stt")
|
||||
class HiggsAudioModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
def set_vocab(self):
|
||||
super().set_vocab()
|
||||
self.gguf_writer.add_chat_template(HIGGS_AUDIO_CHAT_TEMPLATE)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
|
||||
# the audio_* tensors are dropped by TextModel, rename the text head before that
|
||||
if name == "audio_decoder_proj.text_lm_head.weight":
|
||||
name = "lm_head.weight"
|
||||
else:
|
||||
name = "model." + name
|
||||
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
|
||||
@ModelBase.register("HiggsAudio3Model")
|
||||
class HiggsAudioWhisperEncoderModel(WhisperEncoderModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
return self.global_config["audio_encoder_config"]
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# depthwise conv kernel, same as the whisper convs
|
||||
if name == "audio_encoder_proj.temporal.weight":
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
assert self.global_config["projector_type"] == "mlp" and self.global_config["projector_temporal_downsample"] == 2
|
||||
assert self.global_config["audio_adapter_type"] == "stack" and not self.global_config.get("duplicate_audio_triplet", False)
|
||||
super().set_gguf_parameters()
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.HIGGS_AUDIO)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if not name.startswith(("audio_tower.", "audio_encoder_proj.")):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
@@ -0,0 +1,157 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import MmprojModel, ModelBase, gguf
|
||||
from .qwen import Qwen2Model
|
||||
|
||||
|
||||
# the upstream chat_template.jinja is a processor template (list content, audio parts), use plain ChatML with its default system prompt
|
||||
VIBEVOICE_ASR_CHAT_TEMPLATE = (
|
||||
"{%- if messages[0]['role'] == 'system' -%}"
|
||||
"{{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' -}}"
|
||||
"{%- set loop_messages = messages[1:] -%}"
|
||||
"{%- else -%}"
|
||||
"{{- '<|im_start|>system\\nYou are a helpful assistant that transcribes audio input into text output in JSON format.<|im_end|>\\n' -}}"
|
||||
"{%- set loop_messages = messages -%}"
|
||||
"{%- endif -%}"
|
||||
"{%- for message in loop_messages -%}"
|
||||
"{{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' -}}"
|
||||
"{%- endfor -%}"
|
||||
"{%- if add_generation_prompt -%}"
|
||||
"{{- '<|im_start|>assistant\\n' -}}"
|
||||
"{%- endif -%}"
|
||||
)
|
||||
|
||||
|
||||
@ModelBase.register("VibeVoiceAsrForConditionalGeneration")
|
||||
@ModelBase.example("microsoft/VibeVoice-ASR-HF")
|
||||
class VibeVoiceAsrModel(Qwen2Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN2
|
||||
|
||||
def set_vocab(self):
|
||||
super().set_vocab()
|
||||
self.gguf_writer.add_chat_template(VIBEVOICE_ASR_CHAT_TEMPLATE)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if not name.startswith("language_model."):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
|
||||
@ModelBase.register("VibeVoiceAsrForConditionalGeneration")
|
||||
class VibeVoiceAsrAudioModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
|
||||
# values hard-coded in tools/mtmd/models/vibevoice-asr.cpp
|
||||
_encoder_consts = {"channels": 1, "kernel_size": 7, "num_filters": 32, "ffn_expansion": 4, "hidden_act": "gelu", "rms_norm_eps": 1e-5}
|
||||
_encoders = ("acoustic_tokenizer_encoder", "semantic_tokenizer_encoder")
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
c = self.global_config["acoustic_tokenizer_encoder_config"]
|
||||
return {
|
||||
**c,
|
||||
"intermediate_size": c["ffn_expansion"] * c["hidden_size"],
|
||||
"num_attention_heads": 1,
|
||||
"num_hidden_layers": sum(c["depths"]),
|
||||
}
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
for enc in self._encoders:
|
||||
c = self.global_config[f"{enc}_config"]
|
||||
for k, v in self._encoder_consts.items():
|
||||
assert c[k] == v, f"{enc}.{k} = {c[k]}, expected {v}"
|
||||
assert c["depths"] == self.hparams["depths"] and c["downsampling_ratios"] == self.hparams["downsampling_ratios"]
|
||||
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.VIBEVOICE_ASR)
|
||||
# the input is the raw waveform, one channel takes the place of the mel bins
|
||||
self.gguf_writer.add_audio_num_mel_bins(1)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams["rms_norm_eps"])
|
||||
self.gguf_writer.add_audio_conv_kernel_size(self.hparams["kernel_size"])
|
||||
# stride of each stage, the stem has none
|
||||
self.gguf_writer.add_audio_patch_sizes([1] + [int(r) for r in self.hparams["downsampling_ratios"]])
|
||||
self.gguf_writer.add_audio_depths([int(d) for d in self.hparams["depths"]])
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
if ".dw." in new_name:
|
||||
return gguf.GGMLQuantizationType.F32 # ggml_ssm_conv
|
||||
if ".conv." in new_name and new_name.endswith(".weight"):
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if not name.startswith(cls._encoders + ("multi_modal_projector.",)):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
_block_tensors = {
|
||||
"norm.weight": gguf.MODEL_TENSOR.A_ENC_CNX_NORM,
|
||||
"mixer.conv.weight": gguf.MODEL_TENSOR.A_ENC_CNX_DW,
|
||||
"mixer.conv.bias": gguf.MODEL_TENSOR.A_ENC_CNX_DW,
|
||||
"gamma": gguf.MODEL_TENSOR.A_ENC_CNX_LS1,
|
||||
"ffn_norm.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_NORM,
|
||||
"ffn.linear1.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_UP,
|
||||
"ffn.linear1.bias": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_UP,
|
||||
"ffn.linear2.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
|
||||
"ffn.linear2.bias": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
|
||||
"ffn_gamma": gguf.MODEL_TENSOR.A_ENC_CNX_LS2,
|
||||
}
|
||||
|
||||
# mm.a.mlp.{0,1} + mm.a.mlp_norm.0 for the acoustic connector, {2,3} + 2 for the semantic one
|
||||
_proj_tensors = {
|
||||
"acoustic_linear_1": (gguf.MODEL_TENSOR.A_MMPROJ, 0),
|
||||
"acoustic_norm": (gguf.MODEL_TENSOR.A_MMPROJ_NORM, 0),
|
||||
"acoustic_linear_2": (gguf.MODEL_TENSOR.A_MMPROJ, 1),
|
||||
"semantic_linear_1": (gguf.MODEL_TENSOR.A_MMPROJ, 2),
|
||||
"semantic_norm": (gguf.MODEL_TENSOR.A_MMPROJ_NORM, 2),
|
||||
"semantic_linear_2": (gguf.MODEL_TENSOR.A_MMPROJ, 3),
|
||||
}
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# the layer scales (gamma) have no suffix upstream
|
||||
suffix = "." + name.rsplit(".", 1)[-1] if name.endswith((".weight", ".bias")) else ".weight"
|
||||
|
||||
if name.startswith("multi_modal_projector."):
|
||||
key, idx = self._proj_tensors[name.split(".")[1]]
|
||||
yield (self.format_tensor_name(key, idx, suffix=suffix), data_torch)
|
||||
return
|
||||
|
||||
enc, _, rest = name.partition(".")
|
||||
enc_id = self._encoders.index(enc)
|
||||
parts = rest.split(".")
|
||||
depths = [int(d) for d in self.hparams["depths"]]
|
||||
|
||||
# causal convs of the stages: 0 = stem, 1..6 = downsample, 7 = head
|
||||
if rest.startswith("stem.conv."):
|
||||
conv_id = 0
|
||||
elif parts[0] == "conv_layers" and parts[2] == "conv":
|
||||
conv_id = int(parts[1]) + 1
|
||||
elif parts[0] == "head":
|
||||
conv_id = len(depths)
|
||||
else:
|
||||
conv_id = None
|
||||
if conv_id is not None:
|
||||
yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_ENC_CNX_CONV].format(bid=enc_id, xid=conv_id) + suffix, data_torch)
|
||||
return
|
||||
|
||||
# ConvNeXt blocks are numbered per stage upstream and flat here
|
||||
if parts[0] == "stem":
|
||||
stage, blk, tail = 0, int(parts[2]), ".".join(parts[3:])
|
||||
else:
|
||||
assert parts[0] == "conv_layers" and parts[2] == "stage", name
|
||||
stage, blk, tail = int(parts[1]) + 1, int(parts[3]), ".".join(parts[4:])
|
||||
xid = sum(depths[:stage]) + blk
|
||||
|
||||
key = self._block_tensors[tail]
|
||||
if key == gguf.MODEL_TENSOR.A_ENC_CNX_DW and tail.endswith(".weight"):
|
||||
data_torch = data_torch.squeeze(1) # [C, 1, K] -> [C, K] for ggml_ssm_conv
|
||||
yield (gguf.TENSOR_NAMES[key].format(bid=enc_id, xid=xid) + suffix, data_torch)
|
||||
@@ -1386,6 +1386,14 @@ class MODEL_TENSOR(IntEnum):
|
||||
A_ENC_SEANET_RES_CONV1 = auto() # residual unit, dilated conv
|
||||
A_ENC_SEANET_RES_CONV2 = auto() # residual unit, pointwise conv
|
||||
A_ENC_SEANET_SCALE_CONV = auto() # strided downsample conv
|
||||
A_ENC_CNX_CONV = auto() # vibevoice: causal conv of a stage (stem, downsample, head), bid = encoder
|
||||
A_ENC_CNX_NORM = auto() # vibevoice: ConvNeXt block, bid = encoder, xid = block
|
||||
A_ENC_CNX_DW = auto()
|
||||
A_ENC_CNX_LS1 = auto()
|
||||
A_ENC_CNX_FFN_NORM = auto()
|
||||
A_ENC_CNX_FFN_UP = auto()
|
||||
A_ENC_CNX_FFN_DOWN = auto()
|
||||
A_ENC_CNX_LS2 = auto()
|
||||
A_ENC_ATTN_SCALE = auto() # layer scale (gamma) on the attn output
|
||||
A_ENC_FFN_SCALE_LS = auto() # layer scale (gamma) on the FFN output
|
||||
A_ENC_SPEAKER_PROJ = auto() # voice latent -> backbone embd
|
||||
@@ -1419,6 +1427,7 @@ class MODEL_TENSOR(IntEnum):
|
||||
A_GEN_WAV_OUTPUT = auto()
|
||||
A_MMPROJ = auto()
|
||||
A_MMPROJ_FC = auto()
|
||||
A_MMPROJ_NORM = auto() # vibevoice: norm after mm.a.mlp.{bid}
|
||||
A_MM_NORM_PRE = auto()
|
||||
A_MM_NORM_MID = auto()
|
||||
A_MM_EMBEDDING = auto() # gemma3n
|
||||
@@ -2317,6 +2326,14 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
|
||||
MODEL_TENSOR.A_ENC_SEANET_RES_CONV1: "a.seanet.blk.{bid}.res_conv1",
|
||||
MODEL_TENSOR.A_ENC_SEANET_RES_CONV2: "a.seanet.blk.{bid}.res_conv2",
|
||||
MODEL_TENSOR.A_ENC_SEANET_SCALE_CONV: "a.seanet.blk.{bid}.scale_conv",
|
||||
MODEL_TENSOR.A_ENC_CNX_CONV: "a.cnx.{bid}.conv.{xid}",
|
||||
MODEL_TENSOR.A_ENC_CNX_NORM: "a.cnx.{bid}.blk.{xid}.norm",
|
||||
MODEL_TENSOR.A_ENC_CNX_DW: "a.cnx.{bid}.blk.{xid}.dw",
|
||||
MODEL_TENSOR.A_ENC_CNX_LS1: "a.cnx.{bid}.blk.{xid}.ls1",
|
||||
MODEL_TENSOR.A_ENC_CNX_FFN_NORM: "a.cnx.{bid}.blk.{xid}.ffn_norm",
|
||||
MODEL_TENSOR.A_ENC_CNX_FFN_UP: "a.cnx.{bid}.blk.{xid}.ffn_up",
|
||||
MODEL_TENSOR.A_ENC_CNX_FFN_DOWN: "a.cnx.{bid}.blk.{xid}.ffn_down",
|
||||
MODEL_TENSOR.A_ENC_CNX_LS2: "a.cnx.{bid}.blk.{xid}.ls2",
|
||||
MODEL_TENSOR.A_ENC_ATTN_SCALE: "a.blk.{bid}.ls1",
|
||||
MODEL_TENSOR.A_ENC_FFN_SCALE_LS: "a.blk.{bid}.ls2",
|
||||
MODEL_TENSOR.A_ENC_SPEAKER_PROJ: "a.speaker_proj",
|
||||
@@ -2349,6 +2366,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
|
||||
MODEL_TENSOR.A_GEN_WAV_OUTPUT: "a.gen.wav.output",
|
||||
MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}",
|
||||
MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc",
|
||||
MODEL_TENSOR.A_MMPROJ_NORM: "mm.a.mlp_norm.{bid}",
|
||||
MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre",
|
||||
MODEL_TENSOR.A_MM_NORM_MID: "mm.a.norm_mid",
|
||||
MODEL_TENSOR.A_MM_INP_PROJ: "mm.a.input_projection", # gemma3n
|
||||
@@ -2641,6 +2659,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.A_ENC_RVQ_CODEBOOK,
|
||||
MODEL_TENSOR.A_MMPROJ,
|
||||
MODEL_TENSOR.A_MMPROJ_FC,
|
||||
MODEL_TENSOR.A_MMPROJ_NORM,
|
||||
MODEL_TENSOR.A_MM_NORM_PRE,
|
||||
MODEL_TENSOR.A_MM_NORM_MID,
|
||||
MODEL_TENSOR.A_MM_CODE_EMBD,
|
||||
@@ -2741,6 +2760,14 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.A_ENC_SEANET_RES_CONV1,
|
||||
MODEL_TENSOR.A_ENC_SEANET_RES_CONV2,
|
||||
MODEL_TENSOR.A_ENC_SEANET_SCALE_CONV,
|
||||
MODEL_TENSOR.A_ENC_CNX_CONV,
|
||||
MODEL_TENSOR.A_ENC_CNX_NORM,
|
||||
MODEL_TENSOR.A_ENC_CNX_DW,
|
||||
MODEL_TENSOR.A_ENC_CNX_LS1,
|
||||
MODEL_TENSOR.A_ENC_CNX_FFN_NORM,
|
||||
MODEL_TENSOR.A_ENC_CNX_FFN_UP,
|
||||
MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
|
||||
MODEL_TENSOR.A_ENC_CNX_LS2,
|
||||
MODEL_TENSOR.A_ENC_ATTN_SCALE,
|
||||
MODEL_TENSOR.A_ENC_FFN_SCALE_LS,
|
||||
MODEL_TENSOR.A_ENC_SPEAKER_PROJ,
|
||||
@@ -7155,6 +7182,8 @@ class VisionProjectorType:
|
||||
QWEN2A = "qwen2a" # audio
|
||||
QWEN3A = "qwen3a" # audio
|
||||
GLMA = "glma" # audio
|
||||
HIGGS_AUDIO = "higgs_audio" # audio
|
||||
VIBEVOICE_ASR = "vibevoice_asr" # audio
|
||||
QWEN25O = "qwen2.5o" # omni
|
||||
VOXTRAL = "voxtral"
|
||||
VOXTRAL_RT = "voxtral_rt"
|
||||
|
||||
@@ -2973,6 +2973,7 @@ class TensorNameMap:
|
||||
|
||||
MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV: (
|
||||
"encoder.down_sample_layer.0", # mimo-audio-tokenizer
|
||||
"audio_encoder_proj.temporal", # higgs-audio
|
||||
),
|
||||
|
||||
MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM: (
|
||||
@@ -3026,6 +3027,7 @@ class TensorNameMap:
|
||||
"mlp.model.{bid}", # lfm3a
|
||||
"audio_tower.proj{bid}", # qwen3omni
|
||||
"sound_projection.linear{bid}", # parakeet (linear1, linear2)
|
||||
"audio_encoder_proj.linear{bid}", # higgs-audio (linear1, linear2)
|
||||
),
|
||||
|
||||
MODEL_TENSOR.A_MMPROJ_FC: (
|
||||
|
||||
@@ -63,6 +63,7 @@ add_library(mtmd
|
||||
models/nemotron-transcribe.cpp
|
||||
models/muse-glimmer.cpp
|
||||
models/muse-spark-audio.cpp
|
||||
models/vibevoice-asr.cpp
|
||||
models/llama4.cpp
|
||||
models/llava.cpp
|
||||
models/minicpmv.cpp
|
||||
|
||||
@@ -268,6 +268,18 @@ struct mtmd_graph_granite_speech : mtmd_graph {
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct mtmd_graph_vibevoice_asr : mtmd_graph {
|
||||
mtmd_graph_vibevoice_asr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
std::map<std::string, ggml_tensor *> state_in;
|
||||
mutable std::vector<std::pair<std::string, ggml_tensor *>> state_out;
|
||||
|
||||
ggml_tensor * with_left_ctx(ggml_tensor * x, const std::string & name) const;
|
||||
ggml_tensor * causal_conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & name) const;
|
||||
ggml_tensor * encode(int e, ggml_tensor * x) const;
|
||||
};
|
||||
|
||||
struct mtmd_graph_muse_spark_audio : mtmd_graph {
|
||||
mtmd_graph_muse_spark_audio(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
@@ -643,6 +655,9 @@ std::vector<c2w_state_slot> list_lfm3a_state_slots(const mtmd_hparams & hparams,
|
||||
// state slots of the streaming Gemma 4 audio encoder (gemma4a)
|
||||
std::vector<c2w_state_slot> list_gemma4a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
|
||||
|
||||
// vibevoice-asr: the left context of every causal conv, a zero state is the causal zero padding
|
||||
std::vector<c2w_state_slot> list_vibevoice_asr_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
|
||||
|
||||
struct mtmd_graph_kimik25 : mtmd_graph {
|
||||
mtmd_graph_kimik25(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
#include "models.h"
|
||||
|
||||
// VibeVoice-ASR audio tower: two causal ConvNeXt pyramids (acoustic and semantic) over the raw 24 kHz waveform
|
||||
// each one is a VAE tokenizer encoder, its latents go through their own connector and the two are summed
|
||||
// see modeling_vibevoice_acoustic_tokenizer.py and modeling_vibevoice_asr.py in transformers
|
||||
//
|
||||
// tensors are T-first [T, C] around the convs and C-first [C, T] in the blocks
|
||||
|
||||
static std::string cnx_slot(int e, const char * kind, int i) {
|
||||
return "e" + std::to_string(e) + "_" + kind + "_" + std::to_string(i);
|
||||
}
|
||||
|
||||
std::vector<c2w_state_slot> list_vibevoice_asr_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
|
||||
std::vector<c2w_state_slot> slots;
|
||||
slots.push_back({"pos", 1, 1}); // unused, set_enc_state_in() expects it
|
||||
const int n_stage = (int) hparams.audio_patch_sizes.size();
|
||||
for (int e = 0; e < 2; ++e) {
|
||||
const auto & enc = model.cnx_enc[e];
|
||||
for (int i = 0; i <= n_stage; ++i) {
|
||||
const int64_t stride = i < n_stage ? hparams.audio_patch_sizes[i] : 1;
|
||||
slots.push_back({cnx_slot(e, "conv", i), enc.conv_w[i]->ne[0] - stride, enc.conv_w[i]->ne[1]});
|
||||
}
|
||||
for (int il = 0; il < (int) enc.blocks.size(); ++il) {
|
||||
const auto * w = enc.blocks[il].conv_w;
|
||||
slots.push_back({cnx_slot(e, "dw", il), w->ne[0] - 1, w->ne[1]});
|
||||
}
|
||||
}
|
||||
return slots;
|
||||
}
|
||||
|
||||
// x: [T, C] -> [left + T, C], the left context comes from the state, its new value is the tail
|
||||
ggml_tensor * mtmd_graph_vibevoice_asr::with_left_ctx(ggml_tensor * x, const std::string & name) const {
|
||||
ggml_tensor * left = state_in.at(name);
|
||||
x = ggml_concat(ctx0, left, x, 0);
|
||||
state_out.push_back({name, ggml_cont(ctx0, ggml_view_2d(ctx0, x, left->ne[0], x->ne[1], x->nb[1], (x->ne[0] - left->ne[0]) * x->nb[0]))});
|
||||
return x;
|
||||
}
|
||||
|
||||
// x: [T, IC], w: [K, IC, OC] -> [T / stride, OC]
|
||||
ggml_tensor * mtmd_graph_vibevoice_asr::causal_conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & name) const {
|
||||
x = with_left_ctx(x, name);
|
||||
ggml_tensor * y = ggml_conv_1d(ctx0, w, x, stride, 0, 1);
|
||||
y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[1]);
|
||||
return ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
|
||||
}
|
||||
|
||||
// x: [T, 1] -> [C_latent, T / hop]
|
||||
ggml_tensor * mtmd_graph_vibevoice_asr::encode(int e, ggml_tensor * x) const {
|
||||
const auto & enc = model.cnx_enc[e];
|
||||
const int n_stage = (int) hparams.audio_patch_sizes.size();
|
||||
|
||||
ggml_tensor * cur = x;
|
||||
int bid = 0;
|
||||
for (int stage = 0; stage < n_stage; ++stage) {
|
||||
cur = causal_conv(cur, enc.conv_w[stage], enc.conv_b[stage], hparams.audio_patch_sizes[stage], cnx_slot(e, "conv", stage));
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cb(cur, "stage_in", stage);
|
||||
|
||||
const int64_t n_pos = cur->ne[1];
|
||||
for (int i = 0; i < hparams.audio_depths[stage]; ++i, ++bid) {
|
||||
const auto & layer = enc.blocks[bid];
|
||||
|
||||
// causal depthwise conv mixer
|
||||
ggml_tensor * h = build_norm(cur, layer.ln_1_w, nullptr, NORM_TYPE_RMS, hparams.eps, bid);
|
||||
h = ggml_cont(ctx0, ggml_transpose(ctx0, h));
|
||||
h = with_left_ctx(h, cnx_slot(e, "dw", bid));
|
||||
h = ggml_reshape_3d(ctx0, h, h->ne[0], h->ne[1], 1);
|
||||
h = ggml_ssm_conv(ctx0, h, layer.conv_w);
|
||||
h = ggml_reshape_2d(ctx0, h, h->ne[0], n_pos);
|
||||
h = ggml_add(ctx0, h, layer.conv_b);
|
||||
cur = ggml_add(ctx0, cur, ggml_mul(ctx0, h, layer.ls_1_w));
|
||||
|
||||
h = build_norm(cur, layer.ln_2_w, nullptr, NORM_TYPE_RMS, hparams.eps, bid);
|
||||
h = build_ffn(h,
|
||||
layer.ff_up_w, layer.ff_up_b,
|
||||
nullptr, nullptr,
|
||||
layer.ff_down_w, layer.ff_down_b,
|
||||
FFN_GELU_ERF, bid);
|
||||
cur = ggml_add(ctx0, cur, ggml_mul(ctx0, h, layer.ls_2_w));
|
||||
cb(cur, "block_out", bid);
|
||||
}
|
||||
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
}
|
||||
|
||||
cur = causal_conv(cur, enc.conv_w[n_stage], enc.conv_b[n_stage], 1, cnx_slot(e, "conv", n_stage));
|
||||
return ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
}
|
||||
|
||||
ggml_cgraph * mtmd_graph_vibevoice_asr::build() {
|
||||
const int64_t hop = hparams.audio_samples_per_token();
|
||||
const int64_t n_frames = img.nx() / hop;
|
||||
const int64_t n_ctx = img.audio_n_ctx / hop;
|
||||
GGML_ASSERT(img.nx() % hop == 0 && img.audio_n_ctx % hop == 0 && n_ctx < n_frames);
|
||||
|
||||
for (const auto & slot : list_vibevoice_asr_state_slots(hparams, model)) {
|
||||
ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
|
||||
ggml_set_name(t, ("state_in_" + slot.name).c_str());
|
||||
ggml_set_input(t);
|
||||
state_in[slot.name] = t;
|
||||
}
|
||||
state_out.push_back({"pos", ggml_scale_bias(ctx0, state_in.at("pos"), 1.0f, (float) n_frames)});
|
||||
|
||||
ggml_tensor * inp = build_inp_raw(1);
|
||||
inp = ggml_reshape_2d(ctx0, inp, img.nx(), 1);
|
||||
|
||||
// the leading context frames only warm up the causal convs
|
||||
auto drop_ctx = [&](ggml_tensor * t) {
|
||||
return ggml_view_2d(ctx0, t, t->ne[0], n_frames - n_ctx, t->nb[1], n_ctx * t->nb[1]);
|
||||
};
|
||||
|
||||
// the reference adds VAE noise to the acoustic latents, skipped here to keep the output deterministic
|
||||
ggml_tensor * acoustic = drop_ctx(encode(0, inp));
|
||||
ggml_tensor * semantic = drop_ctx(encode(1, inp));
|
||||
cb(acoustic, "acoustic_latents", -1);
|
||||
cb(semantic, "semantic_latents", -1);
|
||||
|
||||
acoustic = ggml_add(ctx0, build_mm(model.mm_0_w, acoustic), model.mm_0_b);
|
||||
acoustic = build_norm(acoustic, model.mm_norm_pre_w, nullptr, NORM_TYPE_RMS, 1e-6f, -1);
|
||||
acoustic = ggml_add(ctx0, build_mm(model.mm_1_w, acoustic), model.mm_1_b);
|
||||
|
||||
semantic = ggml_add(ctx0, build_mm(model.mm_2_w, semantic), model.mm_2_b);
|
||||
semantic = build_norm(semantic, model.mm_norm_mid_w, nullptr, NORM_TYPE_RMS, 1e-6f, -1);
|
||||
semantic = ggml_add(ctx0, build_mm(model.mm_3_w, semantic), model.mm_3_b);
|
||||
|
||||
ggml_tensor * cur = ggml_add(ctx0, acoustic, semantic);
|
||||
cb(cur, "projected", -1);
|
||||
|
||||
// the embeddings must stay the last node
|
||||
for (const auto & s : state_out) {
|
||||
ggml_set_name(s.second, ("state_out_" + s.first).c_str());
|
||||
ggml_set_output(s.second);
|
||||
ggml_build_forward_expand(gf, s.second);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
ggml_cgraph * mtmd_graph_whisper_enc::build() {
|
||||
const int n_frames = img.nx();
|
||||
const int n_pos = n_frames / 2;
|
||||
const int n_pos = (n_frames + 1) / 2; // stride-2 conv1d with padding 1
|
||||
GGML_ASSERT(model.position_embeddings->ne[1] >= n_pos);
|
||||
|
||||
ggml_tensor * inp = build_inp_raw(1);
|
||||
@@ -77,6 +77,19 @@ ggml_cgraph * mtmd_graph_whisper_enc::build() {
|
||||
cur = build_mm(model.mm_fc_w, cur);
|
||||
cur = ggml_add(ctx0, cur, model.mm_fc_b);
|
||||
|
||||
} else if (proj_type == PROJECTOR_TYPE_HIGGS_AUDIO) {
|
||||
// depthwise conv (k=3, s=2, p=1) over time, then linear-relu-linear
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cur = ggml_conv_1d_dw(ctx0, model.downsample_conv_w, cur, 2, 1, 1);
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cur = ggml_add(ctx0, cur, model.downsample_conv_b);
|
||||
|
||||
cur = build_mm(model.mm_1_w, cur);
|
||||
cur = ggml_add(ctx0, cur, model.mm_1_b);
|
||||
cur = ggml_relu(ctx0, cur);
|
||||
cur = build_mm(model.mm_2_w, cur);
|
||||
cur = ggml_add(ctx0, cur, model.mm_2_b);
|
||||
|
||||
} else if (proj_type == PROJECTOR_TYPE_VOXTRAL) {
|
||||
// projector
|
||||
cur = build_ffn(cur,
|
||||
|
||||
@@ -277,6 +277,7 @@ static void mtmd_audio_fill_entries(std::vector<mtmd_audio_mel> & mels, bool sha
|
||||
GGML_ASSERT(mel.n_mel <= INT32_MAX && mel.n_mel >= 0);
|
||||
mtmd_image_f32 entry;
|
||||
entry.set_size({(int) mel.n_len, (int) mel.n_mel}, shape_only || mel.data.empty(), /* is_audio */ true);
|
||||
entry.audio_n_ctx = (int32_t) mel.n_ctx;
|
||||
if (!entry.is_placeholder()) {
|
||||
entry.cpy_buf(mel.data);
|
||||
}
|
||||
@@ -688,7 +689,7 @@ bool mtmd_audio_preprocessor_qwen3a::process(const float * sampl
|
||||
//
|
||||
// mtmd_audio_preprocessor_dots3note
|
||||
//
|
||||
// Matches Dots3NoteFeatureExtractor: the waveform is split into 60s chunks and each chunk gets
|
||||
// Matches Dots3NoteFeatureExtractor: the waveform is split into audio_chunk_len chunks and each chunk gets
|
||||
// its own whisper-style log-mel (center=True, log10 + (max-8)/4). Only sample_length//hop frames
|
||||
// per chunk are valid; the reference masks everything beyond them, so we emit exactly that many.
|
||||
//
|
||||
@@ -733,7 +734,7 @@ bool mtmd_audio_preprocessor_dots3note::process(const float * sa
|
||||
continue; // sub-hop tail, contributes no frames
|
||||
}
|
||||
|
||||
// reflect-pad the start; the reference zero-pads partial chunks to 60s before the STFT,
|
||||
// reflect-pad the start; the reference zero-pads partial chunks to the chunk length before the STFT,
|
||||
// so a partial chunk sees zeros past its end while a full chunk reflects its own tail
|
||||
std::vector<float> padded(n_chunk + 2 * pad, 0.0f);
|
||||
for (int i = 0; i < pad; i++) {
|
||||
@@ -780,6 +781,20 @@ bool mtmd_audio_preprocessor_dots3note::process(const float * sa
|
||||
return !output.empty();
|
||||
}
|
||||
|
||||
bool mtmd_audio_preprocessor_higgs_audio::process(const float * samples,
|
||||
size_t n_samples,
|
||||
std::vector<mtmd_audio_mel> & output,
|
||||
bool shape_only) const {
|
||||
if (!mtmd_audio_preprocessor_dots3note::process(samples, n_samples, output, shape_only)) {
|
||||
return false;
|
||||
}
|
||||
// a tail under 3 frames gives no token, the reference drops it too
|
||||
if (output.size() > 1 && output.back().n_len < 3) {
|
||||
output.pop_back();
|
||||
}
|
||||
return !output.empty() && output.back().n_len >= 3;
|
||||
}
|
||||
|
||||
//
|
||||
// mtmd_audio_preprocessor_mimo_audio
|
||||
//
|
||||
@@ -2227,6 +2242,65 @@ bool mtmd_audio_preprocessor_pockettts::process(const float * sa
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// mtmd_audio_preprocessor_vibevoice_asr
|
||||
//
|
||||
// Matches VibeVoiceAcousticTokenizerFeatureExtractor: the clip is scaled to -25 dBFS RMS, then zero-padded to a whole number of frames.
|
||||
// The reference encodes it with a streaming conv cache, here each chunk re-reads audio_n_ctx samples before its start instead, which gives the same output.
|
||||
//
|
||||
|
||||
double mtmd_audio_vibevoice_gain(double sum_sq, int64_t n_samples, double peak) {
|
||||
const double rms = std::max(std::sqrt(sum_sq / std::max<int64_t>(n_samples, 1)), std::pow(10.0, -50.0 / 20.0));
|
||||
double gain = std::pow(10.0, -25.0 / 20.0) / (rms + 1e-6);
|
||||
if (peak * gain > 1.0) {
|
||||
gain /= peak * gain + 1e-6;
|
||||
}
|
||||
return gain;
|
||||
}
|
||||
|
||||
bool mtmd_audio_preprocessor_vibevoice_asr::process(const float * samples,
|
||||
size_t n_samples,
|
||||
std::vector<mtmd_audio_mel> & output,
|
||||
bool /*shape_only*/) const {
|
||||
if (n_samples == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const int64_t hop = hparams.audio_samples_per_token();
|
||||
const int64_t n_ctx = hparams.audio_n_ctx;
|
||||
const int64_t n_chunk = (int64_t) hparams.audio_chunk_len * hparams.audio_sample_rate / hop * hop;
|
||||
const int64_t n_padded = ((int64_t) n_samples + hop - 1) / hop * hop;
|
||||
GGML_ASSERT(n_chunk > 0 && n_ctx % hop == 0);
|
||||
|
||||
double sum_sq = 0.0;
|
||||
double peak = 0.0;
|
||||
for (size_t i = 0; i < n_samples; i++) {
|
||||
sum_sq += (double) samples[i] * samples[i];
|
||||
peak = std::max(peak, (double) std::fabs(samples[i]));
|
||||
}
|
||||
const double scale = mtmd_audio_vibevoice_gain(sum_sq, (int64_t) n_samples, peak);
|
||||
|
||||
std::vector<float> audio((size_t) n_padded, 0.0f);
|
||||
for (size_t i = 0; i < n_samples; i++) {
|
||||
audio[i] = (float) (samples[i] * scale);
|
||||
}
|
||||
|
||||
output.clear();
|
||||
for (int64_t start = 0; start < n_padded; start += n_chunk) {
|
||||
const int64_t beg = std::max<int64_t>(0, start - n_ctx);
|
||||
const int64_t end = std::min(n_padded, start + n_chunk);
|
||||
|
||||
mtmd_audio_mel mel;
|
||||
mel.n_len = end - beg;
|
||||
mel.n_len_org = end - beg;
|
||||
mel.n_mel = 1;
|
||||
mel.n_ctx = start - beg;
|
||||
mel.data.assign(audio.begin() + beg, audio.begin() + end);
|
||||
output.push_back(std::move(mel));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Muse Spark reads the raw waveform: pass the samples through, zero padded to a whole number of embeddings
|
||||
bool mtmd_audio_preprocessor_muse_spark_audio::process(
|
||||
const float * samples,
|
||||
|
||||
@@ -14,6 +14,7 @@ struct mtmd_audio_mel {
|
||||
int64_t n_len;
|
||||
int64_t n_len_org;
|
||||
int64_t n_mel;
|
||||
int64_t n_ctx = 0; // see mtmd_image_f32::audio_n_ctx
|
||||
|
||||
std::vector<float> data;
|
||||
};
|
||||
@@ -128,6 +129,17 @@ struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor {
|
||||
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
|
||||
};
|
||||
|
||||
// vibevoice: gain to -25 dBFS RMS, without clipping the peak
|
||||
// the RMS is floored at -50 dBFS, so that silence or noise alone is not boosted to full scale
|
||||
double mtmd_audio_vibevoice_gain(double sum_sq, int64_t n_samples, double peak);
|
||||
|
||||
// raw 24 kHz waveform, cut in chunks that carry their causal context, see mtmd_hparams::audio_n_ctx
|
||||
struct mtmd_audio_preprocessor_vibevoice_asr : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_vibevoice_asr(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
|
||||
void initialize() override {}
|
||||
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_conformer(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
|
||||
mtmd_audio_preprocessor_conformer(const mtmd_hparams & hparams, ggml_backend_sched_t sched) : mtmd_audio_preprocessor(hparams, sched) {}
|
||||
@@ -198,6 +210,12 @@ struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor {
|
||||
mtmd_audio_cache cache;
|
||||
};
|
||||
|
||||
// same per-chunk log-mel as dots3note, with 4s chunks
|
||||
struct mtmd_audio_preprocessor_higgs_audio : mtmd_audio_preprocessor_dots3note {
|
||||
mtmd_audio_preprocessor_higgs_audio(const mtmd_model * model) : mtmd_audio_preprocessor_dots3note(model) {}
|
||||
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_mimo_audio(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
|
||||
void initialize() override;
|
||||
|
||||
@@ -870,6 +870,7 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
|
||||
case PROJECTOR_TYPE_ULTRAVOX:
|
||||
case PROJECTOR_TYPE_VOXTRAL:
|
||||
case PROJECTOR_TYPE_QWEN2A:
|
||||
case PROJECTOR_TYPE_HIGGS_AUDIO:
|
||||
case PROJECTOR_TYPE_GLMA:
|
||||
case PROJECTOR_TYPE_MERALION:
|
||||
case PROJECTOR_TYPE_MUSIC_FLAMINGO:
|
||||
@@ -880,6 +881,10 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
|
||||
{
|
||||
builder = std::make_unique<mtmd_graph_muse_spark_audio>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VIBEVOICE_ASR:
|
||||
{
|
||||
builder = std::make_unique<mtmd_graph_vibevoice_asr>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_KIMIVL:
|
||||
{
|
||||
builder = std::make_unique<mtmd_graph_kimivl>(ctx, img);
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cinttypes>
|
||||
#include <climits>
|
||||
#include <vector>
|
||||
|
||||
//#define MTMD_AUDIO_DEBUG
|
||||
@@ -939,6 +940,54 @@ static void mtmd_helper_video_set_id(mtmd_helper_video * vctx, const std::string
|
||||
vctx->id = id;
|
||||
}
|
||||
|
||||
int32_t mtmd_helper_get_media_meta(const unsigned char * buf, size_t len, mtmd_helper_init_opt opt, mtmd_helper_media_meta * out) {
|
||||
*out = {MTMD_HELPER_MEDIA_TYPE_UNKNOWN, 0, 0, 0, -1.0f, -1};
|
||||
|
||||
if (audio_helpers::is_audio_file((const char *) buf, len)) {
|
||||
// zero channels and rate: the source format, so the length comes from the header (or an mp3 frame scan)
|
||||
ma_decoder_config cfg = ma_decoder_config_init(ma_format_f32, 0, 0);
|
||||
ma_decoder dec;
|
||||
if (ma_decoder_init_memory(buf, len, &cfg, &dec) != MA_SUCCESS) {
|
||||
return 1;
|
||||
}
|
||||
ma_uint64 n = 0;
|
||||
if (ma_decoder_get_length_in_pcm_frames(&dec, &n) == MA_SUCCESS && dec.outputSampleRate > 0) {
|
||||
out->duration = (float) ((double) n / dec.outputSampleRate);
|
||||
}
|
||||
out->type = MTMD_HELPER_MEDIA_TYPE_AUDIO;
|
||||
out->sample_rate = (int32_t) dec.outputSampleRate;
|
||||
ma_decoder_uninit(&dec);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int nx, ny, nc;
|
||||
if (len <= INT_MAX && stbi_info_from_memory(buf, (int) len, &nx, &ny, &nc)) {
|
||||
out->type = MTMD_HELPER_MEDIA_TYPE_IMAGE;
|
||||
out->width = (uint32_t) nx;
|
||||
out->height = (uint32_t) ny;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// webp (not read by stb_image) and video go through the video decoder
|
||||
mtmd_helper_video_init_params params = opt.video_params;
|
||||
const bool is_webp = is_webp_file(buf, len);
|
||||
if (is_webp && video_builtin_decoder()) {
|
||||
params.decoder = nullptr; // see decode_still_image()
|
||||
}
|
||||
mtmd_helper_video_stream_info src;
|
||||
if (!video_open(nullptr, buf, len, params, src)) {
|
||||
return 1;
|
||||
}
|
||||
out->type = is_webp ? MTMD_HELPER_MEDIA_TYPE_IMAGE : MTMD_HELPER_MEDIA_TYPE_VIDEO;
|
||||
out->width = src.width;
|
||||
out->height = src.height;
|
||||
if (!is_webp) {
|
||||
out->duration = src.duration;
|
||||
out->n_frames = src.n_frames;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
mtmd_helper_video_init_params mtmd_helper_video_init_params_default() {
|
||||
return {
|
||||
/* fps_target */ 4.0f,
|
||||
|
||||
@@ -107,6 +107,32 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(
|
||||
bool placeholder,
|
||||
struct mtmd_helper_init_opt opt);
|
||||
|
||||
enum mtmd_helper_media_type {
|
||||
MTMD_HELPER_MEDIA_TYPE_UNKNOWN,
|
||||
MTMD_HELPER_MEDIA_TYPE_IMAGE,
|
||||
MTMD_HELPER_MEDIA_TYPE_AUDIO,
|
||||
MTMD_HELPER_MEDIA_TYPE_VIDEO,
|
||||
};
|
||||
|
||||
struct mtmd_helper_media_meta {
|
||||
enum mtmd_helper_media_type type;
|
||||
uint32_t width; // image, video (frame size)
|
||||
uint32_t height;
|
||||
int32_t sample_rate; // audio, at the source rate
|
||||
float duration; // audio, video: in seconds, -1 if unknown
|
||||
int32_t n_frames; // video: -1 if unknown
|
||||
};
|
||||
typedef struct mtmd_helper_media_meta mtmd_helper_media_meta;
|
||||
|
||||
// metadata of a media file in a buffer, the same formats as mtmd_helper_bitmap_init_from_buf()
|
||||
// only the headers are read (ffprobe or opt.video_params.decoder for video), nothing is decoded
|
||||
// returns 0 on success, 1 if the format is not recognized
|
||||
// this function is thread-safe
|
||||
MTMD_API int32_t mtmd_helper_get_media_meta(
|
||||
const unsigned char * buf, size_t len,
|
||||
struct mtmd_helper_init_opt opt,
|
||||
struct mtmd_helper_media_meta * out);
|
||||
|
||||
// streaming resampler for raw mono F32 PCM (e.g. realtime input), to match mtmd_get_audio_sample_rate()
|
||||
// it keeps state between calls, use one per stream
|
||||
struct mtmd_helper_resampler;
|
||||
|
||||
@@ -415,6 +415,17 @@
|
||||
#define TN_A_CONV1D_NORM "a.conv1d.%d.norm.%s"
|
||||
#define TN_A_PATCH "a.patch.%d.%s"
|
||||
#define TN_A_PATCH_NORM "a.patch.%d.norm.%s"
|
||||
|
||||
// vibevoice-asr: causal ConvNeXt encoders, first index = encoder (0 = acoustic, 1 = semantic)
|
||||
#define TN_A_CNX_CONV "a.cnx.%d.conv.%d.%s"
|
||||
#define TN_A_CNX_NORM "a.cnx.%d.blk.%d.norm.%s"
|
||||
#define TN_A_CNX_DW "a.cnx.%d.blk.%d.dw.%s"
|
||||
#define TN_A_CNX_LS1 "a.cnx.%d.blk.%d.ls1.%s"
|
||||
#define TN_A_CNX_FFN_NORM "a.cnx.%d.blk.%d.ffn_norm.%s"
|
||||
#define TN_A_CNX_FFN_UP "a.cnx.%d.blk.%d.ffn_up.%s"
|
||||
#define TN_A_CNX_FFN_DOWN "a.cnx.%d.blk.%d.ffn_down.%s"
|
||||
#define TN_A_CNX_LS2 "a.cnx.%d.blk.%d.ls2.%s"
|
||||
#define TN_MM_AUDIO_MLP_NORM "mm.a.mlp_norm.%d.%s"
|
||||
#define TN_A_OUT_PROJ "a.pre_encode.out.%s"
|
||||
#define TN_A_ATTN_PRE_NORM "%s.blk.%d.attn_pre_norm.%s"
|
||||
#define TN_A_ATTN_POST_NORM "%s.blk.%d.attn_post_norm.%s"
|
||||
@@ -522,6 +533,8 @@ enum projector_type {
|
||||
PROJECTOR_TYPE_QWEN2A,
|
||||
PROJECTOR_TYPE_QWEN3A,
|
||||
PROJECTOR_TYPE_GLMA,
|
||||
PROJECTOR_TYPE_HIGGS_AUDIO,
|
||||
PROJECTOR_TYPE_VIBEVOICE_ASR,
|
||||
PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on mtmd_subcontext
|
||||
PROJECTOR_TYPE_VOXTRAL,
|
||||
PROJECTOR_TYPE_VOXTRAL_RT,
|
||||
@@ -612,6 +625,8 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
|
||||
{ PROJECTOR_TYPE_QWEN2A, "qwen2a"},
|
||||
{ PROJECTOR_TYPE_QWEN3A, "qwen3a"},
|
||||
{ PROJECTOR_TYPE_GLMA, "glma"},
|
||||
{ PROJECTOR_TYPE_HIGGS_AUDIO, "higgs_audio"},
|
||||
{ PROJECTOR_TYPE_VIBEVOICE_ASR, "vibevoice_asr"},
|
||||
{ PROJECTOR_TYPE_QWEN25O, "qwen2.5o"},
|
||||
{ PROJECTOR_TYPE_VOXTRAL, "voxtral"},
|
||||
{ PROJECTOR_TYPE_VOXTRAL_RT, "voxtral_rt"},
|
||||
@@ -815,6 +830,8 @@ struct mtmd_image_f32 {
|
||||
// deepseek4v: number of leading IMAGE_PAD embeddings, aligns IMAGE_START to the LLM compressor ratio
|
||||
// depends on the chunk position, set at tokenize time (see mtmd_tokenizer::add_media)
|
||||
int32_t lead_pad = 0;
|
||||
// audio (vibevoice): leading samples that only feed the causal convs, they produce no embedding
|
||||
int32_t audio_n_ctx = 0;
|
||||
// separators appended after the image tokens
|
||||
mtmd_suffix_type suffix_type = MTMD_SUFFIX_NONE;
|
||||
|
||||
|
||||
@@ -785,6 +785,46 @@ struct mtmd_model_loader {
|
||||
}
|
||||
hparams.warmup_audio_size = hparams.audio_samples_per_token();
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VIBEVOICE_ASR:
|
||||
{
|
||||
hparams.audio_sample_rate = 24000; // mono waveform, no frontend
|
||||
hparams.audio_chunk_len = 30; // in seconds, any length works as the convs are causal
|
||||
get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size, true);
|
||||
get_arr_int(KEY_A_PATCH_SIZES, hparams.audio_patch_sizes, true);
|
||||
get_arr_int(KEY_A_DEPTHS, hparams.audio_depths, true);
|
||||
|
||||
// bounds keep the receptive field below from overflowing
|
||||
const size_t n_stage = hparams.audio_patch_sizes.size();
|
||||
if (n_stage == 0 || n_stage > 16 || hparams.audio_depths.size() != n_stage || hparams.n_mel_bins != 1 ||
|
||||
hparams.audio_conv_kernel_size < 1 || hparams.audio_conv_kernel_size > 64) {
|
||||
throw std::runtime_error(mtmd_format("%s: invalid vibevoice audio hparams\n", __func__));
|
||||
}
|
||||
|
||||
// receptive field: stem conv (kernel), downsample convs (2 * stride), blocks (kernel), head conv (kernel)
|
||||
const int64_t k = hparams.audio_conv_kernel_size;
|
||||
int64_t hop = 1;
|
||||
int64_t rf = 0;
|
||||
int64_t n_blocks = 0;
|
||||
for (size_t i = 0; i < n_stage; ++i) {
|
||||
const int64_t stride = hparams.audio_patch_sizes[i];
|
||||
if (stride < 1 || stride > 64 || hop * stride > (1 << 20) || hparams.audio_depths[i] < 0 || hparams.audio_depths[i] > 64) {
|
||||
throw std::runtime_error(mtmd_format("%s: invalid vibevoice audio hparams\n", __func__));
|
||||
}
|
||||
rf += ((i == 0 ? k : 2 * stride) - 1) * hop;
|
||||
hop *= stride;
|
||||
rf += hparams.audio_depths[i] * (k - 1) * hop;
|
||||
n_blocks += hparams.audio_depths[i];
|
||||
}
|
||||
rf += (k - 1) * hop;
|
||||
if (rf > 60 * hparams.audio_sample_rate) {
|
||||
throw std::runtime_error(mtmd_format("%s: vibevoice receptive field of %" PRId64 " samples is too long\n", __func__, rf));
|
||||
}
|
||||
if (n_blocks != hparams.n_layer) {
|
||||
throw std::runtime_error(mtmd_format("%s: sum of audio depths (%" PRId64 ") does not match n_layer (%d)\n", __func__, n_blocks, hparams.n_layer));
|
||||
}
|
||||
hparams.audio_n_ctx = (int32_t) ((rf + hop - 1) / hop * hop);
|
||||
hparams.warmup_audio_size = (int32_t) ((int64_t) hparams.audio_chunk_len * hparams.audio_sample_rate / hop * hop) + hparams.audio_n_ctx;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_INKLING2V:
|
||||
{
|
||||
hparams.ffn_op = FFN_GELU_ERF;
|
||||
@@ -892,6 +932,7 @@ struct mtmd_model_loader {
|
||||
} break;
|
||||
case PROJECTOR_TYPE_ULTRAVOX:
|
||||
case PROJECTOR_TYPE_QWEN2A:
|
||||
case PROJECTOR_TYPE_HIGGS_AUDIO:
|
||||
case PROJECTOR_TYPE_QWEN3A:
|
||||
case PROJECTOR_TYPE_GLMA:
|
||||
case PROJECTOR_TYPE_VOXTRAL:
|
||||
@@ -912,6 +953,12 @@ struct mtmd_model_loader {
|
||||
hparams.audio_n_fft = 400;
|
||||
hparams.audio_window_len = 400;
|
||||
hparams.audio_hop_len = 160;
|
||||
|
||||
if (model.proj_type == PROJECTOR_TYPE_HIGGS_AUDIO) {
|
||||
// each 4s chunk is encoded on its own
|
||||
hparams.audio_chunk_len = 4;
|
||||
hparams.warmup_audio_size = 400;
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VOXTRAL_RT:
|
||||
{
|
||||
@@ -1324,6 +1371,7 @@ struct mtmd_model_loader {
|
||||
// Muse Spark is encoder-free in the same sense: its conv pyramid reads the raw waveform, one channel takes the place of the mel bins
|
||||
const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA &&
|
||||
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
|
||||
model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
|
||||
model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC;
|
||||
|
||||
// Validate audio hparams loaded from GGUF metadata
|
||||
@@ -1497,6 +1545,7 @@ struct mtmd_model_loader {
|
||||
model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN &&
|
||||
model.proj_type != PROJECTOR_TYPE_SOPRANO &&
|
||||
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
|
||||
model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
|
||||
model.proj_type != PROJECTOR_TYPE_CHATTERBOX &&
|
||||
model.proj_type != PROJECTOR_TYPE_CHATTERBOX_SPKENC);
|
||||
|
||||
@@ -1928,6 +1977,52 @@ struct mtmd_model_loader {
|
||||
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight"));
|
||||
model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias"));
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VIBEVOICE_ASR:
|
||||
{
|
||||
const int n_stage = (int) hparams.audio_patch_sizes.size();
|
||||
for (int e = 0; e < 2; ++e) {
|
||||
auto & enc = model.cnx_enc[e];
|
||||
enc.conv_w.resize(n_stage + 1);
|
||||
enc.conv_b.resize(n_stage + 1);
|
||||
for (int i = 0; i <= n_stage; ++i) {
|
||||
enc.conv_w[i] = get_tensor(mtmd_format(TN_A_CNX_CONV, e, i, "weight"));
|
||||
enc.conv_b[i] = get_tensor(mtmd_format(TN_A_CNX_CONV, e, i, "bias"));
|
||||
// the receptive field in load_hparams assumes these kernel sizes
|
||||
const int64_t k_expected = (i == 0 || i == n_stage) ? hparams.audio_conv_kernel_size : 2 * hparams.audio_patch_sizes[i];
|
||||
if (enc.conv_w[i]->ne[0] != k_expected) {
|
||||
throw std::runtime_error(mtmd_format("%s: %s has kernel %" PRId64 ", expected %" PRId64 "\n", __func__, ggml_get_name(enc.conv_w[i]), enc.conv_w[i]->ne[0], k_expected));
|
||||
}
|
||||
}
|
||||
enc.blocks.resize(hparams.n_layer);
|
||||
for (int il = 0; il < hparams.n_layer; ++il) {
|
||||
auto & layer = enc.blocks[il];
|
||||
layer.ln_1_w = get_tensor(mtmd_format(TN_A_CNX_NORM, e, il, "weight"));
|
||||
layer.conv_w = get_tensor(mtmd_format(TN_A_CNX_DW, e, il, "weight"));
|
||||
layer.conv_b = get_tensor(mtmd_format(TN_A_CNX_DW, e, il, "bias"));
|
||||
layer.ls_1_w = get_tensor(mtmd_format(TN_A_CNX_LS1, e, il, "weight"));
|
||||
layer.ln_2_w = get_tensor(mtmd_format(TN_A_CNX_FFN_NORM, e, il, "weight"));
|
||||
layer.ff_up_w = get_tensor(mtmd_format(TN_A_CNX_FFN_UP, e, il, "weight"));
|
||||
layer.ff_up_b = get_tensor(mtmd_format(TN_A_CNX_FFN_UP, e, il, "bias"));
|
||||
layer.ff_down_w = get_tensor(mtmd_format(TN_A_CNX_FFN_DOWN, e, il, "weight"));
|
||||
layer.ff_down_b = get_tensor(mtmd_format(TN_A_CNX_FFN_DOWN, e, il, "bias"));
|
||||
layer.ls_2_w = get_tensor(mtmd_format(TN_A_CNX_LS2, e, il, "weight"));
|
||||
if (layer.conv_w->ne[0] != hparams.audio_conv_kernel_size || layer.conv_w->type != GGML_TYPE_F32) {
|
||||
throw std::runtime_error(mtmd_format("%s: %s must be F32 with kernel %d\n", __func__, ggml_get_name(layer.conv_w), hparams.audio_conv_kernel_size));
|
||||
}
|
||||
}
|
||||
}
|
||||
// acoustic connector: mlp 0 -> norm -> mlp 1, semantic connector: mlp 2 -> norm -> mlp 3
|
||||
model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight"));
|
||||
model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias"));
|
||||
model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP_NORM, 0, "weight"));
|
||||
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
|
||||
model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
|
||||
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
|
||||
model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias"));
|
||||
model.mm_norm_mid_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP_NORM, 2, "weight"));
|
||||
model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight"));
|
||||
model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias"));
|
||||
} break;
|
||||
case PROJECTOR_TYPE_INKLING2V:
|
||||
{
|
||||
// these come from the generic path above, where they are optional
|
||||
@@ -2315,6 +2410,19 @@ struct mtmd_model_loader {
|
||||
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight"));
|
||||
model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias"));
|
||||
} break;
|
||||
case PROJECTOR_TYPE_HIGGS_AUDIO:
|
||||
{
|
||||
model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight"));
|
||||
model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias"));
|
||||
model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight"));
|
||||
model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias"));
|
||||
model.downsample_conv_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight"));
|
||||
model.downsample_conv_b = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "bias"));
|
||||
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
|
||||
model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
|
||||
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
|
||||
model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias"));
|
||||
} break;
|
||||
case PROJECTOR_TYPE_QWEN3A:
|
||||
{
|
||||
model.conv2d_1_w = get_tensor(mtmd_format(TN_CONV2D, 1, "weight"));
|
||||
@@ -4187,6 +4295,10 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
|
||||
const int64_t fold = hparams.audio_samples_per_token();
|
||||
n_patches = (int) ((img->nx() + fold - 1) / fold);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VIBEVOICE_ASR:
|
||||
{
|
||||
n_patches = (img->nx() - img->audio_n_ctx) / hparams.audio_samples_per_token();
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VOXTRAL:
|
||||
case PROJECTOR_TYPE_ULTRAVOX:
|
||||
case PROJECTOR_TYPE_QWEN2A:
|
||||
@@ -4210,6 +4322,13 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
|
||||
n_patches /= 2;
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_HIGGS_AUDIO:
|
||||
{
|
||||
// stride-2 conv1d, avgpool(2), stride-2 depthwise conv1d
|
||||
n_patches = (img->nx() + 1) / 2;
|
||||
n_patches = n_patches / 2;
|
||||
n_patches = (n_patches + 1) / 2;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_QWEN3A:
|
||||
{
|
||||
// chunk_size=100 frames --> 3x stride-2 conv2d --> 13 tokens per chunk
|
||||
@@ -4449,6 +4568,8 @@ int mtmd_model::n_mmproj_embd() const {
|
||||
case PROJECTOR_TYPE_IDEFICS3:
|
||||
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
|
||||
return mm_fc_w->ne[1];
|
||||
case PROJECTOR_TYPE_VIBEVOICE_ASR:
|
||||
return mm_1_w->ne[1];
|
||||
case PROJECTOR_TYPE_ULTRAVOX:
|
||||
case PROJECTOR_TYPE_VOXTRAL:
|
||||
case PROJECTOR_TYPE_VOXTRAL_RT:
|
||||
@@ -4466,6 +4587,7 @@ int mtmd_model::n_mmproj_embd() const {
|
||||
case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
|
||||
return mm_fc_w->ne[1];
|
||||
case PROJECTOR_TYPE_QWEN3A:
|
||||
case PROJECTOR_TYPE_HIGGS_AUDIO:
|
||||
return mm_2_w->ne[1];
|
||||
case PROJECTOR_TYPE_GLMA:
|
||||
case PROJECTOR_TYPE_LFM2:
|
||||
|
||||
+19
-2
@@ -160,7 +160,7 @@ struct mtmd_hparams {
|
||||
int32_t audio_proj_downsample_rate = 0;
|
||||
int32_t audio_proj_head_count = 0;
|
||||
|
||||
// audio pyramid (muse-spark): widths, folded samples and blocks per stage
|
||||
// audio pyramid (muse-spark, vibevoice): widths, folded samples (vibevoice: conv strides) and blocks per stage
|
||||
std::vector<int32_t> audio_hidden_sizes;
|
||||
std::vector<int32_t> audio_patch_sizes;
|
||||
std::vector<int32_t> audio_depths;
|
||||
@@ -174,6 +174,9 @@ struct mtmd_hparams {
|
||||
return (int32_t) n;
|
||||
}
|
||||
|
||||
// vibevoice: leading samples of a chunk that only feed the causal convs, at least their receptive field
|
||||
int32_t audio_n_ctx = 0;
|
||||
|
||||
// audio-to-mel preprocessor params
|
||||
int32_t audio_chunk_len = -1; // in seconds
|
||||
int32_t audio_sample_rate = -1;
|
||||
@@ -528,6 +531,14 @@ struct mtmd_seanet {
|
||||
std::vector<stage> stages;
|
||||
};
|
||||
|
||||
// vibevoice-asr: causal conv pyramid over the raw waveform
|
||||
// stage = strided conv -> ConvNeXt blocks (RMS norm, depthwise conv, layer scale, then RMS norm, MLP, layer scale)
|
||||
struct mtmd_cnx_encoder {
|
||||
std::vector<ggml_tensor *> conv_w; // stem, one downsample conv per stage, head
|
||||
std::vector<ggml_tensor *> conv_b;
|
||||
std::vector<mtmd_layer> blocks; // ln_1, conv (depthwise), ls_1, ln_2, ffn_up, ffn_down, ls_2
|
||||
};
|
||||
|
||||
// pocket-tts flow-matching decoder (SimpleMLPAdaLN)
|
||||
struct mtmd_flow_net {
|
||||
// AdaLN res block: in_ln -> modulate -> Linear -> SiLU -> Linear, gated residual
|
||||
@@ -1160,7 +1171,9 @@ struct mtmd_model {
|
||||
ggml_tensor * mm_norm_mid_w = nullptr;
|
||||
|
||||
// mimo-audio-tokenizer: post-transformer downsample + RVQ codebook
|
||||
ggml_tensor * downsample_conv_w = nullptr; // no bias
|
||||
// higgs-audio: depthwise downsample conv of the projector
|
||||
ggml_tensor * downsample_conv_w = nullptr;
|
||||
ggml_tensor * downsample_conv_b = nullptr; // higgs-audio only
|
||||
ggml_tensor * downsample_norm_w = nullptr;
|
||||
ggml_tensor * downsample_norm_b = nullptr;
|
||||
ggml_tensor * rvq_codebook = nullptr; // merged 3D [n_q, max_bins, dim]
|
||||
@@ -1208,6 +1221,9 @@ struct mtmd_model {
|
||||
// pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path)
|
||||
mtmd_seanet seanet;
|
||||
|
||||
// vibevoice-asr: acoustic and semantic encoders
|
||||
mtmd_cnx_encoder cnx_enc[2];
|
||||
|
||||
// pocket-tts: voice latent -> backbone embd (speaker path)
|
||||
ggml_tensor * spk_proj_w = nullptr;
|
||||
ggml_tensor * downsample_w = nullptr;
|
||||
@@ -1332,6 +1348,7 @@ struct mtmd_model {
|
||||
|
||||
bool audio_has_avgpool() const {
|
||||
return proj_type == PROJECTOR_TYPE_QWEN2A
|
||||
|| proj_type == PROJECTOR_TYPE_HIGGS_AUDIO
|
||||
|| proj_type == PROJECTOR_TYPE_VOXTRAL
|
||||
|| proj_type == PROJECTOR_TYPE_MUSIC_FLAMINGO;
|
||||
}
|
||||
|
||||
@@ -402,6 +402,7 @@ static std::vector<c2w_state_slot> list_enc_state_slots(const mtmd_hparams & hpa
|
||||
switch (model.proj_type) {
|
||||
case PROJECTOR_TYPE_LFM3A: return list_lfm3a_state_slots(hparams, model);
|
||||
case PROJECTOR_TYPE_GEMMA4A: return list_gemma4a_state_slots(hparams, model);
|
||||
case PROJECTOR_TYPE_VIBEVOICE_ASR: return list_vibevoice_asr_state_slots(hparams, model);
|
||||
default: return {};
|
||||
}
|
||||
}
|
||||
@@ -451,7 +452,7 @@ mtmd_subctx_gen_state * mtmd_subcontext::gen_state_init(uint32_t seed) {
|
||||
|
||||
mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) {
|
||||
const auto & hparams = model.hparams;
|
||||
if (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A) {
|
||||
if (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A || model.proj_type == PROJECTOR_TYPE_VIBEVOICE_ASR) {
|
||||
// see list_enc_state_slots()
|
||||
const auto slots = list_enc_state_slots(hparams, model);
|
||||
auto st = std::make_unique<mtmd_subctx_enc_state>();
|
||||
@@ -1662,6 +1663,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
|
||||
case PROJECTOR_TYPE_NEMOTRON_OMNI:
|
||||
case PROJECTOR_TYPE_QWEN2A:
|
||||
case PROJECTOR_TYPE_HIGGS_AUDIO:
|
||||
case PROJECTOR_TYPE_QWEN3A:
|
||||
case PROJECTOR_TYPE_GLMA:
|
||||
case PROJECTOR_TYPE_ULTRAVOX:
|
||||
@@ -1982,6 +1984,10 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
}
|
||||
set_input_i32("positions", positions);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VIBEVOICE_ASR:
|
||||
{
|
||||
set_enc_state_in();
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GEMMA4A:
|
||||
{
|
||||
GGML_ASSERT(imgs.entries.size() == 1);
|
||||
@@ -2627,7 +2633,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (params->enc_state != nullptr && (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A)) {
|
||||
if (params->enc_state != nullptr && (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A || model.proj_type == PROJECTOR_TYPE_VIBEVOICE_ASR)) {
|
||||
const auto & slots = params->enc_state->slots;
|
||||
size_t i = 0;
|
||||
for (const auto & slot : list_enc_state_slots(hparams, model)) {
|
||||
|
||||
+274
-2
@@ -48,7 +48,7 @@ std::string mtmd_format(const char * fmt, ...) {
|
||||
}
|
||||
|
||||
// remember to bump this if the serialization format changes
|
||||
#define MTMD_SERIALIZATION_VERSION 4
|
||||
#define MTMD_SERIALIZATION_VERSION 5
|
||||
|
||||
// oldest compat version that can be loaded
|
||||
#define MTMD_SERIALIZATION_VERSION_MIN 2
|
||||
@@ -137,6 +137,7 @@ void mtmd_image_f32::serialize(mtmd_serialization & ser) const {
|
||||
ser.write((int32_t)anyres.orig_ny);
|
||||
ser.write((int32_t)nx_);
|
||||
ser.write((int32_t)ny_);
|
||||
ser.write(audio_n_ctx);
|
||||
}
|
||||
void mtmd_image_f32::deserialize(mtmd_serialization & ser) {
|
||||
add_viewsep = ser.read<bool>();
|
||||
@@ -155,6 +156,12 @@ void mtmd_image_f32::deserialize(mtmd_serialization & ser) {
|
||||
}
|
||||
nx_ = ser.read<int32_t>();
|
||||
ny_ = ser.read<int32_t>();
|
||||
if (ser.version >= 5) {
|
||||
audio_n_ctx = ser.read<int32_t>();
|
||||
if (audio_n_ctx < 0 || audio_n_ctx > nx_) {
|
||||
throw std::runtime_error("invalid audio_n_ctx");
|
||||
}
|
||||
}
|
||||
buf.clear(); // always a placeholder after loading
|
||||
}
|
||||
|
||||
@@ -1194,6 +1201,19 @@ struct mtmd_context {
|
||||
aud_end = "<|audio_eos|>";
|
||||
audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(models.a.get());
|
||||
} break;
|
||||
case PROJECTOR_TYPE_HIGGS_AUDIO:
|
||||
{
|
||||
// the 4s chunks are joined in one <|audio_bos|> ... <|audio_eos|> span
|
||||
aud_beg = "<|audio_bos|>";
|
||||
aud_end = "<|audio_eos|>";
|
||||
audio_preproc = std::make_unique<mtmd_audio_preprocessor_higgs_audio>(models.a.get());
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VIBEVOICE_ASR:
|
||||
{
|
||||
aud_beg = "<|object_ref_start|>";
|
||||
aud_end = "<|object_ref_end|>";
|
||||
audio_preproc = std::make_unique<mtmd_audio_preprocessor_vibevoice_asr>(models.a.get());
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
|
||||
{
|
||||
aud_beg = "<|soft_audio_start|>";
|
||||
@@ -4270,12 +4290,58 @@ struct mtmd_audio_stream_gemma4a : mtmd_audio_stream {
|
||||
bool encode(int n);
|
||||
};
|
||||
|
||||
// Higgs Audio: each 4 s chunk is encoded on its own, so a chunk goes out once it is complete
|
||||
// a flush encodes the partial chunk as a chunk of its own, the reference also cuts chunks at VAD boundaries
|
||||
struct mtmd_audio_stream_higgs_audio : mtmd_audio_stream {
|
||||
int64_t chunk_len = 0; // samples per complete chunk
|
||||
|
||||
std::vector<float> beg_rows;
|
||||
std::vector<float> end_rows;
|
||||
bool beg_done = false;
|
||||
bool end_done = false;
|
||||
|
||||
bool init(const mtmd_audio_stream_params & params) override;
|
||||
int32_t flush(bool end) override;
|
||||
int32_t step(llama_token last_token) override;
|
||||
|
||||
int64_t n_chunk_rows(int64_t n) const;
|
||||
bool encode(int64_t n);
|
||||
};
|
||||
|
||||
// VibeVoice-ASR: the encoder is causal, its convs keep their left context in the state
|
||||
// the reference scales the whole clip to -25 dBFS, here the gain comes from the audio pushed so far (see mtmd_audio_vibevoice_gain)
|
||||
struct mtmd_audio_stream_vibevoice_asr : mtmd_audio_stream {
|
||||
mtmd_subctx_enc_state * enc_state = nullptr;
|
||||
|
||||
int64_t n_step = 0; // samples per encoder call
|
||||
int64_t n_step_max = 0; // largest call, the warmup size
|
||||
int64_t n_stat = 0; // pushed samples counted in sum_sq and peak
|
||||
double sum_sq = 0.0;
|
||||
double peak = 0.0;
|
||||
|
||||
std::vector<float> beg_rows;
|
||||
std::vector<float> end_rows;
|
||||
bool beg_done = false;
|
||||
bool end_done = false;
|
||||
|
||||
~mtmd_audio_stream_vibevoice_asr() override {
|
||||
delete enc_state;
|
||||
}
|
||||
|
||||
bool init(const mtmd_audio_stream_params & params) override;
|
||||
int32_t flush(bool end) override;
|
||||
int32_t step(llama_token last_token) override;
|
||||
|
||||
bool encode(int64_t n);
|
||||
};
|
||||
|
||||
bool mtmd_support_audio_stream(const mtmd_context * ctx) {
|
||||
if (!ctx->ctx_a || !ctx->text_model) {
|
||||
return false;
|
||||
}
|
||||
const projector_type proj = ctx->ctx_a->model.proj_type;
|
||||
return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A || proj == PROJECTOR_TYPE_GEMMA4A;
|
||||
return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A || proj == PROJECTOR_TYPE_GEMMA4A ||
|
||||
proj == PROJECTOR_TYPE_HIGGS_AUDIO || proj == PROJECTOR_TYPE_VIBEVOICE_ASR;
|
||||
}
|
||||
|
||||
mtmd_audio_stream_params mtmd_audio_stream_params_default(void) {
|
||||
@@ -4298,6 +4364,8 @@ mtmd_audio_stream * mtmd_audio_stream_init(mtmd_context * ctx, mtmd_audio_stream
|
||||
case PROJECTOR_TYPE_VOXTRAL_RT: s = std::make_unique<mtmd_audio_stream_voxtral_rt>(); break;
|
||||
case PROJECTOR_TYPE_LFM3A: s = std::make_unique<mtmd_audio_stream_lfm3a>(); break;
|
||||
case PROJECTOR_TYPE_GEMMA4A: s = std::make_unique<mtmd_audio_stream_gemma4a>(); break;
|
||||
case PROJECTOR_TYPE_HIGGS_AUDIO: s = std::make_unique<mtmd_audio_stream_higgs_audio>(); break;
|
||||
case PROJECTOR_TYPE_VIBEVOICE_ASR: s = std::make_unique<mtmd_audio_stream_vibevoice_asr>(); break;
|
||||
default: GGML_ABORT("unsupported projector");
|
||||
}
|
||||
s->ctx = ctx;
|
||||
@@ -5036,6 +5104,210 @@ int32_t mtmd_audio_stream_gemma4a::step(llama_token /* last_token */) {
|
||||
return (int32_t) n_out;
|
||||
}
|
||||
|
||||
// encodes one audio entry, the rows are appended to embd
|
||||
static bool mtmd_audio_stream_encode(mtmd_audio_stream & s, mtmd_image_f32 && img, mtmd_subctx_enc_state * enc_state) {
|
||||
const int n_out = s.sctx->model.n_output_tokens(&img);
|
||||
|
||||
mtmd_image_f32_batch batch;
|
||||
batch.is_audio = true;
|
||||
batch.entries.push_back(std::move(img));
|
||||
|
||||
std::vector<float> res((size_t) s.n_embd * n_out);
|
||||
mtmd_subctx_encode_params params;
|
||||
params.n_threads = s.ctx->n_threads;
|
||||
params.imgs = &batch;
|
||||
params.out_embd = &res;
|
||||
params.enc_state = enc_state;
|
||||
if (!s.sctx->encode(¶ms)) {
|
||||
LOG_ERR("%s: failed to encode audio\n", __func__);
|
||||
return false;
|
||||
}
|
||||
s.embd.insert(s.embd.end(), res.begin(), res.end());
|
||||
return true;
|
||||
}
|
||||
|
||||
// rows carry no text: all of them go out at once between the audio markers, last_token is not used
|
||||
static int32_t mtmd_audio_stream_emit_rows(mtmd_audio_stream & s, const std::vector<float> & beg_rows, const std::vector<float> & end_rows,
|
||||
bool & beg_done, bool & end_done) {
|
||||
const int64_t n_avail = (int64_t) (s.embd.size() / s.n_embd);
|
||||
s.out.clear();
|
||||
if (n_avail > 0 || s.ended) {
|
||||
if (!beg_done) {
|
||||
beg_done = true;
|
||||
s.out = beg_rows;
|
||||
}
|
||||
s.out.insert(s.out.end(), s.embd.begin(), s.embd.end());
|
||||
s.embd.clear();
|
||||
if (s.ended && !end_done) {
|
||||
end_done = true;
|
||||
s.out.insert(s.out.end(), end_rows.begin(), end_rows.end());
|
||||
}
|
||||
}
|
||||
const int64_t n_out = (int64_t) (s.out.size() / s.n_embd);
|
||||
if (n_out == 0) {
|
||||
return s.ended ? -1 : 0;
|
||||
}
|
||||
s.n_rows += n_out;
|
||||
s.n_rows_done += n_avail; // the markers are not audio rows
|
||||
return (int32_t) n_out;
|
||||
}
|
||||
|
||||
//
|
||||
// Higgs Audio
|
||||
//
|
||||
|
||||
bool mtmd_audio_stream_higgs_audio::init(const mtmd_audio_stream_params & params) {
|
||||
const mtmd_hparams * hparams = &sctx->model.hparams;
|
||||
if (params.n_prompt > 0) {
|
||||
LOG_ERR("%s: Higgs Audio takes no prompt, its audio is a chat turn\n", __func__);
|
||||
return false;
|
||||
}
|
||||
if (!ctx->audio_preproc) {
|
||||
LOG_ERR("%s: the audio preprocessor is not initialized\n", __func__);
|
||||
return false;
|
||||
}
|
||||
chunk_len = (int64_t) hparams->audio_chunk_len * hparams->audio_sample_rate;
|
||||
frames_per_tok = 8; // stride-2 conv, avgpool, stride-2 projector conv
|
||||
samples_per_tok = frames_per_tok * hop;
|
||||
return mtmd_audio_stream_marker_rows(ctx, ctx->aud_beg, beg_rows) && mtmd_audio_stream_marker_rows(ctx, ctx->aud_end, end_rows);
|
||||
}
|
||||
|
||||
// rows of a chunk with n samples, see mtmd_audio_preprocessor_higgs_audio
|
||||
int64_t mtmd_audio_stream_higgs_audio::n_chunk_rows(int64_t n) const {
|
||||
const int64_t n_frames = n / hop;
|
||||
if (n_frames < 3) {
|
||||
return 0;
|
||||
}
|
||||
return ((n_frames + 1) / 2 / 2 + 1) / 2;
|
||||
}
|
||||
|
||||
bool mtmd_audio_stream_higgs_audio::encode(int64_t n) {
|
||||
mtmd_audio_preproc_out pre = ctx->audio_preproc->preprocess(buf.data(), (size_t) n);
|
||||
if (pre.entries.empty() || !ctx->audio_preproc->compute(pre)) {
|
||||
LOG_ERR("%s: unable to preprocess the audio\n", __func__);
|
||||
return false;
|
||||
}
|
||||
for (auto & entry : pre.entries) {
|
||||
if (!mtmd_audio_stream_encode(*this, std::move(entry), nullptr)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
buf.erase(buf.begin(), buf.begin() + n);
|
||||
buf_pos += n;
|
||||
return true;
|
||||
}
|
||||
|
||||
int32_t mtmd_audio_stream_higgs_audio::flush(bool end) {
|
||||
const int64_t n_pending = n_samples - buf_pos;
|
||||
n_rows_flushed = n_rows_done + (int64_t) (embd.size() / n_embd) + (n_pending / chunk_len) * n_chunk_rows(chunk_len) + n_chunk_rows(n_pending % chunk_len);
|
||||
pending_flush = true;
|
||||
ended = end;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t mtmd_audio_stream_higgs_audio::step(llama_token /* last_token */) {
|
||||
const bool flush = pending_flush;
|
||||
pending_flush = false;
|
||||
|
||||
while ((int64_t) buf.size() >= chunk_len) {
|
||||
if (!encode(chunk_len)) {
|
||||
return -2;
|
||||
}
|
||||
}
|
||||
// a tail too short for a row is kept until the next flush, or dropped at the end
|
||||
if ((flush || ended) && !buf.empty()) {
|
||||
if (n_chunk_rows((int64_t) buf.size()) > 0) {
|
||||
if (!encode((int64_t) buf.size())) {
|
||||
return -2;
|
||||
}
|
||||
} else if (ended) {
|
||||
buf_pos += (int64_t) buf.size();
|
||||
buf.clear();
|
||||
}
|
||||
}
|
||||
return mtmd_audio_stream_emit_rows(*this, beg_rows, end_rows, beg_done, end_done);
|
||||
}
|
||||
|
||||
//
|
||||
// VibeVoice-ASR
|
||||
//
|
||||
|
||||
bool mtmd_audio_stream_vibevoice_asr::init(const mtmd_audio_stream_params & params) {
|
||||
const mtmd_hparams * hparams = &sctx->model.hparams;
|
||||
if (params.n_prompt > 0) {
|
||||
LOG_ERR("%s: VibeVoice-ASR takes no prompt, its audio is a chat turn\n", __func__);
|
||||
return false;
|
||||
}
|
||||
hop = hparams->audio_samples_per_token();
|
||||
frames_per_tok = 1;
|
||||
samples_per_tok = hop;
|
||||
n_step = std::max<int64_t>(1, ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 + hop - 1) / hop) * hop;
|
||||
n_step_max = std::max(n_step, (int64_t) hparams->audio_chunk_len * hparams->audio_sample_rate / hop * hop);
|
||||
|
||||
enc_state = sctx->enc_state_init(0);
|
||||
if (!enc_state) {
|
||||
return false;
|
||||
}
|
||||
return mtmd_audio_stream_marker_rows(ctx, ctx->aud_beg, beg_rows) && mtmd_audio_stream_marker_rows(ctx, ctx->aud_end, end_rows);
|
||||
}
|
||||
|
||||
// n is a multiple of hop, see mtmd_audio_preprocessor_vibevoice_asr for the gain
|
||||
bool mtmd_audio_stream_vibevoice_asr::encode(int64_t n) {
|
||||
const double gain = mtmd_audio_vibevoice_gain(sum_sq, n_stat, peak);
|
||||
|
||||
std::vector<float> data((size_t) n);
|
||||
for (int64_t i = 0; i < n; i++) {
|
||||
data[i] = (float) (buf[i] * gain);
|
||||
}
|
||||
mtmd_image_f32 img;
|
||||
img.set_size({(int) n, 1}, false, /* is_audio */ true);
|
||||
img.cpy_buf(data);
|
||||
if (!mtmd_audio_stream_encode(*this, std::move(img), enc_state)) {
|
||||
return false;
|
||||
}
|
||||
buf.erase(buf.begin(), buf.begin() + n);
|
||||
buf_pos += n;
|
||||
return true;
|
||||
}
|
||||
|
||||
int32_t mtmd_audio_stream_vibevoice_asr::flush(bool end) {
|
||||
n_rows_flushed = end ? (n_samples + hop - 1) / hop : n_samples / hop;
|
||||
pending_flush = true;
|
||||
ended = end;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t mtmd_audio_stream_vibevoice_asr::step(llama_token /* last_token */) {
|
||||
const bool flush = pending_flush;
|
||||
pending_flush = false;
|
||||
|
||||
// the padding added at the end does not count in the gain
|
||||
for (int64_t i = n_stat - buf_pos; i < (int64_t) buf.size(); i++) {
|
||||
sum_sq += (double) buf[i] * buf[i];
|
||||
peak = std::max(peak, (double) std::fabs(buf[i]));
|
||||
}
|
||||
n_stat = n_samples;
|
||||
if (ended && n_samples % hop != 0) {
|
||||
add_silence(hop - n_samples % hop);
|
||||
n_stat = n_samples;
|
||||
}
|
||||
|
||||
// whole rows only: a partial row would put its zero padding into the state
|
||||
int64_t n = (int64_t) buf.size();
|
||||
n -= n % hop;
|
||||
if (!flush && !ended) {
|
||||
n -= n % n_step;
|
||||
}
|
||||
while (n > 0) {
|
||||
const int64_t m = std::min(n, n_step_max);
|
||||
if (!encode(m)) {
|
||||
return -2;
|
||||
}
|
||||
n -= m;
|
||||
}
|
||||
return mtmd_audio_stream_emit_rows(*this, beg_rows, end_rows, beg_done, end_done);
|
||||
}
|
||||
|
||||
int64_t mtmd_audio_stream_get_n_rows(const mtmd_audio_stream * s) {
|
||||
return s->n_rows_done;
|
||||
}
|
||||
|
||||
@@ -739,14 +739,22 @@ json convert_transcriptions_to_chatcmpl(
|
||||
if (response_format != "json") {
|
||||
throw std::invalid_argument("Only 'json' response_format is supported for transcription");
|
||||
}
|
||||
const common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language);
|
||||
if (prompt.empty()) {
|
||||
common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language, 0.0, prompt);
|
||||
if (preset.needs_audio_length) {
|
||||
mtmd_helper_media_meta meta;
|
||||
if (mtmd_helper_get_media_meta(out_files[0].data(), out_files[0].size(), mtmd_helper_init_opt_default(), &meta) != 0 ||
|
||||
meta.type != MTMD_HELPER_MEDIA_TYPE_AUDIO || meta.duration < 0) {
|
||||
throw std::invalid_argument("the input file is not a supported audio file");
|
||||
}
|
||||
preset = common_chat_get_asr_prompt(tmpls, language, meta.duration, prompt);
|
||||
}
|
||||
if (prompt.empty() || preset.has_context) {
|
||||
prompt = preset.user;
|
||||
}
|
||||
if (!language.empty() && !preset.has_language) {
|
||||
prompt += string_format(" (language: %s)", language.c_str());
|
||||
}
|
||||
prompt += get_media_marker();
|
||||
prompt = preset.media_first ? get_media_marker() + prompt : prompt + get_media_marker();
|
||||
|
||||
json messages = json::array();
|
||||
if (!preset.system.empty()) {
|
||||
|
||||
@@ -155,6 +155,7 @@ struct realtime_conversation {
|
||||
bool started = false; // something went into the slot, the session config is fixed
|
||||
bool turn_open = false; // a user turn is being filled
|
||||
bool has_audio = false; // the open turn has audio not committed yet
|
||||
int64_t n_turn_samples = 0; // audio of the open turn, at the model rate
|
||||
bool tool_results = false; // tool results went in after the last response, the next one answers them
|
||||
int n_turns = 0;
|
||||
std::string next_header; // text that goes in before the next user turn
|
||||
@@ -376,8 +377,10 @@ struct realtime_conversation {
|
||||
if (!resample(samples) || samples.empty() || !ensure_turn()) {
|
||||
return;
|
||||
}
|
||||
const int64_t n = (int64_t) samples.size();
|
||||
if (control(SERVER_CONTROL_ACTION_AUDIO_APPEND, {}, std::move(samples))) {
|
||||
has_audio = true;
|
||||
n_turn_samples += n;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -410,7 +413,8 @@ struct realtime_conversation {
|
||||
if (!control(SERVER_CONTROL_ACTION_AUDIO_END)) {
|
||||
return false;
|
||||
}
|
||||
has_audio = false;
|
||||
has_audio = false;
|
||||
n_turn_samples = 0;
|
||||
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
const json prev = last_item_id.empty() ? json(nullptr) : json(last_item_id);
|
||||
@@ -424,6 +428,7 @@ struct realtime_conversation {
|
||||
}
|
||||
|
||||
void on_commit() {
|
||||
const int64_t n_audio = n_turn_samples;
|
||||
if (!commit() || type != "transcription") {
|
||||
return;
|
||||
}
|
||||
@@ -433,11 +438,19 @@ struct realtime_conversation {
|
||||
transcribe_item_id = last_item_id;
|
||||
transcript.clear();
|
||||
}
|
||||
// some models (VibeVoice-ASR) take an instruction after the audio, with its length
|
||||
const auto * tmpls = deps.meta.chat_params.tmpls.get();
|
||||
common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language);
|
||||
if (preset.needs_audio_length) {
|
||||
preset = common_chat_get_asr_prompt(tmpls, language, (double) n_audio / mtmd_get_audio_sample_rate(deps.mctx));
|
||||
}
|
||||
const std::string after_audio = preset.media_first ? preset.user : "";
|
||||
|
||||
server_control_response response;
|
||||
response.n_predict = max_output_tokens;
|
||||
response.out_mode = MTMD_HELPER_GEN_AUDIO_OUT_TEXT;
|
||||
response.sampling = response_sampling(nullptr);
|
||||
if (!control(SERVER_CONTROL_ACTION_RESPONSE_CREATE, user_footer + assistant_header, {}, response)) {
|
||||
if (!control(SERVER_CONTROL_ACTION_RESPONSE_CREATE, after_audio + user_footer + assistant_header, {}, response)) {
|
||||
return;
|
||||
}
|
||||
control(SERVER_CONTROL_ACTION_CLEAR);
|
||||
|
||||
Reference in new issue
Block a user