From a68c6136bb980d35281ae68d275d6f5318accc4a Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Mon, 16 Nov 2026 17:32:27 +0100 Subject: [PATCH] mtmd: support fish audio s2 (#32330) * add fisk-audio-s2 support * add voice cloning * cont merge master * nits * nits * clean up * nits * use new batch api --- common/arg.cpp | 8 + common/common.h | 1 + conversion/__init__.py | 2 + conversion/base.py | 3 + conversion/drama3.py | 51 +++-- conversion/fish_s2.py | 276 ++++++++++++++++++++++++++ docs/multimodal/tts.md | 43 +++++ gguf-py/gguf/constants.py | 66 +++++++ tools/mtmd/CMakeLists.txt | 2 + tools/mtmd/README-dev.md | 5 +- tools/mtmd/models/fish-s2-gen.cpp | 74 +++++++ tools/mtmd/models/fish-s2-spkenc.cpp | 135 +++++++++++++ tools/mtmd/models/models.h | 29 ++- tools/mtmd/models/qwen3tts-gen.cpp | 15 +- tools/mtmd/mtmd-audio.cpp | 10 + tools/mtmd/mtmd-audio.h | 6 + tools/mtmd/mtmd-graph.cpp | 16 +- tools/mtmd/mtmd-helper-gen.cpp | 279 ++++++++++++++++++++++++--- tools/mtmd/mtmd-helper.h | 6 +- tools/mtmd/mtmd-impl.h | 29 +++ tools/mtmd/mtmd-model.cpp | 107 +++++++++- tools/mtmd/mtmd-model.h | 9 + tools/mtmd/mtmd-subcontext.cpp | 14 +- tools/mtmd/mtmd.cpp | 8 + tools/mtmd/mtmd.h | 1 + tools/server/README.md | 2 + tools/server/server-context.cpp | 1 + tools/tts/README.md | 1 + tools/tts/tts.cpp | 3 +- 29 files changed, 1140 insertions(+), 62 deletions(-) create mode 100644 conversion/fish_s2.py create mode 100644 tools/mtmd/models/fish-s2-gen.cpp create mode 100644 tools/mtmd/models/fish-s2-spkenc.cpp diff --git a/common/arg.cpp b/common/arg.cpp index c5fcecac9..5dc7c8d3a 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -4522,6 +4522,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.tts_speaker_file = value; } ).set_examples({LLAMA_EXAMPLE_TTS})); + add_opt(common_arg( + {"--tts-speaker-text"}, "TEXT", + "transcript of --tts-speaker-file, for models that need it to clone the voice\n" + "see docs/multimodal/tts.md for per-model usage notes", + [](common_params & params, const std::string & value) { + params.tts_speaker_text = value; + } + ).set_examples({LLAMA_EXAMPLE_TTS})); // // diffusion params diff --git a/common/common.h b/common/common.h index 4cfc1b8f3..9b9e892d8 100644 --- a/common/common.h +++ b/common/common.h @@ -796,6 +796,7 @@ struct common_params { std::string tts_lang = ""; std::string tts_voice = ""; std::string tts_speaker_file = ""; + std::string tts_speaker_text = ""; bool is_gen_docs = false; // whether we are running inside llama-gen-docs }; diff --git a/conversion/__init__.py b/conversion/__init__.py index 2ec0551e8..05f55fccf 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -336,6 +336,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "XverseForCausalLM": "xverse", "YoutuForCausalLM": "deepseek", "YoutuVLForConditionalGeneration": "deepseek", + "fish_qwen3_omni": "fish_s2", "modeling_grove_moe.GroveMoeForCausalLM": "grovemoe", "modeling_sarvam_moe.SarvamMoEForCausalLM": "bailingmoe", } @@ -438,6 +439,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "VoxtralForConditionalGeneration": "ultravox", "VoxtralRealtimeForConditionalGeneration": "ultravox", "YoutuVLForConditionalGeneration": "youtuvl", + "fish_qwen3_omni": "fish_s2", } diff --git a/conversion/base.py b/conversion/base.py index b8eba85c7..a95ecc008 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -3488,6 +3488,9 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st # some models (llava_onevision2) only declare the architecture inside text_config if arch is None and text_config.get("architectures") is not None: arch = text_config["architectures"][0] + # some configs only have a model_type (fishaudio/s2-pro) + if arch is None: + arch = hparams.get("model_type") if arch is None: raise ValueError("Failed to detect model architecture") return arch diff --git a/conversion/drama3.py b/conversion/drama3.py index ac01f481c..68bbb556b 100644 --- a/conversion/drama3.py +++ b/conversion/drama3.py @@ -18,6 +18,30 @@ _TOWER_SHARED = "audio_decoder_mtp_shared.layers." _TOWER_RESIDUAL = "audio_decoder_mtp.layers." +def load_codec(dir_model) -> dict[str, Tensor]: + sd = torch.load(dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True) + return sd.get("state_dict", sd) + + +def fold_conv(sd: dict[str, Tensor], prefix: str) -> Tensor: + # fold weight norm + if prefix + ".conv.weight" in sd: + return sd[prefix + ".conv.weight"].float() + g = sd[prefix + ".conv.parametrizations.weight.original0"].float() + v = sd[prefix + ".conv.parametrizations.weight.original1"].float() + return g * v / v.norm(dim=(1, 2), keepdim=True) + + +def fold_wn(g: Tensor, v: Tensor) -> Tensor: + # fold old-style weight norm of a 1x1 conv, returns [out, in] + return (g.float() * v.float() / v.float().norm(dim=(1, 2), keepdim=True)).squeeze(-1) + + +def fold_snake(alpha: Tensor) -> tuple[Tensor, Tensor]: + alpha = alpha.float().flatten() + return alpha, 1.0 / (alpha + 1e-9) + + @ModelBase.register("FishOmniModel") @ModelBase.example("fishaudio/drama-3") class Drama3TextModel(Qwen3_5MoeTextModel): @@ -36,6 +60,7 @@ class Drama3TextModel(Qwen3_5MoeTextModel): class Drama3MmprojModel(MmprojModel): has_vision_encoder = False has_audio_encoder = False + gen_projector_type = gguf.VisionProjectorType.DRAMA3_GEN _LAYER_MAP = { "enorm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ENORM, True), @@ -69,7 +94,7 @@ class Drama3MmprojModel(MmprojModel): cfg = self.hparams self.gguf_writer.add_file_type(self.ftype) self.gguf_writer.add_clip_has_gen_audio_encoder(True) - self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.DRAMA3_GEN) + self.gguf_writer.add_clip_gen_audio_projector_type(self.gen_projector_type) self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text) self.gguf_writer.add_gen_audio_embedding_length(cfg["dim"]) self.gguf_writer.add_gen_audio_feed_forward_length(cfg["intermediate_size"]) @@ -153,32 +178,20 @@ class Drama3MmprojModel(MmprojModel): def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: # only the DAC decoder, the encoder is for voice cloning (not supported yet) - sd = torch.load(self.dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True) - sd = sd.get("state_dict", sd) + sd = load_codec(self.dir_model) T = gguf.MODEL_TENSOR def get(name: str) -> Tensor: return sd[name].float() def conv(prefix: str) -> Tensor: - # fold weight norm - if prefix + ".conv.weight" in sd: - return get(prefix + ".conv.weight") - g = get(prefix + ".conv.parametrizations.weight.original0") - v = get(prefix + ".conv.parametrizations.weight.original1") - return g * v / v.norm(dim=(1, 2), keepdim=True) + return fold_conv(sd, prefix) def quant_table(prefix: str) -> Tensor: # fold out_proj into the codebook, so decoding is a row lookup - g = get(prefix + ".out_proj.weight_g") - v = get(prefix + ".out_proj.weight_v") - w = (g * v / v.norm(dim=(1, 2), keepdim=True)).squeeze(-1) + w = fold_wn(sd[prefix + ".out_proj.weight_g"], sd[prefix + ".out_proj.weight_v"]) return get(prefix + ".codebook.weight") @ w.T + get(prefix + ".out_proj.bias") - def snake(alpha: Tensor) -> tuple[Tensor, Tensor]: - alpha = alpha.flatten() - return alpha, 1.0 / (alpha + 1e-9) - q = "quantizer." yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), quant_table(q + "semantic_quantizer.quantizers.0")) n_rest = sum(1 for k in sd if k.startswith(q + "quantizer.quantizers.") and k.endswith(".codebook.weight")) @@ -228,7 +241,7 @@ class Drama3MmprojModel(MmprojModel): yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".bias"), get(d + "0.conv.bias")) for bid in range(4): p = f"{d}{bid + 1}.block." - a, b = snake(get(p + "0.alpha")) + a, b = fold_snake(get(p + "0.alpha")) yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".alpha"), a) yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".beta"), b) yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".weight"), conv(p + "1")) @@ -236,7 +249,7 @@ class Drama3MmprojModel(MmprojModel): for xid in range(3): r = f"{p}{xid + 2}.block." for act, tensor in (("0", T.A_GEN_WAV_DAC_RES_ACT1), ("2", T.A_GEN_WAV_DAC_RES_ACT2)): - a, b = snake(get(r + act + ".alpha")) + a, b = fold_snake(get(r + act + ".alpha")) name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid) yield (name + ".alpha", a) yield (name + ".beta", b) @@ -244,7 +257,7 @@ class Drama3MmprojModel(MmprojModel): name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid) yield (name + ".weight", conv(r + cv)) yield (name + ".bias", get(r + cv + ".conv.bias")) - a, b = snake(get(d + "5.alpha")) + a, b = fold_snake(get(d + "5.alpha")) yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".alpha"), a) yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b) yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), conv(d + "6")) diff --git a/conversion/fish_s2.py b/conversion/fish_s2.py new file mode 100644 index 000000000..4c5aeff13 --- /dev/null +++ b/conversion/fish_s2.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, Callable, Iterable, TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, gguf +from .drama3 import Drama3MmprojModel, fold_conv, fold_snake, fold_wn, load_codec +from .qwen import Qwen3Model + +# Fish Audio S2 Pro (dual-AR) +# text model: Qwen3 backbone (slow AR), samples the semantic code of each frame +# mmproj: fast AR code predictor (other codebooks) and Fish DAC decoder (codec.pth, same as drama3) + +# residual codebook size of codec.pth, the fast AR head is cut to it +_N_RESIDUAL_CODES = 1024 + +# codec encoder strides (encoder_rates of the codec config), the quantizer then downsamples 2x2 +_CODEC_ENC_STRIDES = (2, 4, 8, 8) + +# codec encoder transformers, both 1024 wide with 16 heads of 64 +# generic audio layers a.blk.0..3: last encoder block, a.blk.4..11: quantizer pre_module +_N_ENC_TFM_LAYERS = 4 +_N_PRE_TFM_LAYERS = 8 +_CODEC_HEAD_DIM = 64 + + +def _load_hparams(dir_model: Path) -> dict[str, Any]: + hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False) + cfg = hparams["text_config"] + hparams["text_config"] = { + "hidden_size": cfg["dim"], + "num_hidden_layers": cfg["n_layer"], + "num_attention_heads": cfg["n_head"], + "num_key_value_heads": cfg["n_local_heads"], + "head_dim": cfg["head_dim"], + "intermediate_size": cfg["intermediate_size"], + "rms_norm_eps": cfg["norm_eps"], + "rope_theta": cfg["rope_base"], + "max_position_embeddings": cfg["max_seq_len"], + "vocab_size": cfg["vocab_size"], + "tie_word_embeddings": cfg["tie_word_embeddings"], + } + return hparams + + +def _neox_rows(w: Tensor, head_dim: int) -> Tensor: + # checkpoint uses interleaved RoPE, reorder the rows of each head so NEOX RoPE gives the same result + order = torch.cat([torch.arange(0, head_dim, 2), torch.arange(1, head_dim, 2)]) + return w.reshape(-1, head_dim, *w.shape[1:])[:, order].reshape(w.shape) + + +@ModelBase.register("fish_qwen3_omni") +@ModelBase.example("fishaudio/s2-pro") +class FishS2TextModel(Qwen3Model): + model_arch = gguf.MODEL_ARCH.QWEN3 + + def __init__(self, dir_model: Path, *args, **kwargs): + kwargs.pop("hparams", None) + super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # the rest is mapped by tensor_mapping.py + name = name.removeprefix("text_model.") + T = gguf.MODEL_TENSOR + head_dim = self.hparams["head_dim"] + if name.endswith(".attention.wqkv.weight"): + n_q = self.hparams["num_attention_heads"] * head_dim + n_kv = self.hparams["num_key_value_heads"] * head_dim + q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0) + yield (self.format_tensor_name(T.ATTN_Q, bid), _neox_rows(q, head_dim)) + yield (self.format_tensor_name(T.ATTN_K, bid), _neox_rows(k, head_dim)) + yield (self.format_tensor_name(T.ATTN_V, bid), v) + return + if name.endswith(".attention.q_norm.weight"): + yield (self.format_tensor_name(T.ATTN_Q_NORM, bid), _neox_rows(data_torch, head_dim)) + return + if name.endswith(".attention.k_norm.weight"): + yield (self.format_tensor_name(T.ATTN_K_NORM, bid), _neox_rows(data_torch, head_dim)) + return + yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("fish_qwen3_omni") +@ModelBase.example("fishaudio/s2-pro") +class FishS2MmprojModel(Drama3MmprojModel): + has_audio_encoder = True # codec encoder, for voice cloning + gen_projector_type = gguf.VisionProjectorType.FISH_S2_GEN + + _FAST_AR_MAP = { + "attention_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM, + "attention.wo.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT, + "ffn_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM, + "feed_forward.w1.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE, + "feed_forward.w3.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP, + "feed_forward.w2.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN, + } + + def __init__(self, dir_model: Path, *args, **kwargs): + kwargs.pop("hparams", None) + super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs) + + def get_audio_config(self) -> dict[str, Any] | None: + cfg = dict(self.global_config["audio_decoder_config"]) + cfg["num_hidden_layers"] = cfg["n_layer"] + return cfg + + def set_gguf_parameters(self): + super().set_gguf_parameters() + # codec encoder: raw 44.1 kHz waveform -> backbone input rows, other hparams are hardcoded in tools/mtmd + self.gguf_writer.add_clip_has_audio_encoder(True) + self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.FISH_S2_SPKENC) + self.gguf_writer.add_audio_projection_dim(self.n_embd_text) + self.gguf_writer.add_audio_num_mel_bins(1) + self.gguf_writer.add_audio_block_count(_N_ENC_TFM_LAYERS + _N_PRE_TFM_LAYERS) + self.gguf_writer.add_audio_embedding_length(1024) + self.gguf_writer.add_audio_feed_forward_length(3072) + self.gguf_writer.add_audio_head_count(1024 // _CODEC_HEAD_DIM) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) + + def tensor_force_quant(self, name, new_name, bid, n_dims): + # the VQ picks the nearest codebook entry, small errors in the encoder change the codes + if new_name.startswith("a.codec."): + return gguf.GGMLQuantizationType.F32 + # codec transformers: F16 gives codes closer to the reference than BF16 + if new_name.startswith("a.blk.") and n_dims == 2: + return gguf.GGMLQuantizationType.F16 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name = item[0] + if name.startswith("audio_decoder.") or name == "text_model.model.embeddings.weight": + return item + return None + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + T = gguf.MODEL_TENSOR + cfg = self.hparams + + # already named by generate_extra_tensors() + if name.startswith("a."): + yield (name, data_torch) + return + + if name == "text_model.model.embeddings.weight": + # semantic band of the backbone embeddings + begin = self.global_config["semantic_start_token_id"] + yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch[begin:begin + cfg["vocab_size"]]) + return + + if name == "audio_decoder.codebook_embeddings.weight": + # backbone input of a speech row, one table per codebook + yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), data_torch.reshape(cfg["num_codebooks"], -1, data_torch.shape[-1])) + return + + if name == "audio_decoder.embeddings.weight": + yield (self.format_tensor_name(T.A_GEN_CODE_TOK_EMBD), data_torch) + return + + if name == "audio_decoder.output.weight": + yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), data_torch[:_N_RESIDUAL_CODES]) + return + + if name == "audio_decoder.norm.weight": + yield (self.format_tensor_name(T.A_GEN_CODE_OUTPUT_NORM), data_torch) + return + + assert bid is not None and name.startswith(f"audio_decoder.layers.{bid}.") + key = name.removeprefix(f"audio_decoder.layers.{bid}.") + if key == "attention.wqkv.weight": + head_dim = cfg["head_dim"] + n_q = cfg["n_head"] * head_dim + n_kv = cfg["n_local_heads"] * head_dim + q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0) + yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_Q, bid), _neox_rows(q, head_dim)) + yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_K, bid), _neox_rows(k, head_dim)) + yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, bid), v) + return + yield (self.format_tensor_name(self._FAST_AR_MAP[key], bid), data_torch) + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + yield from super().generate_extra_tensors() + + sd = load_codec(self.dir_model) + T = gguf.MODEL_TENSOR + + def get(name: str) -> Tensor: + return sd[name].float() + + def tn(tensor: gguf.MODEL_TENSOR, bid: int | None = None, xid: int | None = None) -> str: + return gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid) + + def conv_wb(tensor: gguf.MODEL_TENSOR, prefix: str, bid: int | None = None, xid: int | None = None, stride: int = 1): + # taps-major [K / stride, OC, stride * IC], see conv() in tools/mtmd/models/fish-s2-spkenc.cpp + w = fold_conv(sd, prefix) + n_out, n_in, k = w.shape + w = w.reshape(n_out, n_in, k // stride, stride).permute(2, 0, 3, 1).reshape(k // stride, n_out, stride * n_in) + yield (tn(tensor, bid, xid) + ".weight", w) + yield (tn(tensor, bid, xid) + ".bias", get(prefix + ".conv.bias")) + + def snake_ab(tensor: gguf.MODEL_TENSOR, name: str, bid: int | None = None, xid: int | None = None): + a, b = fold_snake(get(name)) + yield (tn(tensor, bid, xid) + ".alpha", a) + yield (tn(tensor, bid, xid) + ".beta", b) + + def tfm_layers(prefix: str, n_layer: int, offset: int): + for il in range(n_layer): + p = f"{prefix}layers.{il}." + bid = offset + il + wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0) + yield (self.format_tensor_name(T.A_ENC_ATTN_Q, bid), _neox_rows(wq, _CODEC_HEAD_DIM)) + yield (self.format_tensor_name(T.A_ENC_ATTN_K, bid), _neox_rows(wk, _CODEC_HEAD_DIM)) + yield (self.format_tensor_name(T.A_ENC_ATTN_V, bid), wv) + yield (self.format_tensor_name(T.A_ENC_OUTPUT, bid), get(p + "attention.wo.weight")) + yield (self.format_tensor_name(T.A_ENC_INPUT_NORM, bid), get(p + "attention_norm.weight")) + yield (self.format_tensor_name(T.A_ENC_ATTN_SCALE, bid), get(p + "attention_layer_scale.gamma")) + yield (self.format_tensor_name(T.A_ENC_OUTPUT_NORM, bid), get(p + "ffn_norm.weight")) + yield (self.format_tensor_name(T.A_ENC_FFN_GATE, bid), get(p + "feed_forward.w1.weight")) + yield (self.format_tensor_name(T.A_ENC_FFN_UP, bid), get(p + "feed_forward.w3.weight")) + yield (self.format_tensor_name(T.A_ENC_FFN_DOWN, bid), get(p + "feed_forward.w2.weight")) + yield (self.format_tensor_name(T.A_ENC_FFN_SCALE_LS, bid), get(p + "ffn_layer_scale.gamma")) + + # encoder: entry conv, 4x (3 residual units, Snake, strided conv), Snake, post conv + e = "encoder.block." + yield from conv_wb(T.A_CODEC_ENC_ENTRY, e + "0") + for bid, stride in enumerate(_CODEC_ENC_STRIDES): + p = f"{e}{bid + 1}.block." + for xid in range(3): + r = f"{p}{xid}.block." + yield from snake_ab(T.A_CODEC_ENC_RES_ACT1, r + "0.alpha", bid, xid) + yield from conv_wb(T.A_CODEC_ENC_RES_CONV1, r + "1", bid, xid) + yield from snake_ab(T.A_CODEC_ENC_RES_ACT2, r + "2.alpha", bid, xid) + yield from conv_wb(T.A_CODEC_ENC_RES_CONV2, r + "3", bid, xid) + yield from snake_ab(T.A_CODEC_ENC_SNAKE, p + "3.alpha", bid) + yield from conv_wb(T.A_CODEC_ENC_CONV, p + "4", bid, stride=stride) + # the last block ends with a windowed transformer + yield from tfm_layers(f"{e}4.block.5.", _N_ENC_TFM_LAYERS, 0) + yield (self.format_tensor_name(T.A_POST_NORM), get(f"{e}4.block.5.norm.weight")) + yield from snake_ab(T.A_CODEC_ENC_POST_SNAKE, e + "5.alpha") + yield from conv_wb(T.A_CODEC_ENC_POST_CONV, e + "6") + + # quantizer: 2x (strided conv + ConvNeXt), windowed transformer, then semantic + residual VQ + q = "quantizer." + for bid in range(2): + p = f"{q}downsample.{bid}." + yield from conv_wb(T.A_CODEC_DOWN_CONV, p + "0", bid, stride=2) + # depthwise: one [C] row per tap + yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".weight", fold_conv(sd, p + "1.dwconv").squeeze(1).T.contiguous()) + yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".bias", get(p + "1.dwconv.conv.bias")) + for key, tensor in (("norm", T.A_CODEC_DOWN_NORM), ("pwconv1", T.A_CODEC_DOWN_PW1), ("pwconv2", T.A_CODEC_DOWN_PW2)): + yield (tn(tensor, bid) + ".weight", get(f"{p}1.{key}.weight")) + yield (tn(tensor, bid) + ".bias", get(f"{p}1.{key}.bias")) + yield (tn(T.A_CODEC_DOWN_GAMMA, bid), get(p + "1.gamma")) + yield from tfm_layers(q + "pre_module.", _N_PRE_TFM_LAYERS, _N_ENC_TFM_LAYERS) + yield (self.format_tensor_name(T.A_CODEC_PRE_NORM), get(q + "pre_module.norm.weight")) + + # nearest codebook entry by cosine similarity, see VectorQuantize.decode_latents() + def vq(prefix: str) -> tuple[Tensor, Tensor, Tensor]: + w = fold_wn(sd[prefix + ".in_proj.weight_g"], sd[prefix + ".in_proj.weight_v"]) + cb = torch.nn.functional.normalize(get(prefix + ".codebook.weight"), dim=-1) + return w, get(prefix + ".in_proj.bias"), cb + + w, b, cb = vq(q + "semantic_quantizer.quantizers.0") + yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".weight"), w) + yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".bias"), b) + yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_CB), cb) + n_rest = self.hparams["num_codebooks"] - 1 + rest = [vq(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)] + yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".weight"), torch.stack([r[0] for r in rest])) + yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".bias"), torch.stack([r[1] for r in rest])) + yield (self.format_tensor_name(T.A_CODEC_VQ_REST_CB), torch.stack([r[2] for r in rest])) diff --git a/docs/multimodal/tts.md b/docs/multimodal/tts.md index 1dca6c461..301d2394e 100644 --- a/docs/multimodal/tts.md +++ b/docs/multimodal/tts.md @@ -12,6 +12,7 @@ Supported models: - [Chatterbox](#chatterbox) - [Drama 3](#drama-3) - [ZONOS3](#zonos3) +- [Fish Audio S2 Pro](#fish-audio-s2-pro) The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)). @@ -215,3 +216,45 @@ Convert the root of the release repository, not the `lm/` subdirectory. The mmpr python convert_hf_to_gguf.py path/to/ZONOS3-Mini --outtype q8_0 --outfile zonos3-mini.gguf python convert_hf_to_gguf.py path/to/ZONOS3-Mini --mmproj --outtype f32 --outfile mmproj-zonos3-mini.gguf ``` + +## Fish Audio S2 Pro + +[Fish Audio S2 Pro](https://huggingface.co/fishaudio/s2-pro): a Qwen3 backbone (4B) samples each frame's semantic token, a 4-layer fast AR predicts the 9 other codes, and the Fish DAC (same as Drama 3) decodes them to 44.1 kHz audio. + +Available params: +- Speaker tags `<|speaker:0|>`, `<|speaker:1|>`, ... in the text give each character its own voice, a text without a tag starts with `<|speaker:0|>` +- Inline instructions in brackets control the delivery, for example `[laughing]`, `[whisper]` or `[pause]` +- The language is the one of the text, `--tts-lang` is not used +- Voice cloning: `--tts-speaker-file` with the reference clip, optionally `--tts-speaker-text` with its transcript (it is put in the prompt as the reference does). The clip is encoded by the codec encoder half of `codec.pth`, it is truncated to the first 30 s +- Note: `--temp`, `--top-k` and `--top-p` are not used (only `--seed`), frames are sampled with the settings of the reference (semantic token: repetition penalty 1.1 over the last 16 frames, top-k 30, top-p 0.8, temperature 0.8, or top-p 0.9 and temperature 1.0 when the last 4 frames repeat a token; other codes: greedy) +- Each frame is 2048 samples (about 46 ms), generation stops at the end-of-speech token + +Example usage: + +```sh +llama-tts -m s2-pro.gguf \ + -mm mmproj-s2-pro.gguf \ + -p "<|speaker:0|>Did you remember to buy the bread? <|speaker:1|>Of course I did [laughing] it is on the kitchen table." \ + -n 1000 \ + --output out.wav +``` + +Voice cloning (`--tts-speaker-text` can be left out): + +```sh +llama-tts -m s2-pro.gguf \ + -mm mmproj-s2-pro.gguf \ + --tts-speaker-file reference.wav \ + --tts-speaker-text "The words spoken in reference.wav." \ + -p "Hello, this is a cloned voice." \ + --output out.wav +``` + +**Note for GGUF conversion:** + +The mmproj holds the fast AR and `codec.pth` (the encoder half for voice cloning, the decoder half for the audio), it is best kept in BF16. The codec encoder is always stored in F16/F32, small errors in it change the codes picked by its VQ: + +```sh +python convert_hf_to_gguf.py path/to/s2-pro --outtype q8_0 --outfile s2-pro.gguf +python convert_hf_to_gguf.py path/to/s2-pro --mmproj --outtype bf16 --outfile mmproj-s2-pro.gguf +``` diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 037e1332a..e6c1465c2 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -1335,6 +1335,28 @@ class MODEL_TENSOR(IntEnum): A_GEN_CODE_EH_PROJ = auto() # drama3: per-layer [code embd; hidden] -> hidden projection A_GEN_CODE_BLK_EMBD = auto() # drama3: per-layer code embedding table A_GEN_CODE_BLK_OUT_NORM = auto() # drama3: per-layer output norm + A_GEN_CODE_TOK_EMBD = auto() # fish_s2: code predictor input embedding, shared by all codebooks + # fish_s2 codec encoder (voice cloning): reference clip -> codes + A_CODEC_ENC_ENTRY = auto() # entry conv + A_CODEC_ENC_SNAKE = auto() # encoder block, Snake before the strided conv + A_CODEC_ENC_CONV = auto() # encoder block, strided conv + A_CODEC_ENC_RES_ACT1 = auto() + A_CODEC_ENC_RES_CONV1 = auto() + A_CODEC_ENC_RES_ACT2 = auto() + A_CODEC_ENC_RES_CONV2 = auto() + A_CODEC_ENC_POST_SNAKE = auto() + A_CODEC_ENC_POST_CONV = auto() + A_CODEC_DOWN_CONV = auto() # quantizer downsample, strided conv + ConvNeXt + A_CODEC_DOWN_DWCONV = auto() + A_CODEC_DOWN_NORM = auto() + A_CODEC_DOWN_PW1 = auto() + A_CODEC_DOWN_PW2 = auto() + A_CODEC_DOWN_GAMMA = auto() + A_CODEC_PRE_NORM = auto() # output norm of the quantizer pre_module + A_CODEC_VQ_FIRST_IN = auto() # semantic VQ input projection + A_CODEC_VQ_FIRST_CB = auto() # semantic VQ codebook, l2-normalized + A_CODEC_VQ_REST_IN = auto() # residual VQ input projections, merged 3D + A_CODEC_VQ_REST_CB = auto() # residual VQ codebooks, l2-normalized, merged 3D # qwen3tts code2wav: RVQ codes -> raw PCM A_GEN_WAV_QUANT_FIRST_IN = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D) A_GEN_WAV_QUANT_FIRST_OUT = auto() # semantic RVQ, out_proj @@ -2277,6 +2299,27 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.A_GEN_CODE_EH_PROJ: "a.gen.code.blk.{bid}.eh_proj", MODEL_TENSOR.A_GEN_CODE_BLK_EMBD: "a.gen.code.blk.{bid}.embd", MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM: "a.gen.code.blk.{bid}.out_norm", + MODEL_TENSOR.A_GEN_CODE_TOK_EMBD: "a.gen.code.tok_embd", + MODEL_TENSOR.A_CODEC_ENC_ENTRY: "a.codec.enc.entry", + MODEL_TENSOR.A_CODEC_ENC_SNAKE: "a.codec.enc.blk.{bid}.snake", + MODEL_TENSOR.A_CODEC_ENC_CONV: "a.codec.enc.blk.{bid}.conv", + MODEL_TENSOR.A_CODEC_ENC_RES_ACT1: "a.codec.enc.blk.{bid}.res.{xid}.act1", + MODEL_TENSOR.A_CODEC_ENC_RES_CONV1: "a.codec.enc.blk.{bid}.res.{xid}.conv1", + MODEL_TENSOR.A_CODEC_ENC_RES_ACT2: "a.codec.enc.blk.{bid}.res.{xid}.act2", + MODEL_TENSOR.A_CODEC_ENC_RES_CONV2: "a.codec.enc.blk.{bid}.res.{xid}.conv2", + MODEL_TENSOR.A_CODEC_ENC_POST_SNAKE: "a.codec.enc.post_snake", + MODEL_TENSOR.A_CODEC_ENC_POST_CONV: "a.codec.enc.post_conv", + MODEL_TENSOR.A_CODEC_DOWN_CONV: "a.codec.down.blk.{bid}.conv", + MODEL_TENSOR.A_CODEC_DOWN_DWCONV: "a.codec.down.blk.{bid}.dwconv", + MODEL_TENSOR.A_CODEC_DOWN_NORM: "a.codec.down.blk.{bid}.norm", + MODEL_TENSOR.A_CODEC_DOWN_PW1: "a.codec.down.blk.{bid}.pw1", + MODEL_TENSOR.A_CODEC_DOWN_PW2: "a.codec.down.blk.{bid}.pw2", + MODEL_TENSOR.A_CODEC_DOWN_GAMMA: "a.codec.down.blk.{bid}.gamma", + MODEL_TENSOR.A_CODEC_PRE_NORM: "a.codec.pre_norm", + MODEL_TENSOR.A_CODEC_VQ_FIRST_IN: "a.codec.vq.first.in_proj", + MODEL_TENSOR.A_CODEC_VQ_FIRST_CB: "a.codec.vq.first.codebook", + MODEL_TENSOR.A_CODEC_VQ_REST_IN: "a.codec.vq.rest.in_proj", + MODEL_TENSOR.A_CODEC_VQ_REST_CB: "a.codec.vq.rest.codebook", MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN: "a.gen.wav.quant.first.in_proj", MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj", MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB: "a.gen.wav.quant.first.codebook", @@ -2711,6 +2754,27 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.A_GEN_CODE_EH_PROJ, MODEL_TENSOR.A_GEN_CODE_BLK_EMBD, MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM, + MODEL_TENSOR.A_GEN_CODE_TOK_EMBD, + MODEL_TENSOR.A_CODEC_ENC_ENTRY, + MODEL_TENSOR.A_CODEC_ENC_SNAKE, + MODEL_TENSOR.A_CODEC_ENC_CONV, + MODEL_TENSOR.A_CODEC_ENC_RES_ACT1, + MODEL_TENSOR.A_CODEC_ENC_RES_CONV1, + MODEL_TENSOR.A_CODEC_ENC_RES_ACT2, + MODEL_TENSOR.A_CODEC_ENC_RES_CONV2, + MODEL_TENSOR.A_CODEC_ENC_POST_SNAKE, + MODEL_TENSOR.A_CODEC_ENC_POST_CONV, + MODEL_TENSOR.A_CODEC_DOWN_CONV, + MODEL_TENSOR.A_CODEC_DOWN_DWCONV, + MODEL_TENSOR.A_CODEC_DOWN_NORM, + MODEL_TENSOR.A_CODEC_DOWN_PW1, + MODEL_TENSOR.A_CODEC_DOWN_PW2, + MODEL_TENSOR.A_CODEC_DOWN_GAMMA, + MODEL_TENSOR.A_CODEC_PRE_NORM, + MODEL_TENSOR.A_CODEC_VQ_FIRST_IN, + MODEL_TENSOR.A_CODEC_VQ_FIRST_CB, + MODEL_TENSOR.A_CODEC_VQ_REST_IN, + MODEL_TENSOR.A_CODEC_VQ_REST_CB, MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN, MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT, MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB, @@ -7223,6 +7287,8 @@ class VisionProjectorType: DRAMA3_GEN = "drama3_gen" # audio generation: codebook tower + Fish DAC decoder ZONOS3_GEN = "zonos3_gen" # audio generation: 10 codebook heads + ZAC decoder ZONOS3_SPKENC = "zonos3_spkenc" # audio: MECT-B2 speaker encoder + FISH_S2_GEN = "fish_s2_gen" # audio generation: fast AR code predictor + Fish DAC decoder + FISH_S2_SPKENC = "fish_s2_spkenc" # audio: Fish DAC encoder + RVQ, reference clip -> prompt rows HUNYUANVL = "hunyuanvl" PARAKEET = "parakeet" # audio MINIMAXM3 = "minimax_m3" diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index b23fd5e6b..d514a72cd 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -84,6 +84,8 @@ add_library(mtmd models/qwen3tts-spkenc.cpp models/qwen3tts-gen.cpp models/drama3-gen.cpp + models/fish-s2-gen.cpp + models/fish-s2-spkenc.cpp models/zonos3-gen.cpp models/zonos3-spkenc.cpp models/pockettts-seanet.cpp diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md index b6ed5a577..b0d153be3 100644 --- a/tools/mtmd/README-dev.md +++ b/tools/mtmd/README-dev.md @@ -39,6 +39,7 @@ Rules of thumb: - Models and backends must outlive the subcontexts, `mtmd_context` declares them first for that reason - Models having both audio and vision encoders must have different arch names (projector types), postfixed by `a` and `v`. See Gemma 4 as an example (`gemma4a` / `gemma4v`) - Locking happens only at the public API boundary (`mtmd.cpp`): a new public function that runs a graph must take `compute_mutex`. The internals (subcontexts, preprocessors, `mtmd_audio_log_mel()`) assume it is held and never lock +- No magic numbers for a model's fixed shapes (layer counts, codebook sizes, window lengths...): define them once as `static constexpr` in the graph struct in `models/models.h`, and refer to them from the loader or the subcontext (e.g. `mtmd_graph_zonos3_gen::n_codebooks`, `mtmd_graph_fish_s2_spkenc::n_tfm_layer`) ## Pipeline @@ -109,7 +110,9 @@ Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system i - 60% changes inside `mtmd-helper-gen.cpp` - 10% changes inside `libmtmd` core (`mtmd-model`, `mtmd-subcontext`, `models/`) - The rest downstream code (CLI, server) should have no changes at all -5. Update usage documentation in `docs/multimodal/tts.md` +5. Validate with several concurrent requests on `llama-server` (e.g. `-np 2`, one plain and one streaming request at the same time), not only with `llama-tts` + - The server interleaves the slots on one `llama_context`, so a pipeline must copy what it needs (hidden state, logits) right after its own decode, not read it at the next step +6. Update usage documentation in `docs/multimodal/tts.md` IMPORTANT: If your model needs changes that don't fit the existing infrastructure, **open an issue first for discussion**. diff --git a/tools/mtmd/models/fish-s2-gen.cpp b/tools/mtmd/models/fish-s2-gen.cpp new file mode 100644 index 000000000..4ac93f7bb --- /dev/null +++ b/tools/mtmd/models/fish-s2-gen.cpp @@ -0,0 +1,74 @@ +#include "models.h" + +#include +#include + +// one frame: backbone hidden + semantic code -> the codes of this frame + next backbone input +// fast AR row 0 is the backbone hidden, row p > 0 is the embedding of code p - 1 and picks code p (greedy, as the reference) +ggml_cgraph * mtmd_graph_fish_s2_gen::build_fast_ar() { + const int64_t n_cb = model.gen_code_embd_w->ne[2]; + const int64_t n_embd_cb = model.gen_code_embd_w->ne[0]; + const int64_t n_embd = model.gen_code_tok_embd_w->ne[0]; + const int64_t n_kv = model.layers[0].k_w->ne[1]; + const int n_layer = (int) model.layers.size(); + + ggml_tensor * h = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd); + ggml_set_name(h, "inp_raw"); + ggml_set_input(h); + + ggml_tensor * inp_code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); + ggml_set_name(inp_code0, "inp_code0"); + ggml_set_input(inp_code0); + + mtmd_graph_qwen3tts_gen::code_gen cg(*this, top_k, top_p); + + std::vector k_cache(n_layer), v_cache(n_layer); + for (int il = 0; il < n_layer; il++) { + k_cache[il] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, n_cb), 0.0f); + v_cache[il] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, n_cb), 0.0f); + } + + ggml_tensor * out_codes = cg.cache_set(ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, 1, n_cb), 0, inp_code0); + + for (int p = 0; p < n_cb; p++) { + ggml_tensor * cur = h; + if (p > 0) { + ggml_tensor * code = ggml_view_1d(ctx0, out_codes, 1, (size_t) (p - 1) * out_codes->nb[1]); + cur = ggml_reshape_1d(ctx0, ggml_get_rows(ctx0, model.gen_code_tok_embd_w, code), n_embd); + } + ggml_tensor * kq_mask = cg.causal_mask_row(n_cb, p); + ggml_tensor * inp_pos = cg.const_i32(k_cache[0], (float) p); + for (int il = 0; il < n_layer; il++) { + cur = cg.layer_forward(cur, model.layers[il], inp_pos, kq_mask, k_cache[il], v_cache[il], n_cb, p, il); + } + if (p == 0) { + continue; // only fills the KV + } + cur = ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, hparams.eps), model.gen_code_norm_w); + ggml_tensor * logits = ggml_mul_mat(ctx0, model.gen_code_head_w, cur); + out_codes = cg.cache_set(out_codes, p, ggml_argmax(ctx0, logits)); + } + + out_codes = ggml_cont(ctx0, out_codes); + ggml_set_name(out_codes, "out_codes"); + ggml_set_output(out_codes); + ggml_build_forward_expand(gf, out_codes); + + // backbone input of the next row: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1) + ggml_tensor * out_embd = ggml_get_rows(ctx0, model.gen_code_out_embd_w, inp_code0); + for (int64_t g = 0; g < n_cb; g++) { + ggml_tensor * table = ggml_view_2d(ctx0, model.gen_code_embd_w, n_embd_cb, model.gen_code_embd_w->ne[1], + model.gen_code_embd_w->nb[1], (size_t) g * model.gen_code_embd_w->nb[2]); + ggml_tensor * code = ggml_view_1d(ctx0, out_codes, 1, (size_t) g * out_codes->nb[1]); + out_embd = ggml_add(ctx0, out_embd, ggml_get_rows(ctx0, table, code)); + } + out_embd = ggml_scale(ctx0, out_embd, 1.0f / sqrtf((float) (n_cb + 1))); + cb(out_embd, "gen_audio_out", -1); + ggml_build_forward_expand(gf, out_embd); + + return gf; +} + +ggml_cgraph * mtmd_graph_fish_s2_gen::build() { + return gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV ? build_codec() : build_fast_ar(); +} diff --git a/tools/mtmd/models/fish-s2-spkenc.cpp b/tools/mtmd/models/fish-s2-spkenc.cpp new file mode 100644 index 000000000..a9c51d9cd --- /dev/null +++ b/tools/mtmd/models/fish-s2-spkenc.cpp @@ -0,0 +1,135 @@ +#include "models.h" + +#include +#include + +// reference clip -> codec frames -> backbone input rows, see DAC.encode() and DownsampleResidualVectorQuantize.encode() +// activations are [C, T]; a conv is a sum of matmuls over tap-shifted views: exact in F32, no im2col buffer + +// causal conv, w is taps-major [stride * IC, OC, K / stride], see conversion/fish_s2.py +ggml_tensor * mtmd_graph_fish_s2_spkenc::conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation) const { + const int64_t n_tap = w->ne[2]; + const int pad = (int) (n_tap - 1) * stride * dilation; + if (pad > 0) { + x = ggml_pad_ext(ctx0, x, 0, 0, pad, 0, 0, 0, 0, 0); + } + if (stride > 1) { + // stride frames per column, then each tap group is one matmul + x = ggml_reshape_2d(ctx0, x, x->ne[0] * stride, x->ne[1] / stride); + } + const int64_t n_out = x->ne[1] - (n_tap - 1) * dilation; + ggml_tensor * y = nullptr; + for (int64_t k = 0; k < n_tap; k++) { + ggml_tensor * wk = ggml_view_2d(ctx0, w, w->ne[0], w->ne[1], w->nb[1], (size_t) k * w->nb[2]); + ggml_tensor * xk = ggml_view_2d(ctx0, x, x->ne[0], n_out, x->nb[1], (size_t) k * dilation * x->nb[1]); + ggml_tensor * yk = ggml_mul_mat(ctx0, wk, xk); + y = y ? ggml_add(ctx0, y, yk) : yk; + } + return ggml_add(ctx0, y, b); +} + +// x + sin(alpha * x)^2 / alpha, beta = 1 / alpha is folded at conversion +// TODO @ngxson: on Metal, residual codes match the reference less often than on CPU (89-98% vs 99%), maybe the fast-math sin +ggml_tensor * mtmd_graph_fish_s2_spkenc::snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const { + ggml_tensor * s = ggml_sqr(ctx0, ggml_sin(ctx0, ggml_mul(ctx0, x, alpha))); + return ggml_add(ctx0, x, ggml_mul(ctx0, s, beta)); +} + +// causal depthwise conv (w [C, K]) -> LayerNorm -> pwconv1 -> GELU -> pwconv2 -> layer scale -> residual +ggml_tensor * mtmd_graph_fish_s2_spkenc::convnext(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk) const { + const int64_t n_tap = blk.dwconv_w->ne[1]; + const int64_t n_pos = x->ne[1]; + ggml_tensor * xp = ggml_pad_ext(ctx0, x, 0, 0, (int) n_tap - 1, 0, 0, 0, 0, 0); + ggml_tensor * h = nullptr; + for (int64_t k = 0; k < n_tap; k++) { + ggml_tensor * wk = ggml_view_1d(ctx0, blk.dwconv_w, blk.dwconv_w->ne[0], (size_t) k * blk.dwconv_w->nb[1]); + ggml_tensor * xk = ggml_view_2d(ctx0, xp, xp->ne[0], n_pos, xp->nb[1], (size_t) k * xp->nb[1]); + ggml_tensor * hk = ggml_mul(ctx0, xk, wk); + h = h ? ggml_add(ctx0, h, hk) : hk; + } + h = ggml_add(ctx0, h, blk.dwconv_b); + h = ggml_add(ctx0, ggml_mul(ctx0, ggml_norm(ctx0, h, 1e-6f), blk.norm_w), blk.norm_b); + h = ggml_gelu_erf(ctx0, ggml_add(ctx0, ggml_mul_mat(ctx0, blk.pw1_w, h), blk.pw1_b)); + h = ggml_add(ctx0, ggml_mul_mat(ctx0, blk.pw2_w, h), blk.pw2_b); + return ggml_add(ctx0, x, ggml_mul(ctx0, h, blk.gamma)); +} + +ggml_cgraph * mtmd_graph_fish_s2_spkenc::build() { + const auto & c2w = model.c2w; + + // the preprocessor hands over the waveform as a single-row "mel", [n_samples, 1] is also [1, n_samples] + ggml_tensor * inp_raw = build_inp_raw(1); + ggml_tensor * x = ggml_reshape_2d(ctx0, inp_raw, 1, inp_raw->ne[0]); + + // windowed causal transformer on a cold start: zero state, positions from 0 + mtmd_graph_qwen3tts_gen::code2wav dec(*this); + auto zero_state = [&](const std::string & name, int64_t n0, int64_t n1) { + dec.state_in[name] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n0, n1), 0.0f); + }; + auto transformer = [&](ggml_tensor * cur, int il0, int il1, int64_t window, ggml_tensor * norm_w) { + const int64_t n_kv = model.layers[il0].k_w->ne[1]; + zero_state("tfm_pos", 1, 1); + for (int il = il0; il < il1; il++) { + zero_state("tfm_k_" + std::to_string(il), n_kv, window - 1); + zero_state("tfm_v_" + std::to_string(il), n_kv, window - 1); + cur = dec.tfm_layer_forward(cur, model.layers[il], il, window); + } + return ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, hparams.wav_tfm_eps), norm_w); + }; + + // encoder: entry conv, 4x (3 residual units, Snake, strided conv), transformer, Snake, post conv + x = conv(x, c2w.dac_entry_w, c2w.dac_entry_b, 1, 1); + static constexpr int DAC_DILATIONS[3] = { 1, 3, 9 }; + for (const auto & blk : c2w.dac) { + for (size_t ir = 0; ir < blk.res.size(); ir++) { + const auto & res = blk.res[ir]; + ggml_tensor * h = snake(x, res.act1_alpha, res.act1_beta); + h = conv(h, res.conv1_w, res.conv1_b, 1, DAC_DILATIONS[ir]); + h = snake(h, res.act2_alpha, res.act2_beta); + h = conv(h, res.conv2_w, res.conv2_b, 1, 1); + x = ggml_add(ctx0, x, h); + } + x = snake(x, blk.snake_alpha, blk.snake_beta); + x = conv(x, blk.conv_w, blk.conv_b, (int) (blk.conv_w->ne[0] / x->ne[0]), 1); + } + x = transformer(x, 0, n_enc_tfm_layer, 512, model.post_ln_w); + x = snake(x, c2w.dac_post_snake_alpha, c2w.dac_post_snake_beta); + x = conv(x, c2w.dac_post_conv_w, c2w.dac_post_conv_b, 1, 1); + cb(x, "codec_enc", -1); + + // quantizer: 2x (strided conv + ConvNeXt), transformer + for (const auto & down : c2w.upsample) { + x = conv(x, down.conv_w, down.conv_b, (int) (down.conv_w->ne[0] / x->ne[0]), 1); + x = convnext(x, down); + } + ggml_tensor * z = transformer(x, n_enc_tfm_layer, (int) model.layers.size(), 128, c2w.tfm_output_norm_w); + cb(z, "codec_latent", -1); + + // nearest entry by cosine similarity, then the entry through out_proj is removed from the residual + auto vq = [&](ggml_tensor * in_w, ggml_tensor * in_b, ggml_tensor * cb_norm, ggml_tensor * table) { + ggml_tensor * e = ggml_add(ctx0, ggml_mul_mat(ctx0, in_w, z), in_b); + e = ggml_l2_norm(ctx0, e, 1e-12f); + ggml_tensor * idx = ggml_argmax(ctx0, ggml_mul_mat(ctx0, cb_norm, e)); + z = ggml_sub(ctx0, z, ggml_get_rows(ctx0, table, idx)); + return idx; + }; + auto page = [&](ggml_tensor * t, int64_t g) { + return ggml_view_2d(ctx0, t, t->ne[0], t->ne[1], t->nb[1], (size_t) g * t->nb[2]); + }; + + // backbone row: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1), same as fish_s2_gen + const int64_t n_cb = model.gen_code_embd_w->ne[2]; + ggml_tensor * idx = vq(c2w.quant_first_in_w, c2w.quant_first_in_b, c2w.quant_first_cb_norm_w, c2w.quant_first_cb_w); + ggml_tensor * out = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, idx), + ggml_get_rows(ctx0, page(model.gen_code_embd_w, 0), idx)); + for (int64_t g = 1; g < n_cb; g++) { + ggml_tensor * in_b = ggml_view_1d(ctx0, c2w.quant_rest_in_b, c2w.quant_rest_in_b->ne[0], (size_t) (g - 1) * c2w.quant_rest_in_b->nb[1]); + idx = vq(page(c2w.quant_rest_in_w, g - 1), in_b, page(c2w.quant_rest_cb_norm_w, g - 1), page(c2w.quant_rest_cb_w, g - 1)); + out = ggml_add(ctx0, out, ggml_get_rows(ctx0, page(model.gen_code_embd_w, g), idx)); + } + out = ggml_scale(ctx0, out, 1.0f / sqrtf((float) (n_cb + 1))); + cb(out, "spk_rows", -1); + ggml_build_forward_expand(gf, out); + + return gf; +} diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index a8e6120da..d2de0273a 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -462,7 +462,8 @@ struct mtmd_graph_qwen3tts_gen : mtmd_graph { ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const; ggml_tensor * quant_decode(ggml_tensor * inp_codes) const; - ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il) const; + // window <= 0 uses hparams.wav_tfm_swa + ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il, int64_t window = 0) const; ggml_tensor * convnext_block(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk, const std::string & state_prefix) const; ggml_tensor * dac_res_unit(ggml_tensor * x, const mtmd_code2wav::dac_res & res, int dilation, const std::string & state_name) const; @@ -569,6 +570,32 @@ struct mtmd_graph_drama3_gen : mtmd_graph { ggml_tensor * inp_pos, ggml_tensor * k_slot, ggml_tensor * v_slot, int pos); }; +// +// fish_s2 generation: +// GEN_CODE = fast AR, backbone hidden + semantic code -> other codes of the frame + next backbone input +// GEN_WAV = Fish DAC decoder, same as drama3 +// +struct mtmd_graph_fish_s2_gen : mtmd_graph_drama3_gen { + using mtmd_graph_drama3_gen::mtmd_graph_drama3_gen; + ggml_cgraph * build() override; + + ggml_cgraph * build_fast_ar(); +}; + +// fish_s2 voice cloning: raw 44.1 kHz clip -> codec encoder + RVQ -> one backbone input row per frame +struct mtmd_graph_fish_s2_spkenc : mtmd_graph { + mtmd_graph_fish_s2_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} + ggml_cgraph * build() override; + + // codec encoder transformer layers, the rest of model.layers is the quantizer pre_module + static constexpr int n_enc_tfm_layer = 4; + static constexpr int n_tfm_layer = 12; + + ggml_tensor * conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation) const; + ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const; + ggml_tensor * convnext(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk) const; +}; + // // zonos3 generation: // GEN_CODE = 10 codebook heads, or the backbone row of a sampled frame diff --git a/tools/mtmd/models/qwen3tts-gen.cpp b/tools/mtmd/models/qwen3tts-gen.cpp index 59958fc4c..91d5a5c12 100644 --- a/tools/mtmd/models/qwen3tts-gen.cpp +++ b/tools/mtmd/models/qwen3tts-gen.cpp @@ -156,10 +156,13 @@ ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::layer_forward( q = ggml_reshape_3d(ctx0, q, d_head, n_head, 1); k = ggml_reshape_3d(ctx0, k, d_head, n_head_kv, 1); - q = ggml_rms_norm(ctx0, q, hparams.eps); - q = ggml_mul(ctx0, q, layer.q_norm); - k = ggml_rms_norm(ctx0, k, hparams.eps); - k = ggml_mul(ctx0, k, layer.k_norm); + // fish_s2 fast AR has no qk norm + if (layer.q_norm) { + q = ggml_rms_norm(ctx0, q, hparams.eps); + q = ggml_mul(ctx0, q, layer.q_norm); + k = ggml_rms_norm(ctx0, k, hparams.eps); + k = ggml_mul(ctx0, k, layer.k_norm); + } q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, (int) d_head, GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); @@ -418,12 +421,12 @@ ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_ // one pre_transformer layer over a batch of N = sliding_window new frames // attention runs over [(W-1)-frame prefix from the last batch] + [N new frames] // RoPE positions come from a persisted counter, so phases line up across batches -ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il, int64_t window) const { const int n_head = hparams.wav_tfm_n_head; const int n_head_kv = hparams.wav_tfm_n_head_kv; const int64_t d_head = layer.q_w->ne[1] / n_head; const float kq_scale = 1.0f / sqrtf((float) d_head); - const int64_t W = hparams.wav_tfm_swa; // == N, frames per batch + const int64_t W = window > 0 ? window : hparams.wav_tfm_swa; // code2wav: == N, frames per batch const int64_t N = cur->ne[1]; const int64_t prefix = W - 1; const int64_t total_kv = prefix + N; diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index dfd3b30e7..419bbede1 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -2242,6 +2242,16 @@ bool mtmd_audio_preprocessor_pockettts::process(const float * sa return true; } +bool mtmd_audio_preprocessor_fish_s2::process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const { + const size_t max_samples = (size_t) mtmd_hparams::fish_s2_max_spk_seconds * hparams.audio_sample_rate; + if (n_samples > max_samples) { + LOG_WRN("%s: speaker reference is %.1f s, truncating to the first %d s\n", __func__, + (double) n_samples / hparams.audio_sample_rate, mtmd_hparams::fish_s2_max_spk_seconds); + n_samples = max_samples; + } + return mtmd_audio_preprocessor_muse_spark_audio::process(samples, n_samples, output, shape_only); +} + // // mtmd_audio_preprocessor_vibevoice_asr // diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index b4b1ce692..893739d93 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -129,6 +129,12 @@ struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor { bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; }; +// fish_s2 speaker reference: same as muse spark (raw waveform, whole frames), truncated to fish_s2_max_spk_seconds +struct mtmd_audio_preprocessor_fish_s2 : mtmd_audio_preprocessor_muse_spark_audio { + using mtmd_audio_preprocessor_muse_spark_audio::mtmd_audio_preprocessor_muse_spark_audio; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; +}; + // vibevoice: gain to -25 dBFS RMS, without clipping the peak // the RMS is floored at -50 dBFS, so that silence or noise alone is not boosted to full scale double mtmd_audio_vibevoice_gain(double sum_sq, int64_t n_samples, double peak); diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp index 100bc3c58..cb2029685 100644 --- a/tools/mtmd/mtmd-graph.cpp +++ b/tools/mtmd/mtmd-graph.cpp @@ -973,6 +973,10 @@ std::unique_ptr mtmd_get_graph_builder(mtmd_subcontext * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_FISH_S2_SPKENC: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_POCKETTTS_GEN: { const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; @@ -1035,13 +1039,19 @@ std::unique_ptr mtmd_get_graph_builder(mtmd_subcontext * ctx, const builder = std::make_unique(ctx, img, gen_process, all_known, n_frames); } break; case PROJECTOR_TYPE_DRAMA3_GEN: + case PROJECTOR_TYPE_FISH_S2_GEN: { const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; const int64_t n_cb = ctx->model.gen_code_embd_w->ne[2]; const int n_frames = params && params->codes ? (int) (params->codes->size() / n_cb) : 1; - builder = std::make_unique(ctx, img, gen_process, params ? params->gen_state : nullptr, n_frames, - params ? params->top_k : 30, params ? params->top_p : 0.9f, - params ? params->temp : 1.0f); + const int top_k = params ? params->top_k : 30; + const float top_p = params ? params->top_p : 0.9f; + const float temp = params ? params->temp : 1.0f; + if (ctx->model.proj_type == PROJECTOR_TYPE_FISH_S2_GEN) { + builder = std::make_unique(ctx, img, gen_process, params ? params->gen_state : nullptr, n_frames, top_k, top_p, temp); + } else { + builder = std::make_unique(ctx, img, gen_process, params ? params->gen_state : nullptr, n_frames, top_k, top_p, temp); + } } break; case PROJECTOR_TYPE_QWEN3TTS_GEN: { diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index 2148ec526..79541b813 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -2088,6 +2088,7 @@ public: prompt_tokens.clear(); prompt_pos = 0; frame.clear(); + main_logits.clear(); hist.clear(); codes_buf.clear(); n_decoded = 0; @@ -2110,31 +2111,16 @@ public: sem_begin = find_special_token(vocab, "<|semantic:0|>"); eos = find_special_token(vocab, "<|im_end|>"); - if (sem_begin == LLAMA_TOKEN_NULL || eos == LLAMA_TOKEN_NULL) { - LOG_ERR("mtmd_helper_gen_audio: missing Drama 3 tokens in vocab\n"); + if (sem_begin == LLAMA_TOKEN_NULL || eos == LLAMA_TOKEN_NULL || + sem_begin + n_cb_vocab > llama_vocab_n_tokens(vocab)) { + LOG_ERR("mtmd_helper_gen_audio: missing semantic tokens in vocab\n"); return 1; } - // reference-bracket-v1 prompt without references, tokenize each segment alone as the reference does - std::string text(inp->prompt, inp->prompt_len); - if (text.rfind("<|speaker:", 0) != 0) { - text = "<|speaker:0|> " + text; + if (!tokenize_segments(prompt_segments(std::string(inp->prompt, inp->prompt_len)), prompt_tokens)) { + return 1; } - const char * segments[] = { - "<|im_start|>system\n", "convert the provided text to speech", "<|im_end|>\n", - "<|im_start|>user\n", text.c_str(), "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>", - }; - for (const char * seg : segments) { - const int32_t len = (int32_t) strlen(seg); - std::vector ids(len + 16); - const int32_t n = llama_tokenize(vocab, seg, len, ids.data(), (int32_t) ids.size(), false, true); - if (n < 0) { - LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n"); - return 1; - } - prompt_tokens.insert(prompt_tokens.end(), ids.begin(), ids.begin() + n); - } - if (prompt_tokens.size() >= llama_n_ctx_seq(lctx) || prompt_tokens.size() > n_tower_rows) { + if (prompt_tokens.size() >= llama_n_ctx_seq(lctx) || prompt_tokens.size() > max_rows()) { LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n"); return 1; } @@ -2170,6 +2156,7 @@ public: const float * h = llama_get_embeddings_ith(lctx, i); if (prompt_pos + i == n_prompt - 1) { h_state_buf.assign(h, h + n_embd); // first frame comes from the last prompt row + save_main_logits(i); break; } if (!run_tower(-1, h)) { @@ -2188,9 +2175,9 @@ public: *h_state_out = nullptr; *out_stop = false; - const llama_token main = sample_main(llama_get_logits_ith(lctx, -1)); + const llama_token main = sample_main(main_logits.data()); const size_t n_rows = prompt_tokens.size() - 1 + codes_buf.size() / n_cb; - if (main == eos || (uint32_t) pos >= llama_n_ctx_seq(lctx) || n_rows >= n_tower_rows) { + if (main == eos || (uint32_t) pos >= llama_n_ctx_seq(lctx) || n_rows >= max_rows()) { *out_stop = true; return 0; } @@ -2222,6 +2209,7 @@ public: pos++; const float * h = llama_get_embeddings_ith(lctx, -1); h_state_buf.assign(h, h + n_embd); + save_main_logits(-1); *h_state_out = h_state_buf.data(); if (stream && codes_buf.size() / n_cb - n_decoded >= stream_frames) { @@ -2282,7 +2270,44 @@ public: return decode_wav() ? 0 : 1; } -private: +protected: + // tokenize each segment alone as the reference does + bool tokenize_segments(const std::vector & segments, std::vector & out) const { + for (const std::string & seg : segments) { + std::vector ids(seg.size() + 16); + const int32_t n = llama_tokenize(vocab, seg.c_str(), (int32_t) seg.size(), ids.data(), (int32_t) ids.size(), false, true); + if (n < 0) { + LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n"); + return false; + } + out.insert(out.end(), ids.begin(), ids.begin() + n); + } + return true; + } + + // reference-bracket-v1 prompt without references + virtual std::vector prompt_segments(std::string text) const { + if (text.rfind("<|speaker:", 0) != 0) { + text = "<|speaker:0|> " + text; + } + return { + "<|im_start|>system\n", "convert the provided text to speech", "<|im_end|>\n", + "<|im_start|>user\n", text, "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>", + }; + } + + // prompt rows but the last + generated frames + virtual size_t max_rows() const { return n_tower_rows; } + + // copy right after our own decode, other sequences may decode on lctx before the next step_gen + // layout: eos, then semantic codes 0..n_cb_vocab-1 + void save_main_logits(int32_t i) { + const float * logits = llama_get_logits_ith(lctx, i); + main_logits.resize(n_cb_vocab + 1); + main_logits[0] = logits[eos]; + std::copy(logits + sem_begin, logits + sem_begin + n_cb_vocab, main_logits.begin() + 1); + } + // repetition-aware sampling: if a semantic draw repeats one of the last ras_window main tokens, draw again with fixed settings static constexpr size_t ras_window = 16; static constexpr size_t n_cb = 10; @@ -2332,11 +2357,12 @@ private: } // candidates: eos, then semantic codes 1.. (the reference never samples code 0) - llama_token sample_main(const float * logits) { + // logits as saved by save_main_logits() + virtual llama_token sample_main(const float * logits) { std::vector cand(n_cb_vocab); - cand[0] = logits[eos]; + cand[0] = logits[0]; for (int i = 1; i < n_cb_vocab; i++) { - cand[i] = logits[sem_begin + i]; + cand[i] = logits[1 + i]; } auto to_token = [&](int i) { return i == 0 ? eos : sem_begin + i; }; const llama_token first = to_token(draw(cand, 1.0f, 0, 0.9f)); @@ -2397,6 +2423,7 @@ private: int prompt_pos = 0; std::mt19937 rng; + std::vector main_logits; // see save_main_logits() std::vector frame; // codes of the last frame, input of the next tower row std::vector hist; // last main tokens, for repetition-aware sampling std::vector codes_buf; // frame-major @@ -2411,6 +2438,195 @@ private: bool wav_header_sent = false; }; +// Fish Audio S2 Pro: same tokens, main-token loop and Fish DAC as drama3 +// the fast AR only runs on generated frames, so the prompt is a plain decode +class fish_s2_gen_audio_pipeline : public drama3_gen_audio_pipeline { +public: + using drama3_gen_audio_pipeline::drama3_gen_audio_pipeline; + + void reset() override { + drama3_gen_audio_pipeline::reset(); + parts.clear(); + part_idx = 0; + part_pos = 0; + } + + // voice cloning (sglang-omni fishaudio_s2_pro): a system message with the transcript and the codec rows of the clip + int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { + std::vector ref_rows; + if (inp->speaker_ref && !encode_speaker(inp->speaker_ref, ref_rows)) { + return 1; + } + mtmd_helper_gen_audio_inp text_inp = *inp; + text_inp.speaker_ref = nullptr; + if (drama3_gen_audio_pipeline::set_input(&text_inp) != 0) { + return 1; + } + + size_t n_total = prompt_tokens.size(); + if (!ref_rows.empty()) { + // optional, the reference leaves the text empty when it has none + std::string ref_text = inp->speaker_ref_text ? inp->speaker_ref_text : ""; + if (!ref_text.empty() && ref_text.rfind("<|speaker:", 0) != 0) { + ref_text = "<|speaker:0|>" + ref_text; + } + prompt_part head, rows, tail; + if (!tokenize_segments({ "<|im_start|>system\n", "convert the provided text to speech reference to the following:\n\nText:\n", + ref_text, "\n\nSpeech:\n" }, head.tokens) || + !tokenize_segments({ "<|im_end|>\n" }, tail.tokens)) { + return 1; + } + rows.rows = std::move(ref_rows); + for (auto * part : { &head, &rows, &tail }) { + n_total += part->size(n_embd); + parts.push_back(std::move(*part)); + } + } + if (n_total >= llama_n_ctx_seq(lctx)) { + LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n"); + return 1; + } + prompt_part text; + text.tokens = prompt_tokens; + parts.push_back(std::move(text)); + return 0; + } + + // the first frame comes from the last prompt row + int32_t step_prompt(int32_t n_batch) override { + GGML_ASSERT(n_batch > 0); + if (part_idx >= parts.size()) { + return 0; + } + const prompt_part & part = parts[part_idx]; + const int n_part = part.size(n_embd); + const int n = std::min(n_batch, n_part - part_pos); + const bool last = part_idx + 1 == parts.size() && part_pos + n == n_part; + + int ret = 0; + if (part.rows.empty()) { + llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); + if (!batch_set_tokens(batch.get(), part.tokens.data() + part_pos, n, pos, seq_id)) { + return -1; + } + if (last) { + llama_batch_ext_set_output_logits(batch.get(), n - 1, true); + } + ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()); + } else { + decode_embd_batch batch(part.rows.data() + (size_t) part_pos * n_embd, n, 1, n_embd); + batch.set_position_normal(pos, seq_id); + ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n)); + } + if (ret != 0) { + LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); + return -1; + } + if (last) { + const float * h = llama_get_embeddings_ith(lctx, -1); + h_state_buf.assign(h, h + n_embd); + save_main_logits(-1); + } + pos += n; + part_pos += n; + if (part_pos == n_part) { + part_idx++; + part_pos = 0; + } + int32_t n_left = -part_pos; + for (size_t i = part_idx; i < parts.size(); i++) { + n_left += parts[i].size(n_embd); + } + return n_left; + } + +protected: + // reference (sglang-omni fishaudio_s2_pro) has no system message + std::vector prompt_segments(std::string text) const override { + if (text.rfind("<|speaker:", 0) != 0) { + text = "<|speaker:0|>" + text; + } + return { + "<|im_start|>user\n", text, "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>", + }; + } + + // no codebook tower, only the context limits the length + size_t max_rows() const override { return SIZE_MAX; } + + // token ids, or backbone input rows of n_embd floats + struct prompt_part { + std::vector tokens; + std::vector rows; + int size(int n_embd) const { return rows.empty() ? (int) tokens.size() : (int) (rows.size() / n_embd); } + }; + std::vector parts; + size_t part_idx = 0; + int part_pos = 0; + + // same as the reference: repetition penalty over the last ras_window main tokens, top-k, top-p on the + // untempered logits (the crossing token is dropped), then temperature + // a repeat within the last 4 main tokens switches to the RAS settings + llama_token sample_main(const float * logits) override { + static constexpr int32_t top_k = 30; + static constexpr float rep_penalty = 1.1f; + + std::vector> cand; + cand.reserve(n_cb_vocab + 1); + cand.push_back({ logits[0], eos }); + for (int i = 0; i < n_cb_vocab; i++) { + cand.push_back({ logits[1 + i], sem_begin + i }); + } + for (auto & c : cand) { + if (std::find(hist.begin(), hist.end(), c.second) != hist.end()) { + c.first = c.first < 0.0f ? c.first * rep_penalty : c.first / rep_penalty; + } + } + + bool ras = false; + if (hist.size() >= 4) { + std::vector last4(hist.end() - 4, hist.end()); + std::sort(last4.begin(), last4.end()); + ras = std::adjacent_find(last4.begin(), last4.end()) != last4.end(); + } + const float temp = ras ? 1.0f : 0.8f; + const float top_p = ras ? 0.9f : 0.8f; + + const size_t k = std::min(top_k, cand.size()); + std::partial_sort(cand.begin(), cand.begin() + k, cand.end(), [](const auto & a, const auto & b) { return a.first > b.first; }); + + std::vector prob(k); + double sum = 0.0; + for (size_t i = 0; i < k; i++) { + prob[i] = std::exp((double) (cand[i].first - cand[0].first)); + sum += prob[i]; + } + size_t n_keep = k; + double cum = 0.0; + for (size_t i = 0; i < k; i++) { + cum += prob[i]; + if (i > 0 && cum > top_p * sum) { + n_keep = i; + break; + } + } + + double kept = 0.0; + for (size_t i = 0; i < n_keep; i++) { + prob[i] = std::exp((double) (cand[i].first - cand[0].first) / temp); + kept += prob[i]; + } + double r = std::uniform_real_distribution(0.0, 1.0)(rng) * kept; + for (size_t i = 0; i < n_keep; i++) { + r -= prob[i]; + if (r <= 0.0) { + return cand[i].second; + } + } + return cand[n_keep - 1].second; + } +}; + // ZONOS3: one backbone row per frame of 10 codebooks, codebook j lags j frames class zonos3_gen_audio_pipeline : public mtmd_gen_audio_pipeline { public: @@ -2953,17 +3169,21 @@ static std::unique_ptr make_pipeline(llama_context * lc return std::unique_ptr(new drama3_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_ZONOS3: return std::unique_ptr(new zonos3_gen_audio_pipeline(lctx, mctx)); + case MTMD_GEN_AUDIO_TYPE_FISH_S2: + return std::unique_ptr(new fish_s2_gen_audio_pipeline(lctx, mctx)); default: return nullptr; } } struct mtmd_helper_gen_audio { + llama_context * lctx = nullptr; std::unique_ptr pipeline; }; mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(struct llama_context * lctx, struct mtmd_context * mctx) { auto * ctx = new mtmd_helper_gen_audio(); + ctx->lctx = lctx; ctx->pipeline = make_pipeline(lctx, mctx); return ctx; } @@ -3015,6 +3235,11 @@ int32_t mtmd_helper_gen_audio_step_prompt(mtmd_helper_gen_audio * ctx, int32_t n if (!ctx->pipeline) { return -1; } + // the context may use a smaller n_batch than requested (e.g. clamped to n_ctx), llama_process() would fail above it + if (n_batch <= 0 || (uint32_t) n_batch > llama_n_batch(ctx->lctx)) { + LOG_ERR("mtmd_helper_gen_audio: n_batch must be in [1, %u], got %d\n", llama_n_batch(ctx->lctx), n_batch); + return -1; + } return ctx->pipeline->step_prompt(n_batch); } diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index b33ed65ec..82e86bd1f 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -310,6 +310,7 @@ struct mtmd_helper_gen_audio_inp { enum mtmd_helper_gen_audio_out_mode out_mode; // n_past > 0 only (default: auto) mtmd_bitmap * speaker_ref; // optional, can be NULL: audio to clone, or a voice bitmap + const char * speaker_ref_text; // optional, can be NULL: transcript of speaker_ref, for models that put it in the prompt (e.g. Fish S2) const char * lang; // optional, can be NULL int32_t top_k; @@ -339,7 +340,7 @@ MTMD_API int32_t mtmd_helper_gen_audio_set_input( mtmd_helper_gen_audio * ctx, const struct mtmd_helper_gen_audio_inp * inp); -// processes at most n_batch prompt tokens per call +// processes at most n_batch prompt tokens per call, n_batch must not exceed llama_n_batch() of the context // returns: >0 = number of prompt tokens remaining, 0 = done, <0 = error MTMD_API int32_t mtmd_helper_gen_audio_step_prompt( mtmd_helper_gen_audio * ctx, @@ -423,6 +424,7 @@ struct gen_audio { mtmd_helper_gen_audio_inp data = mtmd_helper_gen_audio_inp_default(); std::string prompt_str; std::string lang_str; + std::string speaker_ref_text_str; mtmd::bitmap_ptr speaker_ref_ptr; inp() = default; @@ -434,6 +436,7 @@ struct gen_audio { void set_prompt (std::string p) { prompt_str = std::move(p); } void set_lang (std::string l) { lang_str = std::move(l); } void set_speaker_ref(mtmd::bitmap_ptr bmp) { speaker_ref_ptr = std::move(bmp); } + void set_speaker_ref_text(std::string t) { speaker_ref_text_str = std::move(t); } // pointers are only valid as long as *this is alive const mtmd_helper_gen_audio_inp * get() { @@ -441,6 +444,7 @@ struct gen_audio { data.prompt_len = prompt_str.size(); data.lang = lang_str.empty() ? nullptr : lang_str.c_str(); data.speaker_ref = speaker_ref_ptr.get(); + data.speaker_ref_text = speaker_ref_text_str.empty() ? nullptr : speaker_ref_text_str.c_str(); return &data; } }; diff --git a/tools/mtmd/mtmd-impl.h b/tools/mtmd/mtmd-impl.h index 70db37654..ff2b70230 100644 --- a/tools/mtmd/mtmd-impl.h +++ b/tools/mtmd/mtmd-impl.h @@ -271,6 +271,31 @@ #define TN_A_GEN_CODE_BLK_EMBD "a.gen.code.blk.%d.embd.%s" #define TN_A_GEN_CODE_BLK_OUT_NORM "a.gen.code.blk.%d.out_norm.%s" +// fish_s2: fast AR input embedding, shared by all codebooks +#define TN_A_GEN_CODE_TOK_EMBD "a.gen.code.tok_embd.%s" + +// fish_s2 codec encoder, the transformer layers use the generic a.blk.* names +#define TN_A_CODEC_ENC_ENTRY "a.codec.enc.entry.%s" +#define TN_A_CODEC_ENC_SNAKE "a.codec.enc.blk.%d.snake.%s" +#define TN_A_CODEC_ENC_CONV "a.codec.enc.blk.%d.conv.%s" +#define TN_A_CODEC_ENC_RES_ACT1 "a.codec.enc.blk.%d.res.%d.act1.%s" +#define TN_A_CODEC_ENC_RES_CONV1 "a.codec.enc.blk.%d.res.%d.conv1.%s" +#define TN_A_CODEC_ENC_RES_ACT2 "a.codec.enc.blk.%d.res.%d.act2.%s" +#define TN_A_CODEC_ENC_RES_CONV2 "a.codec.enc.blk.%d.res.%d.conv2.%s" +#define TN_A_CODEC_ENC_POST_SNAKE "a.codec.enc.post_snake.%s" +#define TN_A_CODEC_ENC_POST_CONV "a.codec.enc.post_conv.%s" +#define TN_A_CODEC_DOWN_CONV "a.codec.down.blk.%d.conv.%s" +#define TN_A_CODEC_DOWN_DWCONV "a.codec.down.blk.%d.dwconv.%s" +#define TN_A_CODEC_DOWN_NORM "a.codec.down.blk.%d.norm.%s" +#define TN_A_CODEC_DOWN_PW1 "a.codec.down.blk.%d.pw1.%s" +#define TN_A_CODEC_DOWN_PW2 "a.codec.down.blk.%d.pw2.%s" +#define TN_A_CODEC_DOWN_GAMMA "a.codec.down.blk.%d.gamma" +#define TN_A_CODEC_PRE_NORM "a.codec.pre_norm.%s" +#define TN_A_CODEC_VQ_FIRST_IN "a.codec.vq.first.in_proj.%s" +#define TN_A_CODEC_VQ_FIRST_CB "a.codec.vq.first.codebook.%s" +#define TN_A_CODEC_VQ_REST_IN "a.codec.vq.rest.in_proj.%s" +#define TN_A_CODEC_VQ_REST_CB "a.codec.vq.rest.codebook.%s" + // qwen3tts code2wav (RVQ codes -> raw PCM) // pre_transformer layers use the generic TN_ATTN_*/TN_FFN_*/TN_LN_*/TN_LS_* macros, prefix "a.gen.wav.tfm" #define TN_A_GEN_WAV_QUANT_FIRST_IN "a.gen.wav.quant.first.in_proj.%s" @@ -588,6 +613,8 @@ enum projector_type { PROJECTOR_TYPE_DRAMA3_GEN, PROJECTOR_TYPE_ZONOS3_GEN, PROJECTOR_TYPE_ZONOS3_SPKENC, + PROJECTOR_TYPE_FISH_S2_GEN, + PROJECTOR_TYPE_FISH_S2_SPKENC, PROJECTOR_TYPE_MUSE_GLIMMER, PROJECTOR_TYPE_SENSENOVA_U1, PROJECTOR_TYPE_COHERE2V, @@ -680,6 +707,8 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_DRAMA3_GEN, "drama3_gen"}, { PROJECTOR_TYPE_ZONOS3_GEN, "zonos3_gen"}, { PROJECTOR_TYPE_ZONOS3_SPKENC, "zonos3_spkenc"}, + { PROJECTOR_TYPE_FISH_S2_GEN, "fish_s2_gen"}, + { PROJECTOR_TYPE_FISH_S2_SPKENC, "fish_s2_spkenc"}, { PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"}, { PROJECTOR_TYPE_SENSENOVA_U1, "sensenova_u1"}, { PROJECTOR_TYPE_COHERE2V, "cohere2v"}, diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp index 86bb127fb..9e2b2fec1 100644 --- a/tools/mtmd/mtmd-model.cpp +++ b/tools/mtmd/mtmd-model.cpp @@ -1094,7 +1094,11 @@ struct mtmd_model_loader { hparams.audio_sample_rate = 44100; } break; case PROJECTOR_TYPE_DRAMA3_GEN: + case PROJECTOR_TYPE_FISH_S2_GEN: { + // fish_s2 fast AR, drama3 hardcodes its tower RoPE in the graph + hparams.rope_theta = 1000000.0f; + // Fish DAC decoder, codec.pth has no config hparams.audio_sample_rate = 44100; hparams.wav_tfm_n_layer = 8; @@ -1125,6 +1129,19 @@ struct mtmd_model_loader { hparams.flow_n_step = 1; hparams.gen_eos_threshold = -4.0f; } break; + case PROJECTOR_TYPE_FISH_S2_SPKENC: + { + // Fish DAC encoder, codec.pth has no config + // one row per 2048 samples: encoder strides, then the quantizer downsample + hparams.audio_sample_rate = 44100; + hparams.audio_patch_sizes = { 2, 4, 8, 8, 2, 2 }; + // whole frames, about 10 s like a typical reference clip + hparams.warmup_audio_size = hparams.audio_samples_per_token() * 215; + hparams.wav_tfm_n_head = 16; + hparams.wav_tfm_n_head_kv = 16; + hparams.wav_tfm_eps = 1e-5f; + hparams.wav_tfm_rope_theta = 10000.0f; + } break; case PROJECTOR_TYPE_CHATTERBOX: { // the classic cfm solver unrolls up to 10 steps x 2 cfg estimator evaluations in one graph @@ -1372,7 +1389,8 @@ struct mtmd_model_loader { const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA && model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO && model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR && - model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC; + model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC && + model.proj_type != PROJECTOR_TYPE_FISH_S2_SPKENC; // Validate audio hparams loaded from GGUF metadata if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) { @@ -2526,6 +2544,71 @@ struct mtmd_model_loader { model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight")); model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias")); } break; + case PROJECTOR_TYPE_FISH_S2_SPKENC: + { + // codec encoder in the code2wav structs: dac = encoder blocks, upsample = quantizer downsample + // generic a.blk.* layers: 0..3 last encoder block, 4..11 quantizer pre_module + auto & c2w = model.c2w; + c2w.dac_entry_w = get_tensor(mtmd_format(TN_A_CODEC_ENC_ENTRY, "weight")); + c2w.dac_entry_b = get_tensor(mtmd_format(TN_A_CODEC_ENC_ENTRY, "bias")); + c2w.dac.resize(4); + for (int il = 0; il < (int) c2w.dac.size(); il++) { + auto & blk = c2w.dac[il]; + blk.snake_alpha = get_tensor(mtmd_format(TN_A_CODEC_ENC_SNAKE, il, "alpha")); + blk.snake_beta = get_tensor(mtmd_format(TN_A_CODEC_ENC_SNAKE, il, "beta")); + blk.conv_w = get_tensor(mtmd_format(TN_A_CODEC_ENC_CONV, il, "weight")); + blk.conv_b = get_tensor(mtmd_format(TN_A_CODEC_ENC_CONV, il, "bias")); + blk.res.resize(3); + for (int ir = 0; ir < (int) blk.res.size(); ir++) { + auto & res = blk.res[ir]; + res.act1_alpha = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_ACT1, il, ir, "alpha")); + res.act1_beta = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_ACT1, il, ir, "beta")); + res.conv1_w = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_CONV1, il, ir, "weight")); + res.conv1_b = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_CONV1, il, ir, "bias")); + res.act2_alpha = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_ACT2, il, ir, "alpha")); + res.act2_beta = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_ACT2, il, ir, "beta")); + res.conv2_w = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_CONV2, il, ir, "weight")); + res.conv2_b = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_CONV2, il, ir, "bias")); + } + } + c2w.dac_post_snake_alpha = get_tensor(mtmd_format(TN_A_CODEC_ENC_POST_SNAKE, "alpha")); + c2w.dac_post_snake_beta = get_tensor(mtmd_format(TN_A_CODEC_ENC_POST_SNAKE, "beta")); + c2w.dac_post_conv_w = get_tensor(mtmd_format(TN_A_CODEC_ENC_POST_CONV, "weight")); + c2w.dac_post_conv_b = get_tensor(mtmd_format(TN_A_CODEC_ENC_POST_CONV, "bias")); + + c2w.upsample.resize(2); + for (int il = 0; il < (int) c2w.upsample.size(); il++) { + auto & down = c2w.upsample[il]; + down.conv_w = get_tensor(mtmd_format(TN_A_CODEC_DOWN_CONV, il, "weight")); + down.conv_b = get_tensor(mtmd_format(TN_A_CODEC_DOWN_CONV, il, "bias")); + down.dwconv_w = get_tensor(mtmd_format(TN_A_CODEC_DOWN_DWCONV, il, "weight")); + down.dwconv_b = get_tensor(mtmd_format(TN_A_CODEC_DOWN_DWCONV, il, "bias")); + down.norm_w = get_tensor(mtmd_format(TN_A_CODEC_DOWN_NORM, il, "weight")); + down.norm_b = get_tensor(mtmd_format(TN_A_CODEC_DOWN_NORM, il, "bias")); + down.pw1_w = get_tensor(mtmd_format(TN_A_CODEC_DOWN_PW1, il, "weight")); + down.pw1_b = get_tensor(mtmd_format(TN_A_CODEC_DOWN_PW1, il, "bias")); + down.pw2_w = get_tensor(mtmd_format(TN_A_CODEC_DOWN_PW2, il, "weight")); + down.pw2_b = get_tensor(mtmd_format(TN_A_CODEC_DOWN_PW2, il, "bias")); + down.gamma = get_tensor(mtmd_format(TN_A_CODEC_DOWN_GAMMA, il)); + } + c2w.tfm_output_norm_w = get_tensor(mtmd_format(TN_A_CODEC_PRE_NORM, "weight")); + if (hparams.n_layer != mtmd_graph_fish_s2_spkenc::n_tfm_layer) { + throw std::runtime_error(mtmd_format("%s: fish_s2_spkenc expects %d transformer layers\n", __func__, mtmd_graph_fish_s2_spkenc::n_tfm_layer)); + } + // VQ: search with the normalized codebooks, subtract the codec decoder's out_proj tables + c2w.quant_first_in_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_FIRST_IN, "weight")); + c2w.quant_first_in_b = get_tensor(mtmd_format(TN_A_CODEC_VQ_FIRST_IN, "bias")); + c2w.quant_first_cb_norm_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_FIRST_CB, "weight")); + c2w.quant_rest_in_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_REST_IN, "weight")); + c2w.quant_rest_in_b = get_tensor(mtmd_format(TN_A_CODEC_VQ_REST_IN, "bias")); + c2w.quant_rest_cb_norm_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_REST_CB, "weight")); + c2w.quant_first_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight")); + c2w.quant_rest_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_CB, "weight")); + + // codes -> backbone rows, same tables as fish_s2_gen + model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight")); + model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); + } break; case PROJECTOR_TYPE_POCKETTTS_SPKENC: { load_seanet(model.seanet, false); @@ -2827,6 +2910,20 @@ struct mtmd_model_loader { model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); // backbone rows of the semantic band load_code2wav(false); } break; + case PROJECTOR_TYPE_FISH_S2_GEN: + { + // generic a.gen.code.blk.* tensors are loaded by the loop above + model.gen_code_tok_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_TOK_EMBD, "weight")); // fast AR input + model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight")); // fast AR head, residual codebooks + model.gen_code_norm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_NORM, "weight")); + model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight")); // merged 3D, backbone input tables + model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); // backbone rows of the semantic band + load_code2wav(false); + // the graph reads model.layers[0] + if (hparams.n_layer < 1) { + throw std::runtime_error(mtmd_format("%s: fish_s2_gen has no fast AR layer\n", __func__)); + } + } break; case PROJECTOR_TYPE_CHATTERBOX: { auto & c = model.cbx; @@ -4440,10 +4537,16 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const { case PROJECTOR_TYPE_LFM3A_GEN: case PROJECTOR_TYPE_DRAMA3_GEN: case PROJECTOR_TYPE_ZONOS3_GEN: + case PROJECTOR_TYPE_FISH_S2_GEN: { // one hidden-state vector fed back to the talker per call n_patches = 1; } break; + case PROJECTOR_TYPE_FISH_S2_SPKENC: + { + // one backbone row per codec frame, the preprocessor pads to whole frames + n_patches = (int) (img->nx() / hparams.audio_samples_per_token()); + } break; case PROJECTOR_TYPE_POCKETTTS_SPKENC: { // one conditioning row per 12.5Hz frame @@ -4630,6 +4733,8 @@ int mtmd_model::n_mmproj_embd() const { case PROJECTOR_TYPE_LFM3A_GEN: case PROJECTOR_TYPE_DRAMA3_GEN: case PROJECTOR_TYPE_ZONOS3_GEN: + case PROJECTOR_TYPE_FISH_S2_GEN: + case PROJECTOR_TYPE_FISH_S2_SPKENC: return gen_code_out_embd_w->ne[0]; case PROJECTOR_TYPE_POCKETTTS_SPKENC: return spk_proj_w->ne[1]; diff --git a/tools/mtmd/mtmd-model.h b/tools/mtmd/mtmd-model.h index 44863636b..9af815981 100644 --- a/tools/mtmd/mtmd-model.h +++ b/tools/mtmd/mtmd-model.h @@ -213,6 +213,8 @@ struct mtmd_hparams { // pocket-tts static constexpr int32_t pockettts_max_spk_seconds = 30; + // fish_s2: the codec encoder runs at 44.1 kHz, memory grows with the clip + static constexpr int32_t fish_s2_max_spk_seconds = 30; int32_t seanet_n_stage = 0; std::vector seanet_ratios; // encoder order (reversed compared to the config) int32_t mimi_downsample = 0; // encoder frame rate / model frame rate @@ -622,6 +624,12 @@ struct mtmd_code2wav { ggml_tensor * quant_rest_out_w = nullptr; ggml_tensor * quant_rest_cb_w = nullptr; // codebooks, merged 3D [15, vocab, dim] + // fish_s2 codec encoder: in_proj bias and l2-normalized codebooks for the nearest-entry search + ggml_tensor * quant_first_in_b = nullptr; + ggml_tensor * quant_rest_in_b = nullptr; + ggml_tensor * quant_first_cb_norm_w = nullptr; + ggml_tensor * quant_rest_cb_norm_w = nullptr; + ggml_tensor * pre_conv_w = nullptr; ggml_tensor * pre_conv_b = nullptr; @@ -1208,6 +1216,7 @@ struct mtmd_model { ggml_tensor * gen_code_head_w = nullptr; // per-codebook output head, merged 3D ggml_tensor * gen_code_out_embd_w = nullptr; // codebook-0 embedding, fed back into the talker ggml_tensor * gen_code_norm_w = nullptr; // final norm + ggml_tensor * gen_code_tok_embd_w = nullptr; // fish_s2: input embedding shared by all codebooks // lfm3a depthformer (MaskGIT) ggml_tensor * gen_code_head_norm_w = nullptr; // per-codebook norm before the tied head, merged 2D diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp index db54ec9c9..db3b241d4 100644 --- a/tools/mtmd/mtmd-subcontext.cpp +++ b/tools/mtmd/mtmd-subcontext.cpp @@ -389,6 +389,7 @@ bool mtmd_subcontext::encode_batch(int n_threads, const mtmd_image_f32_batch * i static std::vector list_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { switch (model.proj_type) { case PROJECTOR_TYPE_QWEN3TTS_GEN: return list_c2w_state_slots(hparams, model); + case PROJECTOR_TYPE_FISH_S2_GEN: return list_c2w_state_slots(hparams, model); case PROJECTOR_TYPE_DRAMA3_GEN: return list_drama3_state_slots(hparams, model); case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model); case PROJECTOR_TYPE_LFM3A_GEN: return list_lfm3a_gen_state_slots(hparams, model); @@ -1680,6 +1681,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: case PROJECTOR_TYPE_QWEN3TTS_SPKENC: case PROJECTOR_TYPE_ZONOS3_SPKENC: + case PROJECTOR_TYPE_FISH_S2_SPKENC: { // do nothing } break; @@ -1789,6 +1791,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { } } break; case PROJECTOR_TYPE_DRAMA3_GEN: + case PROJECTOR_TYPE_FISH_S2_GEN: { const int64_t n_cb = model.gen_code_embd_w->ne[2]; const int64_t n_vocab = model.gen_code_embd_w->ne[1]; @@ -1807,6 +1810,13 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { } set_input_i32("inp_codes", codes); set_gen_state_in(); + } else if (model.proj_type == PROJECTOR_TYPE_FISH_S2_GEN) { + if (params->code0 < 0 || params->code0 >= n_vocab) { + LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) n_vocab); + return false; + } + std::vector code0 = { params->code0 }; + set_input_i32("inp_code0", code0); } else { // code0 < 0: next row is text (prompt), the tower only fills its KV // codes: input frame codes for a speech row, empty for a text row @@ -2622,8 +2632,8 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { mtmd_graph_soprano::decode_spectrum(model, spectrum, out_audio); } - // drop the tail audio that comes from the code-0 rear padding, drama3 has no padding - if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN) { + // drop the tail audio that comes from the code-0 rear padding, drama3 and fish_s2 have no padding + if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN && model.proj_type != PROJECTOR_TYPE_FISH_S2_GEN) { const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0; const int64_t n_frames_w = hparams.wav_tfm_swa; const int64_t n_frames = params->codes ? (int64_t) params->codes->size() / n_codes : n_frames_w; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index f8f7ea048..9669afc2a 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1314,6 +1314,10 @@ struct mtmd_context { { audio_preproc = std::make_unique(models.a.get()); } break; + case PROJECTOR_TYPE_FISH_S2_SPKENC: + { + audio_preproc = std::make_unique(models.a.get()); + } break; case PROJECTOR_TYPE_CHATTERBOX_SPKENC: { audio_preproc = std::make_unique(models.a.get(), cbx_is_mtl(*models.a)); @@ -2490,6 +2494,10 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { info.type = MTMD_GEN_AUDIO_TYPE_ZONOS3; info.sample_rate = 44100; break; + case PROJECTOR_TYPE_FISH_S2_GEN: + info.type = MTMD_GEN_AUDIO_TYPE_FISH_S2; + info.sample_rate = 44100; + break; default: info.type = MTMD_GEN_AUDIO_TYPE_NONE; break; diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index b0d673dbe..58c7c57a8 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -473,6 +473,7 @@ enum mtmd_gen_audio_type { MTMD_GEN_AUDIO_TYPE_CHATTERBOX, MTMD_GEN_AUDIO_TYPE_DRAMA3, MTMD_GEN_AUDIO_TYPE_ZONOS3, + MTMD_GEN_AUDIO_TYPE_FISH_S2, }; struct mtmd_gen_audio_info { diff --git a/tools/server/README.md b/tools/server/README.md index 9d1028d25..07b530df0 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -866,6 +866,8 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For the list `speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence. Rejected when the model has no voice encoder (e.g. Pocket TTS without voice cloning). +`speaker_ref_text`: Transcript of the reference audio, put in the prompt by some models (e.g. Fish Audio S2 Pro). Optional. + `top_k`, `top_p`: Sampling params for both the backbone sampler and the acoustic code predictor. Optional, `0` or absent keeps the server `--top-k`/`--top-p` for the backbone and the model defaults for the code predictor. `temperature`: Temperature for the backbone sampler and the audio frames, for models that support it. `0` is greedy sampling. Optional, absent keeps the server `--temp` for the backbone and the model defaults for the audio. diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 28e01c69b..abaaf55b6 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -6992,6 +6992,7 @@ std::unique_ptr server_routes::handle_tts_impl(const serve } task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt); task.tts_inp.set_lang(lang); + task.tts_inp.set_speaker_ref_text(json_value(body, "speaker_ref_text", std::string())); task.tts_inp.data.stream = stream; task.tts_inp.data.out_type = response_format == "pcm" || pcm16 ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM diff --git a/tools/tts/README.md b/tools/tts/README.md index 9608db569..9df5078bb 100644 --- a/tools/tts/README.md +++ b/tools/tts/README.md @@ -17,6 +17,7 @@ Note: it's highly recommended to always provide a speaker reference voice (via ` Common params: - Sampling params such as `--top-k`, `--top-p`, `--temp`, etc. `--temp` also sets the temperature of the audio frames for models that support it, without it they use the model default - `-n ` limits the output length, e.g. `-n 500`. Note that how many milliseconds each frame represents varies by model +- `--tts-speaker-text` (optional) gives the transcript of `--tts-speaker-file`, some models put it in the prompt (e.g. Fish Audio S2 Pro) - Core inference params such as `-ngl`, `-b`, `-ub`, etc. ## Supported models diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp index 7edf605b8..bbc9ac794 100644 --- a/tools/tts/tts.cpp +++ b/tools/tts/tts.cpp @@ -147,6 +147,7 @@ int main(int argc, char ** argv) { inp.prompt = prompt.c_str(); inp.prompt_len = prompt.size(); inp.speaker_ref = speaker_bitmap.get(); + inp.speaker_ref_text = params.tts_speaker_text.empty() ? nullptr : params.tts_speaker_text.c_str(); inp.lang = params.tts_lang.c_str(); inp.top_k = params.sampling.top_k; inp.top_p = params.sampling.top_p; @@ -168,7 +169,7 @@ int main(int argc, char ** argv) { const int64_t t_prompt_start_us = ggml_time_us(); for (;;) { - int32_t ret = gen.step_prompt(params.n_batch); + int32_t ret = gen.step_prompt(llama_n_batch(lctx)); if (ret < 0) { LOG_ERR("prompt processing failed\n"); return 1;