From 7ae3c6b231e38057e825ff257bb734e456633d9a Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Sun, 15 Nov 2026 19:44:44 +0100 Subject: [PATCH] mtmd: add zonos3-mini TTS model (#32020) * zonos3 init * add audio gen pipeline * fix string_format * add default voice as gguf metadata * better ref * support voice clone * add delivery params * optimize speed * nits * nits 2 * add text normalization * nits --- conversion/__init__.py | 2 + conversion/zonos3.py | 575 ++++++++++++++++ docs/multimodal/tts.md | 35 + gguf-py/gguf/constants.py | 38 ++ gguf-py/gguf/gguf_writer.py | 3 + models/templates/Zyphra-ZONOS3-Mini-tts.jinja | 627 ++++++++++++++++++ src/llama-arch.cpp | 2 + src/llama-arch.h | 1 + src/llama-context.cpp | 1 + src/llama-model.cpp | 12 +- src/llama-vocab.cpp | 10 + src/llama-vocab.h | 1 + src/models/models.h | 22 + src/models/zonos3.cpp | 351 ++++++++++ tests/fusion/MTL.csv | 5 + tests/test-jinja.cpp | 71 ++ tests/test-llama-archs.cpp | 8 +- tests/test-server-component.cpp | 6 +- tools/mtmd/CMakeLists.txt | 2 + tools/mtmd/README-dev.md | 1 + tools/mtmd/models/models.h | 62 ++ tools/mtmd/models/zonos3-gen.cpp | 207 ++++++ tools/mtmd/models/zonos3-spkenc.cpp | 221 ++++++ tools/mtmd/mtmd-audio.cpp | 367 +++++++--- tools/mtmd/mtmd-audio.h | 55 ++ tools/mtmd/mtmd-graph.cpp | 14 + tools/mtmd/mtmd-helper-gen.cpp | 543 +++++++++++++++ tools/mtmd/mtmd-helper.h | 5 + tools/mtmd/mtmd-impl.h | 5 + tools/mtmd/mtmd-model.cpp | 149 +++++ tools/mtmd/mtmd-model.h | 57 ++ tools/mtmd/mtmd-subcontext.cpp | 50 ++ tools/mtmd/mtmd.cpp | 95 ++- tools/mtmd/mtmd.h | 4 +- tools/server/README.md | 6 +- tools/server/server-chat.cpp | 4 - tools/server/server-context.cpp | 9 +- tools/server/server-schema.cpp | 6 + 38 files changed, 3532 insertions(+), 100 deletions(-) create mode 100644 conversion/zonos3.py create mode 100644 models/templates/Zyphra-ZONOS3-Mini-tts.jinja create mode 100644 src/models/zonos3.cpp create mode 100644 tools/mtmd/models/zonos3-gen.cpp create mode 100644 tools/mtmd/models/zonos3-spkenc.cpp diff --git a/conversion/__init__.py b/conversion/__init__.py index fb6b8464c..e915753a8 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -282,6 +282,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "Qwen3NextForCausalLM": "qwen", "Qwen3OmniMoeForConditionalGeneration": "qwen3vl", "PocketTTSModel": "pockettts", + "Zonos3Model": "zonos3", "Qwen3TTSForConditionalGeneration": "qwen3tts", "Qwen3VLForConditionalGeneration": "qwen3vl", "Qwen3VLMoeForConditionalGeneration": "qwen3vl", @@ -356,6 +357,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "EmbeddingGemma2Model": "gemma", "Exaone4_5_ForConditionalGeneration": "exaone", "FishOmniModel": "drama3", + "Zonos3Model": "zonos3", "Gemma3ForConditionalGeneration": "gemma", "Gemma3nForConditionalGeneration": "gemma", "Gemma4ForConditionalGeneration": "gemma", diff --git a/conversion/zonos3.py b/conversion/zonos3.py new file mode 100644 index 000000000..bf1c7a625 --- /dev/null +++ b/conversion/zonos3.py @@ -0,0 +1,575 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Callable, Iterable, TYPE_CHECKING + +import numpy as np +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, TextModel, MmprojModel, LazyTorchTensor, gguf, logger + +LM_DIR = Path("lm") / "zonos3_mini" + +# codec decoder: quantizers and semantic head from the generator, the rest from the head checkpoint +CODEC_GENERATOR = Path("autoencoder") / "generator.pt" +CODEC_VARIANT = "keep2_plain" +CODEC_HEAD = Path("autoencoder") / "heads" / CODEC_VARIANT / "step_0050000.pt" +CODEC_STAGES = [0, 2, 4, 6, 8] # transformer stages of the decoder + +DEFAULT_VOICE = "Savannah_en-American_F_1" + +# speaker encoder (ownmect_s2), MECT_B2 blocks: (cnn depth, out channels, (stride f, stride t), expansion) +SPK_CKPT = Path("speaker_encoders") / "ownmect_s2" / "ckpt_0037.pth" +SPK_STAGES = [ + [(3, 32, (1, 1), 4)] * 2, + [(3, 64, (2, 1), 2)] + [(3, 64, (1, 2), 2)] * 3, + [(3, 128, (2, 1), 2)] + [(3, 128, (1, 2), 2)] * 4, + [(3, 256, (2, 1), 1)] + [(2, 256, (1, 2), 1)] * 2, +] +VOICE_MAX_FRAMES = 900 # reference_max_frames of the reference + +# GDN geometry, hard-coded in the reference (zonos3.reference.lm) +GDN_HEADS, GDN_HEAD_DIM, GDN_KERNEL = 4, 128, 4 + +# the text table keeps the tokenizer ids shifted by this offset, ids 2 and 3 are BOS and EOS +TEXT_OFFSET = 4 +TEXT_BOS, TEXT_EOS = 2, 3 + + +def _round_up(value: float, multiple: int) -> int: + return multiple * ((int(value) + multiple - 1) // multiple) + + +def _load_params(dir_model: Path) -> dict[str, Any] | None: + path = dir_model / LM_DIR / "params.json" + if not path.is_file(): + return None + with open(path, encoding="utf-8") as f: + params = json.load(f).get("model", {}) + return params if params.get("model_type") == "zonos2" else None + + +def _is_zonos3_checkpoint(dir_model: Path) -> bool: + return _load_params(dir_model) is not None + + +@ModelBase.register_hparams_loader(_is_zonos3_checkpoint) +def _load_zonos3_hparams(dir_model: Path) -> dict[str, Any]: + logger.info("gguf: detected ZONOS3 checkpoint") + params = _load_params(dir_model) + assert params is not None + for key, want in (("attention_pattern", "gdn3_global1"), ("residual_type", "full_attnres"), + ("moe_router_type", "linear"), ("moe_router_use_eda", False), ("use_xsa", False), + ("moe_end_from_layer", 0), ("speaker_enabled", True)): + if params.get(key) != want: + raise NotImplementedError(f"ZONOS3: unsupported {key} = {params.get(key)!r}, expected {want!r}") + text_config = dict(params, hidden_size=params["dim"]) + return {"architectures": ["Zonos3Model"], "text_config": text_config, "audio_config": {"num_hidden_layers": 0}} + + +def _index_lm_tensors(model: ModelBase) -> dict[str, Callable[[], Tensor]]: + path = model.dir_model / LM_DIR / "model.pth" + logger.info(f"gguf: indexing model part '{LM_DIR / 'model.pth'}'") + state = torch.load(str(path), map_location="cpu", mmap=True, weights_only=True) + tensors: dict[str, Callable[[], Tensor]] = {} + for name, data in state.items(): + gen = (lambda data=data: LazyTorchTensor.from_eager(data)) if model.lazy else (lambda data=data: data) + titem = model.filter_tensors((name, gen)) + model._source_tensor_names[name] = titem[0] if titem else None + if titem: + tensors[titem[0]] = titem[1] + return tensors + + +@ModelBase.register("Zonos3Model") +class Zonos3TextModel(TextModel): + model_arch = gguf.MODEL_ARCH.ZONOS3 + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + hp = self.hparams + self.n_embd = hp["dim"] + self.n_codebooks = hp["n_codebooks"] + self.audio_vocab = hp["codebook_size"] + 2 # codes, EOA, pad + self.audio_pad_id = hp["audio_pad_id"] + self.text_rows = hp["text_vocab"] + 1 # + the padding row, used by the generated rows + self.cond_counts = [n + 1 for n in hp["condition_bucket_counts"]] # + the unknown bucket + self.gdn_period = int(hp["attention_pattern"][len("gdn"):].split("_")[0]) + 1 + self.n_ff = _round_up(hp["ffn_dim_multiplier"] * hp["dim"], hp["multiple_of"]) + self.n_ff_exp = _round_up(hp["moe_ffn_dim_multiplier"] * hp["dim"], hp["moe_multiple_of"]) + self.n_latent = int(hp["dim"] * hp["moe_alpha"]) + self._embd: dict[str, Tensor] = {} + self._router: dict[int, dict[str, Tensor]] = {} + + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + return _index_lm_tensors(self) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, _ = item + # audio heads and speaker projection go to the mmproj + if name.startswith(("multi_output.", "speaker_projection.")): + return None + return item + + def set_vocab(self): + with open(self.dir_model / LM_DIR / "tokenizer" / "superbpe8192.json", encoding="utf-8") as f: + tok = json.load(f) + model = tok["model"] + if model["type"] != "BPE" or model.get("byte_fallback") or tok.get("normalizer") is not None: + raise NotImplementedError("ZONOS3: unexpected tokenizer type") + + vocab: dict[str, int] = model["vocab"] + n_bpe = len(vocab) + if n_bpe + TEXT_OFFSET != self.hparams["text_vocab"]: + raise ValueError(f"ZONOS3: tokenizer has {n_bpe} tokens, text_vocab is {self.hparams['text_vocab']}") + added = {t["id"] for t in tok.get("added_tokens", [])} + + tokens: list[str] = ["<|unused_0|>", "<|unused_1|>", "<|bos|>", "<|eos|>"] + toktypes: list[int] = [gguf.TokenType.CONTROL] * TEXT_OFFSET + inv = {i: t for t, i in vocab.items()} + for i in range(n_bpe): + tokens.append(inv[i]) + toktypes.append(gguf.TokenType.CONTROL if i in added else gguf.TokenType.NORMAL) + + # padding row of the text table (the text column of every generated row) + tokens.append("<|text_pad|>") + toktypes.append(gguf.TokenType.CONTROL) + + # condition rows, the last bucket of each table is "unknown" + for cond, n in zip(self.hparams["condition_names"], self.cond_counts): + tokens += [f"<|{cond}_{i}|>" for i in range(n - 1)] + [f"<|{cond}_unk|>"] + toktypes += [gguf.TokenType.CONTROL] * n + + merges = [" ".join(m) if isinstance(m, list) else m for m in model["merges"]] + + self.gguf_writer.add_tokenizer_model("gpt2") + self.gguf_writer.add_tokenizer_pre("zonos3") + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_types(toktypes) + self.gguf_writer.add_token_merges(merges) + self.gguf_writer.add_bos_token_id(TEXT_BOS) + self.gguf_writer.add_eos_token_id(TEXT_EOS) + self.gguf_writer.add_pad_token_id(TEXT_OFFSET + 1) # <|pad|> of the tokenizer + # the reference wraps the text as [BOS] + ids + [EOS] + self.gguf_writer.add_add_bos_token(True) + self.gguf_writer.add_add_eos_token(True) + # the reference normalizes the text with NeMo, the template covers the English numbers + template_path = Path(__file__).parent.parent / "models" / "templates" / "Zyphra-ZONOS3-Mini-tts.jinja" + with open(template_path, "r", encoding="utf-8") as f: + self.gguf_writer.add_chat_template([{"name": "tts", "template": f.read()}]) + + def set_gguf_parameters(self): + hp = self.hparams + n_layer = self.block_count + self.gguf_writer.add_file_type(self.ftype) + self.gguf_writer.add_block_count(n_layer) + self.gguf_writer.add_context_length(hp["max_seqlen"]) + self.gguf_writer.add_embedding_length(self.n_embd) + self.gguf_writer.add_feed_forward_length(self.n_ff) + self.gguf_writer.add_head_count(hp["n_heads"]) + self.gguf_writer.add_head_count_kv(hp["n_kv_heads"]) + self.gguf_writer.add_key_length(hp["head_dim"]) + self.gguf_writer.add_value_length(hp["head_dim"]) + self.gguf_writer.add_rope_dimension_count(hp["head_dim"]) + self.gguf_writer.add_rope_freq_base(hp["rope_theta"]) + self.gguf_writer.add_layer_norm_rms_eps(hp["norm_eps"]) + self.gguf_writer.add_full_attention_interval(self.gdn_period) + + # gated delta net + self.gguf_writer.add_ssm_conv_kernel(GDN_KERNEL) + self.gguf_writer.add_ssm_inner_size(GDN_HEADS * GDN_HEAD_DIM) + self.gguf_writer.add_ssm_state_size(GDN_HEAD_DIM) + self.gguf_writer.add_ssm_time_step_rank(GDN_HEADS) + self.gguf_writer.add_ssm_group_count(GDN_HEADS) + + # latent MoE, the gating (softmax over the selected logits) is hard-coded in libllama + self.gguf_writer.add_leading_dense_block_count(hp["moe_start_from_layer"]) + self.gguf_writer.add_expert_count(hp["moe_n_experts"]) + self.gguf_writer.add_expert_used_count(hp["moe_router_topk"]) + self.gguf_writer.add_expert_feed_forward_length(self.n_ff_exp) + self.gguf_writer.add_moe_latent_size(self.n_latent) + + def tensor_force_quant(self, name, new_name, bid, n_dims): + # router logits pick the experts, keep them exact + if new_name.endswith(("ffn_gate_inp.weight", "exp_probs_b.bias")): + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + def _token_embd(self) -> Tensor: + n_cb = self.n_codebooks + pad = sum(self._embd[f"multi_embedder.embedders.{cb}.weight"][self.audio_pad_id].float() for cb in range(n_cb)) + text = self._embd[f"multi_embedder.embedders.{n_cb}.weight"].float() + if text.shape[0] != self.text_rows: + raise ValueError(f"ZONOS3: text table has {text.shape[0]} rows, expected {self.text_rows}") + conds = [self._embd[f"condition_embeddings.{i}.weight"].float() for i in range(len(self.cond_counts))] + for t, n in zip(conds, self.cond_counts): + assert t.shape[0] == n + # a text row is [pad x n_codebooks, text_id], condition rows replace the summed row + return torch.cat([text + pad] + conds, dim=0) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + T = gguf.MODEL_TENSOR + hp = self.hparams + n_layer = self.block_count + + if name.startswith(("multi_embedder.", "condition_embeddings.")): + self._embd[name] = data_torch + if len(self._embd) == self.n_codebooks + 1 + len(self.cond_counts): + yield (self.format_tensor_name(T.TOKEN_EMBD), self._token_embd()) + self._embd.clear() + return + + if name == "block_attnres_queries": + # one query per routing point: 2*il before the mixer, 2*il+1 before the FFN, the last one before out_norm + assert data_torch.shape[0] == 2 * n_layer + 1 + for il in range(n_layer): + yield (self.format_tensor_name(T.ATTN_RES_SCORE, il), data_torch[2 * il]) + yield (self.format_tensor_name(T.FFN_RES_SCORE, il), data_torch[2 * il + 1]) + yield (self.format_tensor_name(T.OUTPUT_RES_SCORE), data_torch[2 * n_layer]) + return + + if name == "out_norm.weight": + yield (self.format_tensor_name(T.OUTPUT_NORM), data_torch) + return + + assert bid is not None, name + key = name.split(".", 2)[2] # strip "layers.{bid}." + + if key == "attention_norm.weight": + yield (self.format_tensor_name(T.ATTN_NORM, bid), data_torch) + return + if key == "ffn_norm.weight": + yield (self.format_tensor_name(T.FFN_NORM, bid), data_torch) + return + + # gated delta net + if key == "attention.in_proj.weight": + # split the fused [qkv | z | a | b] weight into the qwen35 GDN tensors + # attn_gate holds z, per-head output gate in the attention layers + inner = GDN_HEADS * GDN_HEAD_DIM + qkv, z, a, b = data_torch.split([3 * inner, inner, GDN_HEADS, GDN_HEADS], dim=0) + yield (self.format_tensor_name(T.ATTN_QKV, bid), qkv) + yield (self.format_tensor_name(T.ATTN_GATE, bid), z) + yield (self.format_tensor_name(T.SSM_ALPHA, bid), a) + yield (self.format_tensor_name(T.SSM_BETA, bid), b) + return + if key == "attention.conv.weight": + yield (self.format_tensor_name(T.SSM_CONV1D, bid), data_torch.squeeze(1)) + return + if key == "attention.A_log": + yield (self.format_tensor_name(T.SSM_A, bid, suffix=""), -torch.exp(data_torch.float())) + return + if key == "attention.dt_bias": + yield (self.format_tensor_name(T.SSM_DT, bid, suffix=".bias"), data_torch) + return + if key == "attention.out_weight": + yield (self.format_tensor_name(T.SSM_NORM, bid), data_torch) + return + if key == "attention.wo.weight" and (bid + 1) % self.gdn_period != 0: + yield (self.format_tensor_name(T.SSM_OUT, bid), data_torch) + return + + # global attention + if key == "attention.wq.weight": + yield (self.format_tensor_name(T.ATTN_Q, bid), data_torch) + return + if key == "attention.wkv.weight": + yield (self.format_tensor_name(T.ATTN_K, bid), data_torch[0]) + yield (self.format_tensor_name(T.ATTN_V, bid), data_torch[1]) + return + if key == "attention.wo.weight": + yield (self.format_tensor_name(T.ATTN_OUT, bid), data_torch) + return + if key == "attention.gater.weight": + # one sigmoid gate per head, same tensor name as the GDN z gate + yield (self.format_tensor_name(T.ATTN_GATE, bid), data_torch) + return + if key == "attention.temp": + # Q = rms_norm(q) * |temp|, the per-head scale is stored as the Q norm weight + temp = data_torch.float().abs().reshape(hp["n_heads"], 1).expand(hp["n_heads"], hp["head_dim"]) + yield (self.format_tensor_name(T.ATTN_Q_NORM, bid), temp.contiguous()) + return + + # dense FFN (layer 0): out = w_out(w_in[0](x) * silu(w_in[1](x))) + if key == "feed_forward.w_in.weight": + yield (self.format_tensor_name(T.FFN_UP, bid), data_torch[0]) + yield (self.format_tensor_name(T.FFN_GATE, bid), data_torch[1]) + return + if key == "feed_forward.w_out.weight": + yield (self.format_tensor_name(T.FFN_DOWN, bid), data_torch) + return + + # latent MoE + if key in ("feed_forward.router.router_linear.weight", "feed_forward.router.rmsnorm_eda.weight"): + # logits = router_linear(rms_norm(x) * w), the norm weight is folded into the router columns + r = self._router.setdefault(bid, {}) + r[key] = data_torch + if len(r) == 2: + w = r["feed_forward.router.router_linear.weight"].float() * r["feed_forward.router.rmsnorm_eda.weight"].float()[None, :] + yield (self.format_tensor_name(T.FFN_GATE_INP, bid), w) + del self._router[bid] + return + if key == "feed_forward.router.balancing_biases": + # the reference selects on logits - bias + yield (self.format_tensor_name(T.FFN_EXP_PROBS_B, bid, suffix=".bias"), -data_torch.float()) + return + if key == "feed_forward.latent_down.weight": + yield (self.format_tensor_name(T.MOE_LATENT_DOWN, bid), data_torch) + return + if key == "feed_forward.latent_up.weight": + yield (self.format_tensor_name(T.MOE_LATENT_UP, bid), data_torch) + return + if key == "feed_forward.experts.gate_up_proj": + gate, up = data_torch.chunk(2, dim=1) + yield (self.format_tensor_name(T.FFN_GATE_EXP, bid), gate) + yield (self.format_tensor_name(T.FFN_UP_EXP, bid), up) + return + if key == "feed_forward.experts.down_proj": + yield (self.format_tensor_name(T.FFN_DOWN_EXP, bid), data_torch) + return + + raise ValueError(f"ZONOS3: unexpected tensor {name}") + + +@ModelBase.register("Zonos3Model") +class Zonos3MmprojModel(MmprojModel): + has_vision_encoder = False + has_audio_encoder = True + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + hp = self.global_config["text_config"] + self.n_codebooks = hp["n_codebooks"] + self.text_vocab = hp["text_vocab"] + self._embd: dict[str, Tensor] = {} + + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + return _index_lm_tensors(self) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, _ = item + if name.startswith(("multi_output.", "multi_embedder.", "speaker_projection.")): + return item + return None + + def set_gguf_parameters(self): + self.gguf_writer.add_file_type(self.ftype) + + # speaker encoder for voice cloning, the front-end hparams are hard-coded in tools/mtmd + self.gguf_writer.add_clip_has_audio_encoder(True) + self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.ZONOS3_SPKENC) + self.gguf_writer.add_audio_projection_dim(self.n_embd_text) + self.gguf_writer.add_audio_num_mel_bins(80) + self.gguf_writer.add_audio_block_count(0) + self.gguf_writer.add_audio_embedding_length(192) + self.gguf_writer.add_audio_head_count(1) + self.gguf_writer.add_audio_feed_forward_length(192) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) + + self.gguf_writer.add_clip_has_gen_audio_encoder(True) + self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.ZONOS3_GEN) + self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text) + self.gguf_writer.add_gen_audio_embedding_length(self.n_embd_text) + self.gguf_writer.add_gen_audio_feed_forward_length(0) + self.gguf_writer.add_gen_audio_block_count(0) + self.gguf_writer.add_gen_audio_head_count(0) + self.gguf_writer.add_gen_audio_attention_layernorm_eps(self.global_config["text_config"]["norm_eps"]) + self.gguf_writer.add_gen_audio_model_variant(CODEC_VARIANT) + if any(self.dir_model.glob(f"gguf/voices/*/{DEFAULT_VOICE}.codes.npy")): + self.gguf_writer.add_gen_audio_default_voice(DEFAULT_VOICE) + # softcap and the codec hparams are hard-coded in tools/mtmd + + def tensor_force_quant(self, name, new_name, bid, n_dims): + # read on the host, or used by get_rows / ggml_ssm_conv / the conv ops of the speaker encoder + if new_name.startswith(("voice.", "a.spk.")) or new_name.endswith((".table", ".dw.weight", ".mixer.weight", ".resample.weight")): + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + @staticmethod + def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor: + # torch weight norm, over every dim but 0 + g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float() + return (g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))).squeeze(-1) + + def _speaker_encoder(self) -> Iterable[tuple[str, Tensor]]: + # same layout as the reference ggml runtime (tools/convert_gguf.py speaker-encoder) + sd = torch.load(self.dir_model / SPK_CKPT, map_location="cpu", weights_only=True)["modules"]["spk_model"] + sd = {k[len("model."):]: v.double() for k, v in sd.items() if not k.endswith("num_batches_tracked")} + p = "a.spk" + + def bn(dst: str, src: str) -> Iterable[tuple[str, Tensor]]: + scale = sd[src + ".weight"] / torch.sqrt(sd[src + ".running_var"] + 1e-5) + yield (f"{p}.{dst}.scale", scale) + yield (f"{p}.{dst}.shift", sd[src + ".bias"] - sd[src + ".running_mean"] * scale) + + def put(dst: str, t: Tensor) -> tuple[str, Tensor]: + return (f"{p}.{dst}", t) + + fe = "feature_extractor." + yield put("fe.head.conv.weight", sd[fe + "head.0.weight"]) + yield put("fe.head.conv.bias", sd[fe + "head.0.bias"]) + yield from bn("fe.head.bn", fe + "head.1") + c_in, f_in = 32, 80 + for si, stage in enumerate(SPK_STAGES): + for k, (depth, out_ch, (s_f, s_t), expansion) in enumerate(stage): + src, dst = f"{fe}mect_block{si + 1}.{k}.", f"fe.b{si + 1}.{k}." + for r in range(depth): + cs, cd = f"{src}cnn_block.{r}.", f"{dst}cnn.{r}." + # grouped 3x3 convs, one depthwise kernel per input slot of a group + for conv in ("conv1", "conv2"): + w = sd[cs + conv + ".weight"] + for j in range(w.shape[1]): + yield put(f"{cd}{conv}.w{j}", w[:, j:j + 1]) + for i in (1, 2): + yield put(f"{cd}pw{i}.weight", sd[f"{cs}conv_pw{i}.weight"]) + yield put(f"{cd}pw{i}.bias", sd[f"{cs}conv_pw{i}.bias"]) + yield from bn(f"{cd}bn{i}", f"{cs}bn{i}") + if cs + "shortcut.0.weight" in sd: + yield put(cd + "sc.weight", sd[cs + "shortcut.0.weight"]) + yield from bn(cd + "sc.bn", cs + "shortcut.1") + if expansion > 1: + yield put(dst + "reduce.weight", sd[src + "reduce_conv.0.weight"]) + yield put(dst + "reduce.bias", sd[src + "reduce_conv.0.bias"]) + yield from bn(dst + "reduce.bn", src + "reduce_conv.1") + yield put(dst + "rproj.weight", sd[src + "reduce_project.0.weight"].squeeze(-1)) + yield put(dst + "rproj.bias", sd[src + "reduce_project.0.bias"]) + yield put(dst + "rproj.ln.weight", sd[src + "reduce_project.1.weight"]) + yield put(dst + "rproj.ln.bias", sd[src + "reduce_project.1.bias"]) + yield put(dst + "pos.dw.weight", sd[src + "pos_conv.pos_conv.0.weight"].squeeze(1)) + yield put(dst + "pos.dw.bias", sd[src + "pos_conv.pos_conv.0.bias"]) + yield from bn(dst + "pos.bn", src + "pos_conv.pos_conv.1") + yield put(dst + "pos.pw.weight", sd[src + "pos_conv.pos_conv.3.weight"].squeeze(-1)) + yield put(dst + "pos.pw.bias", sd[src + "pos_conv.pos_conv.3.bias"]) + tr = src + "transformer." + for a, b in (("q", "q_proj"), ("k", "k_proj"), ("v", "v_proj"), ("o", "out_proj")): + yield put(f"{dst}attn.{a}.weight", sd[f"{tr}attention.{b}.weight"]) + yield put(f"{dst}attn.{a}.bias", sd[f"{tr}attention.{b}.bias"]) + yield put(dst + "ln1.weight", sd[tr + "layer_norm.weight"]) + yield put(dst + "ln1.bias", sd[tr + "layer_norm.bias"]) + moe = tr + "feed_forward.intermediate_dense.token_moe." + yield put(dst + "moe.gate.weight", sd[moe + "gate.weight"]) + yield put(dst + "moe.experts.weight", sd[moe + "experts.weight"]) + yield put(dst + "ffn_out.weight", sd[tr + "feed_forward.output_dense.weight"]) + yield put(dst + "ffn_out.bias", sd[tr + "feed_forward.output_dense.bias"]) + yield put(dst + "ln2.weight", sd[tr + "final_layer_norm.weight"]) + yield put(dst + "ln2.bias", sd[tr + "final_layer_norm.bias"]) + yield put(dst + "eproj.weight", sd[src + "expansion_project.0.weight"].squeeze(-1)) + yield put(dst + "eproj.bias", sd[src + "expansion_project.0.bias"]) + c_in, f_in = c_in * s_f, f_in // s_f + # the stage outputs are mixed by softmax weights + yield put("fe.dense_w", torch.softmax(sd[fe + "dense3.inputs_weights"], dim=1)[0, :, :, 0]) + yield put("pool.l1.weight", sd["encoding_layers.linear1.weight"].squeeze(-1)) + yield put("pool.l1.bias", sd["encoding_layers.linear1.bias"]) + yield put("pool.l2.weight", sd["encoding_layers.linear2.weight"].squeeze(-1)) + yield put("pool.l2.bias", sd["encoding_layers.linear2.bias"]) + yield from bn("bn", "bn") + yield put("linear.weight", sd["linear.weight"]) + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + yield from self._speaker_encoder() + logger.info(f"gguf: loading codec decoder from '{CODEC_GENERATOR}' and '{CODEC_HEAD}'") + gen = torch.load(self.dir_model / CODEC_GENERATOR, map_location="cpu", mmap=True, weights_only=True)["generator"] + head = torch.load(self.dir_model / CODEC_HEAD, map_location="cpu", mmap=True, weights_only=True)["trainable"] + p = "a.gen.wav" + + def convnext(dst: str, src: dict[str, Tensor], prefix: str) -> Iterable[tuple[str, Tensor]]: + yield (f"{dst}.dw.weight", src[f"{prefix}.dwconv.weight"].squeeze(1)) + yield (f"{dst}.dw.bias", src[f"{prefix}.dwconv.bias"]) + yield (f"{dst}.norm.weight", src[f"{prefix}.norm.weight"]) + yield (f"{dst}.norm.bias", src[f"{prefix}.norm.bias"]) + yield (f"{dst}.pw1.weight", src[f"{prefix}.pwconv1.weight"]) + yield (f"{dst}.pw1.bias", src[f"{prefix}.pwconv1.bias"]) + yield (f"{dst}.pw2.weight", src[f"{prefix}.pwconv2.weight"]) + yield (f"{dst}.pw2.bias", src[f"{prefix}.pwconv2.bias"]) + + # RVQ codebooks folded through their out_proj, the biases are added once + def rvq(prefix: str) -> tuple[Tensor, Tensor]: + return gen[prefix + ".codebook.weight"].float() @ self._wn(gen, prefix + ".out_proj").T, gen[prefix + ".out_proj.bias"].float() + + t, b = rvq("quantizer.semantic_quantizer.quantizers.0") + yield (f"{p}.sem.table", t) + yield (f"{p}.sem.bias", b) + res = [rvq(f"quantizer.quantizer.quantizers.{i}") for i in range(self.n_codebooks - 1)] + yield (f"{p}.res.table", torch.cat([t for t, _ in res], dim=0)) + # one bias per residual codebook, a frame decoded before all its codes are known leaves the missing ones out + yield (f"{p}.res.bias", torch.stack([b for _, b in res])) + + n_sem = sum(1 for k in gen if k.startswith("semantic_head.") and k.endswith(".dwconv.weight")) + for i in range(n_sem): + yield from convnext(f"{p}.sem.{i}", gen, f"semantic_head.{i}") + yield (f"{p}.sem.out.weight", self._wn(gen, f"semantic_head.{n_sem}")) + yield (f"{p}.sem.out.bias", gen[f"semantic_head.{n_sem}.bias"]) + + names = {"norm1.weight": "norm1.weight", "norm1.bias": "norm1.bias", "qkv.weight": "qkv.weight", + "attention_out.weight": "wo.weight", "attention_scale": "attn_scale", "norm2.weight": "norm2.weight", + "norm2.bias": "norm2.bias", "ffn.0.weight": "ffn1.weight", "ffn.3.weight": "ffn2.weight", "ffn_scale": "ffn_scale"} + for si, s in enumerate(CODEC_STAGES): + yield (f"{p}.stage.{si}.in.weight", head[f"stages.{s}.input_proj.weight"]) + if f"stages.{s}.output_proj.weight" in head: + yield (f"{p}.stage.{si}.out.weight", head[f"stages.{s}.output_proj.weight"]) + n_layer = sum(1 for k in head if k.startswith(f"stages.{s}.layers.") and k.endswith(".qkv.weight")) + for il in range(n_layer): + for src, dst in names.items(): + yield (f"{p}.stage.{si}.blk.{il}.{dst}", head[f"stages.{s}.layers.{il}.{src}"]) + + n_head = sum(1 for k in head if k.startswith("head.blocks.") and k.endswith(".dwconv.weight")) + for i in range(n_head): + yield from convnext(f"{p}.head.{i}", head, f"head.blocks.{i}") + yield (f"{p}.head.norm.weight", head["head.norm.weight"]) + yield (f"{p}.head.norm.bias", head["head.norm.bias"]) + # both iSTFT branches in one projection + yield (f"{p}.head.out.weight", torch.cat([head["head.outs.0.weight"], head["head.outs.1.weight"]], dim=0)) + yield (f"{p}.head.out.bias", torch.cat([head["head.outs.0.bias"], head["head.outs.1.bias"]], dim=0)) + # read on the host + yield (f"{p}.mixer.weight", head["head.mixer.weight"].squeeze(0)) + yield (f"{p}.resample.weight", gen["to_external.kernel"].squeeze(1)) + + # "reference" speaker mode, the default of the reference: the voice is a prefix of delayed code rows + def _voices(self, tables: list[Tensor]) -> Iterable[tuple[str, Tensor]]: + n_cb, pad = self.n_codebooks, self.global_config["text_config"]["audio_pad_id"] + for path in sorted((self.dir_model / "gguf" / "voices").glob("*/*.codes.npy")): + codes = torch.from_numpy(np.load(path, allow_pickle=False).astype(np.int64)) + if codes.ndim != 2 or codes.shape[1] != n_cb or codes.min() < 0 or codes.max() >= pad - 1: + logger.warning(f"voice {path.name}: skipped, expected [frames, {n_cb}] codes") + continue + codes = codes[:VOICE_MAX_FRAMES] + n_frames = codes.shape[0] + # codebook cb of frame t is in row t + cb, the other cells are pad + rows = torch.full((n_frames + n_cb - 1, n_cb), pad, dtype=torch.int64) + for cb in range(n_cb): + rows[cb:cb + n_frames, cb] = codes[:, cb] + embd = sum(tables[cb][rows[:, cb]] for cb in range(n_cb)) + yield (f"voice.{path.name[:-len('.codes.npy')]}", embd) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + T = gguf.MODEL_TENSOR + + if name.startswith(("a.gen.wav.", "a.spk.")): + yield (name, data_torch) + return + + # the speaker encoder ends with the speaker projection of the backbone + if name.startswith("speaker_projection."): + yield ("a.spk.proj." + name.split(".")[-1], data_torch) + return + + if name == "multi_output.weight": + yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), data_torch) + return + + assert name.startswith("multi_embedder.") + self._embd[name] = data_torch + if len(self._embd) < self.n_codebooks + 1: + return + # every row built by the mmproj has text id = text_vocab, so that row is folded into codebook 0 + tables = [self._embd[f"multi_embedder.embedders.{cb}.weight"].float() for cb in range(self.n_codebooks)] + tables[0] = tables[0] + self._embd[f"multi_embedder.embedders.{self.n_codebooks}.weight"][self.text_vocab].float() + yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), torch.cat(tables, dim=0)) + yield from self._voices([LazyTorchTensor.to_eager(t) for t in tables]) + self._embd.clear() diff --git a/docs/multimodal/tts.md b/docs/multimodal/tts.md index 3ffc6d45e..1dca6c461 100644 --- a/docs/multimodal/tts.md +++ b/docs/multimodal/tts.md @@ -11,6 +11,7 @@ Supported models: - [LFM3-Audio](#lfm3-audio) - [Chatterbox](#chatterbox) - [Drama 3](#drama-3) +- [ZONOS3](#zonos3) The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)). @@ -180,3 +181,37 @@ The routed experts of the checkpoint are FP8, they are dequantized at conversion python convert_hf_to_gguf.py path/to/drama-3 --outtype q8_0 --outfile drama3.gguf python convert_hf_to_gguf.py path/to/drama-3 --mmproj --outtype bf16 --outfile mmproj-drama3.gguf ``` + +## ZONOS3 + +[Zyphra ZONOS3 Mini](https://huggingface.co/Zyphra/ZONOS3-Mini): a hybrid Gated DeltaNet / attention MoE backbone (800M, 100M active) predicts the 10 codebooks of each frame with an inter-codebook delay, and the ZAC codec decodes them to 44.1 kHz audio. + +Available params: +- `--tts-voice` picks one of the predefined voices of the release (`Savannah_en-American_F_1` by default); `llama-server` lists them in `GET /props`. A voice is the prompt prefix of its reference codes, the default speaker mode of the reference +- The language is the one of the text; `--tts-lang` (`lang` in `llama-server`) only selects the text normalization +- Numbers are spelled out by the `tts` chat template of the GGUF: cardinals, decimals, money, percents, clock times, common ordinal forms (`1st`, `1er`, `1º`), English and German years. A 4-digit number is read as a year only after a word like `in`, `since`, a month name, or `im Jahr`, `seit` in German (`in 1990` -> "in nineteen ninety", `1234 people` -> "one thousand two hundred thirty-four people"). Languages: `en`, `fr`, `de`, `es`, `it`, `pt`, `ja` and `zh` (or `cmn`), the region is ignored (`fr_fr`, `pt_br`); English when the language is empty, other languages are left as written. Set the language to the one of the text: the separators differ, e.g. `1.234` is a thousand in German +- Note: the reference uses NeMo text normalization, which also covers units, dates, abbreviations and Korean; supply normalized text for them +- `--tts-speaker-file` clones the voice of a clip (the first 60 s are used) with the speaker encoder of the release, the "embedding" speaker mode of the reference +- `llama-server` also takes `speed` (relative to 15 UTF-8 bytes per second) and `loudness` (LUFS), see the `/tts` endpoint; they pick the speaking-rate and loudness condition buckets of the checkpoint, the defaults leave them unknown +- Note: `loudness` has a strong effect with a cloned voice; a predefined voice mostly keeps the loudness of its reference clip +- Note: `--top-k` and `--top-p` are not used, frames are sampled with the settings of the reference: temperature 0.75 (`--temp` changes it), top-k 80, top-p 0.69, repetition penalty 2.4 over the last 96 frames of codebooks 0 to 5 +- Each frame is 2940 samples (about 67 ms), generation stops at the end-of-audio code + +Example usage: + +```sh +llama-tts -m zonos3-mini.gguf \ + -mm mmproj-zonos3-mini.gguf \ + -p "The old lighthouse keeper climbed the spiral stairs every evening." \ + --tts-voice Imogen_en-British_F_1 \ + --output out.wav +``` + +**Note for GGUF conversion:** + +Convert the root of the release repository, not the `lm/` subdirectory. The mmproj holds the codebook heads, the predefined voices, the speaker encoder and the `keep2_plain` codec decoder, it is best kept in F32: + +```sh +python convert_hf_to_gguf.py path/to/ZONOS3-Mini --outtype q8_0 --outfile zonos3-mini.gguf +python convert_hf_to_gguf.py path/to/ZONOS3-Mini --mmproj --outtype f32 --outfile mmproj-zonos3-mini.gguf +``` diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index ed3ade939..c9b27a11f 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -545,6 +545,7 @@ class Keys: PROJECTOR_TYPE = "clip.gen.audio.projector_type" # for mixed modality models # name of the weight variant, for settings that are not in the checkpoint MODEL_VARIANT = "clip.gen.audio.model_variant" + DEFAULT_VOICE = "clip.gen.audio.default_voice" EMBEDDING_LENGTH = "clip.gen.audio.embedding_length" FEED_FORWARD_LENGTH = "clip.gen.audio.feed_forward_length" BLOCK_COUNT = "clip.gen.audio.block_count" @@ -765,6 +766,7 @@ class MODEL_ARCH(IntEnum): NANBEIGE = auto() QWEN3TTS = auto() POCKETTTS = auto() + ZONOS3 = auto() K2HORIZON = auto() KOLIBRI1 = auto() LIMITE = auto() @@ -1703,6 +1705,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.NANBEIGE: "nanbeige", MODEL_ARCH.QWEN3TTS: "qwen3tts", MODEL_ARCH.POCKETTTS: "pockettts", + MODEL_ARCH.ZONOS3: "zonos3", MODEL_ARCH.K2HORIZON: "k2-horizon", MODEL_ARCH.KOLIBRI1: "kolibri1", MODEL_ARCH.LIMITE: "limite", @@ -6722,6 +6725,39 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.ZONOS3: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT_RES_SCORE, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_RES_SCORE, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_GATE, + MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.SSM_A, + MODEL_TENSOR.SSM_CONV1D, + MODEL_TENSOR.SSM_DT, + MODEL_TENSOR.SSM_NORM, + MODEL_TENSOR.SSM_BETA, + MODEL_TENSOR.SSM_ALPHA, + MODEL_TENSOR.SSM_OUT, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_RES_SCORE, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.MOE_LATENT_DOWN, + MODEL_TENSOR.MOE_LATENT_UP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + ], MODEL_ARCH.K2HORIZON: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, @@ -7156,6 +7192,8 @@ class VisionProjectorType: CHATTERBOX = "chatterbox" # audio generation: s3gen flow matching + HiFT vocoder SOPRANO = "soprano" # audio generation: Vocos decoder DRAMA3_GEN = "drama3_gen" # audio generation: codebook tower + Fish DAC decoder + ZONOS3_GEN = "zonos3_gen" # audio generation: 10 codebook heads + ZAC decoder + ZONOS3_SPKENC = "zonos3_spkenc" # audio: MECT-B2 speaker encoder HUNYUANVL = "hunyuanvl" PARAKEET = "parakeet" # audio MINIMAXM3 = "minimax_m3" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 6ca757080..edbe8cd09 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1812,6 +1812,9 @@ class GGUFWriter: def add_gen_audio_model_variant(self, value: str) -> None: self.add_string(Keys.ClipGenAudio.MODEL_VARIANT, value) + def add_gen_audio_default_voice(self, value: str) -> None: + self.add_string(Keys.ClipGenAudio.DEFAULT_VOICE, value) + def add_xielu_alpha_p(self, values: Sequence[float]): self.add_array(Keys.xIELU.ALPHA_P, values) diff --git a/models/templates/Zyphra-ZONOS3-Mini-tts.jinja b/models/templates/Zyphra-ZONOS3-Mini-tts.jinja new file mode 100644 index 000000000..ec1255862 --- /dev/null +++ b/models/templates/Zyphra-ZONOS3-Mini-tts.jinja @@ -0,0 +1,627 @@ +{#- ZONOS3 "tts" template: the text to speak (first message), normalized; the helper adds BOS and EOS itself -#} +{#- numbers, decimals, money, percents, ordinals, years and clock times are spelled out in en, fr, de, es, it, pt, ja and zh -#} +{#- other languages pass through; patterns: tts_norm_scan() / tts_norm_sub(), see docs/tts-norm-patterns.md -#} +{#- the ja / zh conventions follow the NeMo grammars of the reference (e.g. zh: 两千, 二百, 三百一十六, 二零零二年) -#} +{#- optional identifies this template, so that llama.cpp can detect it and apply quick fixes to GGUFs already converted -#} +{%- set template_name = "zonos3-mini" -%} + +{#- English -#} +{%- set ones = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", + "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen"] -%} +{%- set tens = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"] -%} +{%- set scales = ["", " thousand", " million", " billion", " trillion"] -%} +{%- set irregular = {"one": "first", "two": "second", "three": "third", "five": "fifth", "eight": "eighth", "nine": "ninth", "twelve": "twelfth"} -%} + +{%- macro under100(n) -%} +{{- ones[n] if n < 20 else tens[(n / 10)|int] ~ ("-" ~ ones[n % 10] if n % 10 else "") -}} +{%- endmacro -%} + +{%- macro under1000(n) -%} +{{- (ones[(n / 100)|int] ~ " hundred" ~ (" " ~ under100(n % 100) if n % 100 else "")) if n >= 100 else under100(n) -}} +{%- endmacro -%} + +{#- all the *_card macros take a digit string of up to 15 digits -#} +{%- macro en_card(s) -%} +{%- set p = ["", "00", "0"][(s|length) % 3] ~ s -%} +{%- set ns = namespace(out="", ng=((p|length) / 3)|int) -%} +{%- for g in range(ns.ng) -%} +{%- set v = p[g * 3:g * 3 + 3]|int -%} +{%- if v > 0 -%} +{%- set ns.out = ns.out ~ (" " if ns.out else "") ~ under1000(v) ~ scales[ns.ng - 1 - g] -%} +{%- endif -%} +{%- endfor -%} +{{- ns.out if ns.out else "zero" -}} +{%- endmacro -%} + +{#- the last word of a cardinal becomes ordinal: twenty-one -> twenty-first -#} +{%- macro en_ord(w) -%} +{%- set last = (w.split(" ")|last).split("-")|last -%} +{%- set head = w[0:(w|length) - (last|length)] -%} +{{- head ~ (irregular[last] if last in irregular else (last[0:(last|length) - 1] ~ "ieth" if last.endswith("y") else last ~ "th")) -}} +{%- endmacro -%} + +{#- French: traditional spelling, quatre-vingts / deux cents take the plural s only at the end -#} +{%- set fr_u = ["zéro", "un", "deux", "trois", "quatre", "cinq", "six", "sept", "huit", "neuf", "dix", "onze", "douze", "treize", + "quatorze", "quinze", "seize"] -%} +{%- set fr_t = ["", "", "vingt", "trente", "quarante", "cinquante", "soixante", "", "quatre-vingt", ""] -%} +{%- set fr_big = [[], [], ["million", "millions"], ["milliard", "milliards"], ["billion", "billions"]] -%} + +{%- macro fr99(n, final) -%} +{%- if n < 17 -%} +{{- fr_u[n] -}} +{%- elif n < 20 -%} +{{- "dix-" ~ fr_u[n - 10] -}} +{%- else -%} +{%- set t = (n / 10)|int -%} +{%- set u = n % 10 -%} +{%- if t == 7 or t == 9 -%} +{{- ("soixante" if t == 7 else "quatre-vingt") ~ (" et onze" if t == 7 and u == 1 else "-" ~ fr99(n - (60 if t == 7 else 80), final)) -}} +{%- else -%} +{{- fr_t[t] ~ ("s" if t == 8 and u == 0 and final else "") ~ (" et un" if u == 1 and t != 8 else ("-" ~ fr_u[u] if u else "")) -}} +{%- endif -%} +{%- endif -%} +{%- endmacro -%} + +{%- macro fr999(n, final) -%} +{%- set h = (n / 100)|int -%} +{%- set r = n % 100 -%} +{%- set hs = "" if h == 0 else ("cent" if h == 1 else fr_u[h] ~ " cent" ~ ("s" if r == 0 and final else "")) -%} +{{- hs ~ (" " if hs and r else "") ~ (fr99(r, final) if r else "") -}} +{%- endmacro -%} + +{%- macro fr_card(s) -%} +{%- set p = ["", "00", "0"][(s|length) % 3] ~ s -%} +{%- set ns = namespace(out="", ng=((p|length) / 3)|int) -%} +{%- for g in range(ns.ng) -%} +{%- set v = p[g * 3:g * 3 + 3]|int -%} +{%- set k = ns.ng - 1 - g -%} +{%- if v > 0 -%} +{%- if k == 0 -%} +{%- set w = fr999(v, true) -%} +{%- elif k == 1 -%} +{%- set w = ("" if v == 1 else fr999(v, false) ~ " ") ~ "mille" -%} +{%- else -%} +{%- set w = fr999(v, true) ~ " " ~ fr_big[k][0 if v == 1 else 1] -%} +{%- endif -%} +{%- set ns.out = ns.out ~ (" " if ns.out else "") ~ w -%} +{%- endif -%} +{%- endfor -%} +{{- ns.out if ns.out else "zéro" -}} +{%- endmacro -%} + +{%- macro fr_ord(w) -%} +{%- set last = (w.split(" ")|last).split("-")|last -%} +{%- set head = w[0:(w|length) - (last|length)] -%} +{%- set base = last[0:(last|length) - 1] if last in ["vingts", "cents", "millions", "milliards", "billions"] else last -%} +{%- set map = {"cinq": "cinquième", "neuf": "neuvième"} -%} +{{- head ~ (map[base] if base in map else (base[0:(base|length) - 1] ~ "ième" if base.endswith("e") else base ~ "ième")) -}} +{%- endmacro -%} + +{#- German: below a million one word, eins / ein / eine -#} +{%- set de_u = ["null", "eins", "zwei", "drei", "vier", "fünf", "sechs", "sieben", "acht", "neun", "zehn", "elf", "zwölf", + "dreizehn", "vierzehn", "fünfzehn", "sechzehn", "siebzehn", "achtzehn", "neunzehn"] -%} +{%- set de_t = ["", "", "zwanzig", "dreißig", "vierzig", "fünfzig", "sechzig", "siebzig", "achtzig", "neunzig"] -%} +{%- set de_big = [[], [], ["Million", "Millionen"], ["Milliarde", "Milliarden"], ["Billion", "Billionen"]] -%} + +{%- macro de99(n) -%} +{{- de_u[n] if n < 20 else ((("ein" if n % 10 == 1 else de_u[n % 10]) ~ "und") if n % 10 else "") ~ de_t[(n / 10)|int] -}} +{%- endmacro -%} + +{%- macro de999(n) -%} +{%- set h = (n / 100)|int -%} +{%- set r = n % 100 -%} +{{- (((("ein" if h == 1 else de_u[h]) ~ "hundert")) if h else "") ~ (de99(r) if r else "") -}} +{%- endmacro -%} + +{#- attributive form: eins -> ein -#} +{%- macro de_attr(w) -%} +{{- w[0:(w|length) - 1] if w.endswith("eins") else w -}} +{%- endmacro -%} + +{%- macro de_card(s) -%} +{%- set p = ["", "00", "0"][(s|length) % 3] ~ s -%} +{%- set ns = namespace(big="", small="", ng=((p|length) / 3)|int) -%} +{%- for g in range(ns.ng) -%} +{%- set v = p[g * 3:g * 3 + 3]|int -%} +{%- set k = ns.ng - 1 - g -%} +{%- if v > 0 -%} +{%- if k == 0 -%} +{%- set ns.small = ns.small ~ de999(v) -%} +{%- elif k == 1 -%} +{%- set ns.small = ns.small ~ de_attr(de999(v)) ~ "tausend" -%} +{%- else -%} +{%- set ns.big = ns.big ~ (" " if ns.big else "") ~ ("eine" if v == 1 else de_attr(de999(v))) ~ " " ~ de_big[k][0 if v == 1 else 1] -%} +{%- endif -%} +{%- endif -%} +{%- endfor -%} +{%- set out = ns.big ~ (" " if ns.big and ns.small else "") ~ ns.small -%} +{{- out if out else "null" -}} +{%- endmacro -%} + +{#- join the non-empty parts with a space -#} +{%- macro join4(a, b, c, d) -%} +{%- set ns = namespace(out="") -%} +{%- for x in [a, b, c, d] -%} +{%- if x -%} +{%- set ns.out = ns.out ~ (" " if ns.out else "") ~ x -%} +{%- endif -%} +{%- endfor -%} +{{- ns.out -}} +{%- endmacro -%} + +{#- Spanish: long scale (mil millones, billón), un / veintiún before a noun or mil -#} +{%- set es_u = ["cero", "uno", "dos", "tres", "cuatro", "cinco", "seis", "siete", "ocho", "nueve", "diez", "once", "doce", + "trece", "catorce", "quince", "dieciséis", "diecisiete", "dieciocho", "diecinueve", "veinte", "veintiuno", + "veintidós", "veintitrés", "veinticuatro", "veinticinco", "veintiséis", "veintisiete", "veintiocho", "veintinueve"] -%} +{%- set es_t = ["", "", "", "treinta", "cuarenta", "cincuenta", "sesenta", "setenta", "ochenta", "noventa"] -%} +{%- set es_h = ["", "ciento", "doscientos", "trescientos", "cuatrocientos", "quinientos", "seiscientos", "setecientos", + "ochocientos", "novecientos"] -%} + +{%- macro es99(n) -%} +{{- es_u[n] if n < 30 else es_t[(n / 10)|int] ~ (" y " ~ es_u[n % 10] if n % 10 else "") -}} +{%- endmacro -%} + +{%- macro es999(n) -%} +{%- if n == 100 -%} +{{- "cien" -}} +{%- else -%} +{%- set h = (n / 100)|int -%} +{%- set r = n % 100 -%} +{{- es_h[h] ~ (" " if h and r else "") ~ (es99(r) if r else "") -}} +{%- endif -%} +{%- endmacro -%} + +{%- macro es_apoc(w) -%} +{{- (w[0:(w|length) - 9] ~ "veintiún") if w.endswith("veintiuno") else ((w[0:(w|length) - 3] ~ "un") if w.endswith("uno") else w) -}} +{%- endmacro -%} + +{%- macro es6(n) -%} +{%- set a = (n / 1000)|int -%} +{%- set b = n % 1000 -%} +{%- set hi = "" if a == 0 else ("mil" if a == 1 else es_apoc(es999(a)) ~ " mil") -%} +{{- hi ~ (" " if hi and b else "") ~ (es999(b) if b else "") -}} +{%- endmacro -%} + +{%- macro es_card(s) -%} +{%- set p = ("000000000000000000" ~ s)[-18:] -%} +{%- set top = p[0:6]|int -%} +{%- set mid = p[6:12]|int -%} +{%- set low = p[12:18]|int -%} +{%- set a = "" if top == 0 else ("un billón" if top == 1 else es_apoc(es6(top)) ~ " billones") -%} +{%- set b = "" if mid == 0 else ("un millón" if mid == 1 else es_apoc(es6(mid)) ~ " millones") -%} +{%- set out = join4(a, b, es6(low) if low else "", "") -%} +{{- out if out else "cero" -}} +{%- endmacro -%} + +{#- Italian: below a million one word, ventitré, centottanta -#} +{%- set it_u = ["zero", "uno", "due", "tre", "quattro", "cinque", "sei", "sette", "otto", "nove", "dieci", "undici", "dodici", + "tredici", "quattordici", "quindici", "sedici", "diciassette", "diciotto", "diciannove"] -%} +{%- set it_t = ["", "", "venti", "trenta", "quaranta", "cinquanta", "sessanta", "settanta", "ottanta", "novanta"] -%} + +{%- macro it99(n) -%} +{%- if n < 20 -%} +{{- it_u[n] -}} +{%- else -%} +{%- set t = it_t[(n / 10)|int] -%} +{%- set u = n % 10 -%} +{{- (t[0:(t|length) - 1] if u == 1 or u == 8 else t) ~ (it_u[u] if u else "") -}} +{%- endif -%} +{%- endmacro -%} + +{%- macro it999(n) -%} +{%- set h = (n / 100)|int -%} +{%- set r = n % 100 -%} +{%- set hs = "" if h == 0 else ("cento" if h == 1 else it_u[h] ~ "cento") -%} +{{- (hs[0:(hs|length) - 1] if (r / 10)|int == 8 else hs) ~ (it99(r) if r else "") -}} +{%- endmacro -%} + +{#- without the final accent, see it_acc -#} +{%- macro it6(n) -%} +{%- set a = (n / 1000)|int -%} +{%- set b = n % 1000 -%} +{{- ("" if a == 0 else ("mille" if a == 1 else it999(a) ~ "mila")) ~ (it999(b) if b else "") -}} +{%- endmacro -%} + +{%- macro it_acc(w) -%} +{{- (w[0:(w|length) - 3] ~ "tré") if w.endswith("tre") and w != "tre" else w -}} +{%- endmacro -%} + +{%- macro it_card(s) -%} +{%- set p = ("000000000000000000" ~ s)[-18:] -%} +{%- set bil = p[0:6]|int -%} +{%- set mld = p[6:9]|int -%} +{%- set mil = p[9:12]|int -%} +{%- set low = p[12:18]|int -%} +{%- set a = "" if bil == 0 else ("un bilione" if bil == 1 else it_acc(it6(bil)) ~ " bilioni") -%} +{%- set b = "" if mld == 0 else ("un miliardo" if mld == 1 else it_acc(it999(mld)) ~ " miliardi") -%} +{%- set c = "" if mil == 0 else ("un milione" if mil == 1 else it_acc(it999(mil)) ~ " milioni") -%} +{%- set out = join4(a, b, c, it_acc(it6(low)) if low else "") -%} +{{- out if out else "zero" -}} +{%- endmacro -%} + +{#- Portuguese (Brazil): short scale, "e" before the last group when it is below 100 or a round hundred -#} +{%- set pt_u = ["zero", "um", "dois", "três", "quatro", "cinco", "seis", "sete", "oito", "nove", "dez", "onze", "doze", + "treze", "catorze", "quinze", "dezesseis", "dezessete", "dezoito", "dezenove"] -%} +{%- set pt_t = ["", "", "vinte", "trinta", "quarenta", "cinquenta", "sessenta", "setenta", "oitenta", "noventa"] -%} +{%- set pt_h = ["", "cento", "duzentos", "trezentos", "quatrocentos", "quinhentos", "seiscentos", "setecentos", "oitocentos", + "novecentos"] -%} +{%- set pt_big = [[], [], ["milhão", "milhões"], ["bilhão", "bilhões"], ["trilhão", "trilhões"]] -%} + +{%- macro pt99(n) -%} +{{- pt_u[n] if n < 20 else pt_t[(n / 10)|int] ~ (" e " ~ pt_u[n % 10] if n % 10 else "") -}} +{%- endmacro -%} + +{%- macro pt999(n) -%} +{%- if n == 100 -%} +{{- "cem" -}} +{%- else -%} +{%- set h = (n / 100)|int -%} +{%- set r = n % 100 -%} +{{- pt_h[h] ~ (" e " if h and r else "") ~ (pt99(r) if r else "") -}} +{%- endif -%} +{%- endmacro -%} + +{%- macro pt_card(s) -%} +{%- set p = ["", "00", "0"][(s|length) % 3] ~ s -%} +{%- set ns = namespace(out="", ng=((p|length) / 3)|int, last=-1) -%} +{%- for g in range(ns.ng) -%} +{%- if p[g * 3:g * 3 + 3]|int > 0 -%} +{%- set ns.last = g -%} +{%- endif -%} +{%- endfor -%} +{%- for g in range(ns.ng) -%} +{%- set v = p[g * 3:g * 3 + 3]|int -%} +{%- set k = ns.ng - 1 - g -%} +{%- if v > 0 -%} +{%- if k == 0 -%} +{%- set w = pt999(v) -%} +{%- elif k == 1 -%} +{%- set w = "mil" if v == 1 else pt999(v) ~ " mil" -%} +{%- else -%} +{%- set w = ("um" if v == 1 else pt999(v)) ~ " " ~ pt_big[k][0 if v == 1 else 1] -%} +{%- endif -%} +{%- set sep = " e " if g == ns.last and (v < 100 or v % 100 == 0) else " " -%} +{%- set ns.out = ns.out ~ (sep if ns.out else "") ~ w -%} +{%- endif -%} +{%- endfor -%} +{{- ns.out if ns.out else "zero" -}} +{%- endmacro -%} + +{#- Japanese: 万 / 億 / 兆 groups, no 一 before 十 / 百 / 千 (but 一千万), zero digits are skipped -#} +{%- set cjk_d = ["零", "一", "二", "三", "四", "五", "六", "七", "八", "九"] -%} + +{%- macro ja4(n, big) -%} +{%- set th = (n / 1000)|int -%} +{%- set h = ((n / 100)|int) % 10 -%} +{%- set t = ((n / 10)|int) % 10 -%} +{%- set u = n % 10 -%} +{{- ((("一" if big else "") if th == 1 else cjk_d[th]) ~ "千" if th else "") + ~ (("" if h == 1 else cjk_d[h]) ~ "百" if h else "") + ~ (("" if t == 1 else cjk_d[t]) ~ "十" if t else "") + ~ (cjk_d[u] if u else "") -}} +{%- endmacro -%} + +{%- macro ja_card(s) -%} +{%- set p = ("0000000000000000" ~ s)[-16:] -%} +{%- set ns = namespace(out="") -%} +{%- for g in range(4) -%} +{%- set v = p[g * 4:g * 4 + 4]|int -%} +{%- if v > 0 -%} +{%- set ns.out = ns.out ~ ja4(v, g < 3) ~ ["兆", "億", "万", ""][g] -%} +{%- endif -%} +{%- endfor -%} +{{- ns.out if ns.out else "零" -}} +{%- endmacro -%} + +{#- Chinese: 万 / 亿 groups, 两 before 千 / 万 / 亿, 二 before 百, 十 at the head but 一十 inside, one 零 per gap -#} +{%- macro zh4(n, head) -%} +{%- set ds = [(n / 1000)|int, ((n / 100)|int) % 10, ((n / 10)|int) % 10, n % 10] -%} +{%- set ns = namespace(out="", zero=false) -%} +{%- for i in range(4) -%} +{%- set d = ds[i] -%} +{%- if d == 0 -%} +{%- if ns.out -%} +{%- set ns.zero = true -%} +{%- endif -%} +{%- else -%} +{%- set w = "两" if d == 2 and i == 0 else cjk_d[d] -%} +{%- if i == 2 and d == 1 and head and not ns.out -%} +{%- set w = "" -%} +{%- endif -%} +{%- set ns.out = ns.out ~ ("零" if ns.zero else "") ~ w ~ ["千", "百", "十", ""][i] -%} +{%- set ns.zero = false -%} +{%- endif -%} +{%- endfor -%} +{{- ns.out -}} +{%- endmacro -%} + +{%- macro zh_card(s) -%} +{%- set p = ("0000000000000000" ~ s)[-16:] -%} +{%- set ns = namespace(out="", gap=false) -%} +{%- for g in range(4) -%} +{%- set v = p[g * 4:g * 4 + 4]|int -%} +{%- if v == 0 -%} +{%- if ns.out -%} +{%- set ns.gap = true -%} +{%- endif -%} +{%- else -%} +{%- set w = "两" if v == 2 and g < 3 else zh4(v, not ns.out) -%} +{%- set ns.out = ns.out ~ ("零" if ns.out and (ns.gap or v < 1000) else "") ~ w ~ ["万亿", "亿", "万", ""][g] -%} +{%- set ns.gap = false -%} +{%- endif -%} +{%- endfor -%} +{{- ns.out if ns.out else "零" -}} +{%- endmacro -%} + +{#- per-language words -#} +{%- set digit_words = {"en": ones[0:10], "fr": fr_u[0:10], "de": de_u[0:10], "es": es_u[0:10], "it": it_u[0:10], "pt": pt_u[0:10], + "ja": cjk_d, "zh": cjk_d} -%} +{%- set W = { + "en": {"point": " point ", "minus": "minus ", "and": " and ", "percent": " percent", "sep": " "}, + "fr": {"point": " virgule ", "minus": "moins ", "and": " et ", "percent": " pour cent", "sep": " "}, + "de": {"point": " Komma ", "minus": "minus ", "and": " und ", "percent": " Prozent", "sep": " "}, + "es": {"point": " coma ", "minus": "menos ", "and": " con ", "percent": " por ciento", "sep": " "}, + "it": {"point": " virgola ", "minus": "meno ", "and": " e ", "percent": " per cento", "sep": " "}, + "pt": {"point": " vírgula ", "minus": "menos ", "and": " e ", "percent": " por cento", "sep": " "}, + "ja": {"point": "点", "minus": "マイナス", "and": "", "percent": "パーセント", "sep": ""}, + "zh": {"point": "点", "minus": "负", "and": "", "percent": "", "sep": ""}, +} -%} +{#- currency -> [singular, plural, cent singular, cent plural]; ja / zh: the unit after the number -#} +{%- set M = { + "en": {"$": ["dollar", "dollars", "cent", "cents"], "€": ["euro", "euros", "cent", "cents"], + "£": ["pound", "pounds", "penny", "pence"], "¥": ["yen", "yen", "sen", "sen"]}, + "fr": {"$": ["dollar", "dollars", "cent", "cents"], "€": ["euro", "euros", "centime", "centimes"]}, + "de": {"$": ["Dollar", "Dollar", "Cent", "Cent"], "€": ["Euro", "Euro", "Cent", "Cent"], "£": ["Pfund", "Pfund", "Penny", "Pence"]}, + "es": {"$": ["dólar", "dólares", "centavo", "centavos"], "€": ["euro", "euros", "céntimo", "céntimos"]}, + "it": {"$": ["dollaro", "dollari", "centesimo", "centesimi"], "€": ["euro", "euro", "centesimo", "centesimi"]}, + "pt": {"$": ["dólar", "dólares", "centavo", "centavos"], "€": ["euro", "euros", "cêntimo", "cêntimos"], + "R$": ["real", "reais", "centavo", "centavos"]}, + "ja": {"$": "ドル", "€": "ユーロ", "£": "ポンド", "¥": "円"}, + "zh": {"$": "美元", "€": "欧元", "£": "英镑", "¥": "元"}, +} -%} +{%- set es_ord = ["", "primero", "segundo", "tercero", "cuarto", "quinto", "sexto", "séptimo", "octavo", "noveno", "décimo"] -%} +{%- set it_ord = ["", "primo", "secondo", "terzo", "quarto", "quinto", "sesto", "settimo", "ottavo", "nono", "decimo"] -%} +{%- set pt_ord = ["", "primeiro", "segundo", "terceiro", "quarto", "quinto", "sexto", "sétimo", "oitavo", "nono", "décimo"] -%} + +{%- macro card(L, s) -%} +{%- if L == "fr" -%}{{- fr_card(s) -}} +{%- elif L == "de" -%}{{- de_card(s) -}} +{%- elif L == "es" -%}{{- es_card(s) -}} +{%- elif L == "it" -%}{{- it_card(s) -}} +{%- elif L == "pt" -%}{{- pt_card(s) -}} +{%- elif L == "ja" -%}{{- ja_card(s) -}} +{%- elif L == "zh" -%}{{- zh_card(s) -}} +{%- else -%}{{- en_card(s) -}} +{%- endif -%} +{%- endmacro -%} + +{#- attributive form before a noun: ein Euro, un euro, veintiún euros -#} +{%- macro attr(L, w) -%} +{%- if L == "de" -%}{{- de_attr(w) -}} +{%- elif L == "es" -%}{{- es_apoc(w) -}} +{%- elif L == "it" and w == "uno" -%}{{- "un" -}} +{%- else -%}{{- w -}} +{%- endif -%} +{%- endmacro -%} + +{%- macro spell(L, s) -%} +{%- set ns = namespace(out="") -%} +{%- for i in range(s|length) -%} +{%- set ns.out = ns.out ~ (W[L]["sep"] if i > 0 else "") ~ digit_words[L][s[i:i + 1]|int] -%} +{%- endfor -%} +{{- ns.out -}} +{%- endmacro -%} + +{#- decimals: fr / es / it / pt read a short fraction as a number (virgule quatorze), the others digit by digit -#} +{%- macro fraction(L, f) -%} +{%- if L in ["fr", "es", "it", "pt"] and f|length <= 3 -%} +{%- set z = tts_norm_sub(f, "^(0*).*", "%1") -%} +{%- set rest = f[z|length:] -%} +{{- spell(L, z) ~ (" " if z and rest else "") ~ (card(L, rest) if rest else "") -}} +{%- else -%} +{{- spell(L, f) -}} +{%- endif -%} +{%- endmacro -%} + +{#- en: 1990 -> nineteen ninety, 1905 -> nineteen oh five, 2005 -> two thousand five; de: 1990 -> neunzehnhundertneunzig -#} +{%- macro year(L, s) -%} +{%- set hi = s[0:2]|int -%} +{%- set lo = s[2:4]|int -%} +{%- if L == "de" -%} +{{- (de99(hi) ~ "hundert" ~ (de99(lo) if lo else "")) if hi < 20 else de_card(s) -}} +{%- elif (hi == 20 and lo < 10) or (hi % 10 == 0 and lo == 0) -%} +{{- en_card(s) -}} +{%- elif lo == 0 -%} +{{- under100(hi) ~ " hundred" -}} +{%- else -%} +{{- under100(hi) ~ (" oh " ~ ones[lo] if lo < 10 else " " ~ under100(lo)) -}} +{%- endif -%} +{%- endmacro -%} + +{#- ip: digits with optional thousands separators, frac: decimal digits or empty -#} +{%- macro number(L, ip, frac, year_ok) -%} +{%- set d = tts_norm_sub(ip, "%D", "") -%} +{%- if d|length > 15 or (d|length > 1 and d[0:1] == "0" and not frac) -%} +{{- spell(L, d) ~ (W[L]["point"] ~ spell(L, frac) if frac else "") -}} +{%- elif frac -%} +{{- card(L, d) ~ W[L]["point"] ~ fraction(L, frac) -}} +{%- elif year_ok and d == ip and d|length == 4 and d|int >= 1100 and d|int <= 2099 and L in ["en", "de"] -%} +{{- year(L, d) -}} +{%- else -%} +{{- card(L, d) -}} +{%- endif -%} +{%- endmacro -%} + +{#- fr / es / it / pt: million and above are nouns, a round amount takes "de" before the currency (deux millions d'euros) -#} +{%- macro unit_sep(L, ip, unit) -%} +{%- set d = tts_norm_sub(ip, "%D", "") -%} +{%- if L in ["fr", "es", "it", "pt"] and d|length > 6 and d[(d|length) - 6:] == "000000" -%} +{{- " d'" if L == "fr" and unit[0:1] in "aeiou" else " " ~ ("di" if L == "it" else "de") ~ " " -}} +{%- else -%} +{{- " " -}} +{%- endif -%} +{%- endmacro -%} + +{%- macro money(L, cur, ip, cents, scale) -%} +{%- if cur not in M[L] -%} +{{- cur ~ W[L]["sep"] ~ number(L, ip, cents, false) -}} +{%- elif L in ["ja", "zh"] -%} +{{- number(L, ip, cents, false) ~ M[L][cur] -}} +{%- else -%} +{%- set u = M[L][cur] -%} +{%- if scale -%} +{{- number(L, ip, cents, false) ~ " " ~ scale ~ " " ~ u[1] -}} +{%- elif cents|length == 2 -%} +{%- set c = attr(L, card(L, cents)) ~ " " ~ u[2 if cents|int == 1 else 3] -%} +{%- if tts_norm_sub(ip, "%D", "")|int == 0 -%} +{{- c -}} +{%- else -%} +{{- attr(L, number(L, ip, "", false)) ~ unit_sep(L, ip, u[1]) ~ u[0 if ip == "1" else 1] ~ (W[L]["and"] ~ c if cents|int > 0 else "") -}} +{%- endif -%} +{%- elif cents -%} +{{- number(L, ip, cents, false) ~ " " ~ u[1] -}} +{%- else -%} +{{- attr(L, number(L, ip, "", false)) ~ unit_sep(L, ip, u[1]) ~ u[0 if ip == "1" else 1] -}} +{%- endif -%} +{%- endif -%} +{%- endmacro -%} + +{%- macro clock(L, h, m) -%} +{%- set hn = h|int -%} +{%- set mm = m|int if m else 0 -%} +{%- if L == "fr" -%} +{%- set w = fr_card(h) -%} +{{- w ~ ("e" if w.endswith("un") else "") ~ (" heure" if hn <= 1 else " heures") ~ (" " ~ fr_card(m) if mm else "") -}} +{%- elif L == "de" -%} +{{- de_attr(de_card(h)) ~ " Uhr" ~ (" " ~ de_card(m) if mm else "") -}} +{%- elif L == "es" -%} +{%- set w = es_card(h) -%} +{{- (w[0:(w|length) - 1] ~ "a" if w.endswith("uno") else w) ~ (" y " ~ es_card(m) if mm else " en punto") -}} +{%- elif L == "it" -%} +{{- ("una" if hn == 1 else it_card(h)) ~ (" e " ~ it_card(m) if mm else "") -}} +{%- elif L == "pt" -%} +{%- set w = pt_card(h) -%} +{%- set w = w ~ "a" if w.endswith("um") else (w[0:(w|length) - 4] ~ "duas" if w.endswith("dois") else w) -%} +{{- w ~ (" e " ~ pt_card(m) if mm else (" hora" if hn == 1 else " horas")) -}} +{%- elif L == "ja" -%} +{{- ja_card(h) ~ "時" ~ (ja_card(m) ~ "分" if mm else "") -}} +{%- elif L == "zh" -%} +{{- ("两" if hn == 2 else zh_card(h)) ~ "点" ~ (zh_card(m) ~ "分" if mm else "") -}} +{%- else -%} +{{- en_card(h) ~ (" o'clock" if mm == 0 else (" oh " ~ ones[mm] if mm < 10 else " " ~ under100(mm))) -}} +{%- endif -%} +{%- endmacro -%} + +{%- macro ordinal(L, d, suf) -%} +{%- set n = d|int -%} +{%- if L == "fr" -%} +{%- set plural = "s" if suf.endswith("s") else "" -%} +{%- if n == 1 -%} +{{- ("première" if suf.startswith("r") or suf.startswith("è") else "premier") ~ plural -}} +{%- elif n == 2 and suf.startswith("nd") -%} +{{- ("seconde" if suf.startswith("nde") else "second") ~ plural -}} +{%- else -%} +{{- fr_ord(fr_card(d)) ~ plural -}} +{%- endif -%} +{%- elif L in ["es", "it", "pt"] -%} +{%- set fem = suf == "ª" -%} +{%- if n >= 1 and n <= 10 -%} +{%- set w = {"es": es_ord, "it": it_ord, "pt": pt_ord}[L][n] -%} +{{- w[0:(w|length) - 1] ~ "a" if fem else w -}} +{%- elif L == "it" and n < 1000000 -%} +{#- undicesimo, ventitreesimo, ventiseiesimo -#} +{%- set w = it6(n) -%} +{%- set w = w ~ "esimo" if w.endswith("tre") or w.endswith("sei") else w[0:(w|length) - 1] ~ "esimo" -%} +{{- w[0:(w|length) - 1] ~ "a" if fem else w -}} +{%- else -%} +{{- card(L, d) -}} +{%- endif -%} +{%- else -%} +{{- en_ord(en_card(d)) -}} +{%- endif -%} +{%- endmacro -%} + +{%- set l = ((language | lower) if language is defined and language else "").replace("-", "_") -%} +{%- set L = "en" if l == "" else ("zh" if l.startswith("zh") or l.startswith("cmn") else l.split("_")[0]) -%} +{%- set text = messages[0]["content"] -%} +{%- if L not in M -%} +{{- text -}} +{%- else -%} +{%- set t = text -%} +{%- if L not in ["ja", "zh"] -%} +{#- a space between a number and the letters it touches (5km, mp3), except for ordinal suffixes and the French 15h30 -#} +{%- set suffixes = {"en": "st|nd|rd|th|s", "fr": "ers?|res?|ères?|nde?s?|èmes?|es?|h", "es": "º|ª", "it": "º|ª", "pt": "º|ª"} -%} +{%- if L in suffixes -%} +{%- set t = tts_norm_sub(t, "(%d)(?!(?:" ~ suffixes[L] ~ ")(?!%a))(%a)", "%1 %2") -%} +{%- else -%} +{%- set t = tts_norm_sub(t, "(%d)(%a)", "%1 %2") -%} +{%- endif -%} +{%- set t = tts_norm_sub(t, "(? 2 else "")) -}} +{%- elif s.rule == "ordinal" -%} +{{- ordinal(L, g[0], g[1]) -}} +{%- elif s.rule == "year" -%} +{{- spell(L, g[0]) -}} +{%- elif s.rule == "year4" -%} +{{- g[0] ~ g[1] ~ number(L, g[2], "", true) -}} +{%- elif s.rule == "plural" -%} +{#- decades and plurals: 1990s -> nineteen nineties -#} +{%- set w = number(L, g[0], "", true) -%} +{{- (w[0:(w|length) - 1] ~ "ies") if w.endswith("y") else w ~ "s" -}} +{%- elif s.rule == "number" -%} +{{- (W[L]["minus"] if g[0] else "") ~ number(L, g[1], g[2], false) -}} +{%- else -%} +{{- s.text -}} +{%- endif -%} +{%- endfor -%} +{%- endif -%} diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 22419390f..1869c1203 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -175,6 +175,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_NANBEIGE, "nanbeige" }, { LLM_ARCH_QWEN3TTS, "qwen3tts" }, { LLM_ARCH_POCKETTTS, "pockettts" }, + { LLM_ARCH_ZONOS3, "zonos3" }, { LLM_ARCH_K2_HORIZON, "k2-horizon" }, { LLM_ARCH_BEAM, "beam" }, { LLM_ARCH_KOLIBRI1, "kolibri1" }, @@ -1416,6 +1417,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) { case LLM_ARCH_DEEPSEEK4: case LLM_ARCH_DEEPSEEK41: case LLM_ARCH_MINIMAX_01: + case LLM_ARCH_ZONOS3: case LLM_ARCH_INKLING2: return true; default: diff --git a/src/llama-arch.h b/src/llama-arch.h index 74bd6ce67..d015b5d5a 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -176,6 +176,7 @@ enum llm_arch { LLM_ARCH_NANBEIGE, LLM_ARCH_QWEN3TTS, LLM_ARCH_POCKETTTS, + LLM_ARCH_ZONOS3, LLM_ARCH_MINIMAX_01, LLM_ARCH_HRM_TEXT, LLM_ARCH_K2_HORIZON, diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 93edabc02..b66092b17 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -2908,6 +2908,7 @@ uint32_t llama_context::graph_max_nodes() const { model.arch == LLM_ARCH_BAILINGMOE3 || model.arch == LLM_ARCH_QWEN35 || model.arch == LLM_ARCH_QWEN35MOE || + model.arch == LLM_ARCH_ZONOS3 || model.arch == LLM_ARCH_CLEF || model.arch == LLM_ARCH_CLOVER_MOE || model.arch == LLM_ARCH_QWEN4EXP || diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 194bbb92b..9801b4b0c 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -126,6 +126,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_qwen3tts(params); case LLM_ARCH_POCKETTTS: return new llama_model_pockettts(params); + case LLM_ARCH_ZONOS3: + return new llama_model_zonos3(params); case LLM_ARCH_PHI2: return new llama_model_phi2(params); case LLM_ARCH_PHI3: @@ -687,8 +689,9 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str return {{conv_state_size, 3}}; } // TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN] + // zonos3 has the qwen35 GDN layout, its attention layers have no fused qkv if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF || ud->model->arch == LLM_ARCH_CLOVER_MOE || - ud->model->arch == LLM_ARCH_QWEN4EXP) { + ud->model->arch == LLM_ARCH_QWEN4EXP || ud->model->arch == LLM_ARCH_ZONOS3) { // fused full attention layers with Q gate tensors that need n_embd doubled: if (!hparams.is_recr(il) && (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias))) { @@ -731,7 +734,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim); return {{key_dim, 2 + head_ratio}}; } - if (std::regex_match(tensor_name, pattern_attn_gate_weight) || std::regex_match(tensor_name, pattern_ssm_out_weight)) { + if ((hparams.is_recr(il) && std::regex_match(tensor_name, pattern_attn_gate_weight)) || std::regex_match(tensor_name, pattern_ssm_out_weight)) { return {{key_dim, head_ratio}}; } if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) || @@ -2700,6 +2703,7 @@ void llama_model::print_info() const { arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_CLOVER_MOE || + arch == LLM_ARCH_ZONOS3 || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) { LLAMA_LOG_INFO("%s: ssm_d_conv = %u\n", __func__, hparams.ssm_d_conv); @@ -3331,7 +3335,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, filter_recr = [&](uint32_t il) { return hparams.is_recr(il) && hparams.n_ff(il) == 0; }; - } else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_CLOVER_MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_MINIMAX_01) { + } else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_CLOVER_MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_MINIMAX_01 || + arch == LLM_ARCH_ZONOS3) { filter_attn = [&](uint32_t il) { return il < hparams.n_layer() && !hparams.is_recr(il); }; @@ -3745,6 +3750,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_DOTS3NOTE: case LLM_ARCH_NANBEIGE: case LLM_ARCH_POCKETTTS: + case LLM_ARCH_ZONOS3: case LLM_ARCH_LIMITE: return LLAMA_ROPE_TYPE_NORM; case LLM_ARCH_BAILINGMOE3: diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index f2c1854c9..6f23a29d8 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -584,6 +584,12 @@ struct llm_tokenizer_bpe : llm_tokenizer { regex_exprs = { "\\p{N}", "\\s+|[\\p{L}\\p{N}_]+|[^\\p{L}\\p{N}_\\s]+" }; byte_encode = false; break; + case LLAMA_VOCAB_PRE_TYPE_ZONOS3: + // SuperBPE: no split on whitespace, tokens can span several words + regex_exprs = { + "\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]{2,}[\\r\\n/]*| +(?!\\S)", + }; + break; case LLAMA_VOCAB_PRE_TYPE_WHITESPACE: // whitespace pre-tokenizer (jinaai/jina-embeddings-v2-base-zh) regex_exprs = { @@ -2774,6 +2780,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } else if ( tokenizer_pre == "soprano") { pre_type = LLAMA_VOCAB_PRE_TYPE_SOPRANO; + } else if ( + tokenizer_pre == "zonos3") { + pre_type = LLAMA_VOCAB_PRE_TYPE_ZONOS3; + clean_spaces = false; } else { throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str())); } diff --git a/src/llama-vocab.h b/src/llama-vocab.h index 1ae5b65f6..9345275f5 100644 --- a/src/llama-vocab.h +++ b/src/llama-vocab.h @@ -75,6 +75,7 @@ enum llama_vocab_pre_type { LLAMA_VOCAB_PRE_TYPE_NEMOTRON_3_5_EMBD = 64, LLAMA_VOCAB_PRE_TYPE_SOPRANO = 65, LLAMA_VOCAB_PRE_TYPE_TEKKEN_NUM3 = 66, + LLAMA_VOCAB_PRE_TYPE_ZONOS3 = 67, }; // HF Metaspace pre-tokenizer prepend_scheme diff --git a/src/models/models.h b/src/models/models.h index 913c52ade..932887c91 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -769,6 +769,28 @@ struct llama_model_pockettts : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; +struct llama_model_zonos3 : public llama_model_base { + llama_model_zonos3(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_build_delta_net_base { + graph(const llm_graph_params & params); + private: + // Full AttnRes bank: embedding + every sublayer output so far, [n_embd, n_src, n_tokens] + ggml_tensor * bank = nullptr; + + void attn_res_push(ggml_tensor * cur); + ggml_tensor * attn_res_route(ggml_tensor * query, int il); + + ggml_tensor * build_layer_attn(llm_graph_input_attn_kv * inp, ggml_tensor * cur, ggml_tensor * inp_pos, int il); + ggml_tensor * build_layer_gdn(llm_graph_input_rs * inp, ggml_tensor * cur, int il); + ggml_tensor * build_layer_ffn(ggml_tensor * cur, int il); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + struct llama_model_codeshell : public llama_model_base { llama_model_codeshell(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/src/models/zonos3.cpp b/src/models/zonos3.cpp new file mode 100644 index 000000000..c5f62fe29 --- /dev/null +++ b/src/models/zonos3.cpp @@ -0,0 +1,351 @@ +#include "models.h" +#include "llama-memory-recurrent.h" + +// ZONOS3 TTS backbone, the 10 audio heads are in the mmproj + +void llama_model_zonos3::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv); + ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner); + ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state); + ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank); + ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group); + + if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) { + uint32_t full_attn_interval = 0; + ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval); + if (full_attn_interval == 0) { + throw std::runtime_error("zonos3: full_attention_interval must be > 0"); + } + for (uint32_t i = 0; i < hparams.n_layer_all; ++i) { + hparams.is_recr_impl[i] = (i + 1) % full_attn_interval != 0; + } + } + + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); + ml.get_key(LLM_KV_MOE_LATENT_SIZE, hparams.moe_latent_size); + + // the router picks the top-k of logits + bias, then takes a softmax over the picked logits + hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT; + hparams.expert_weights_norm = false; + + switch (hparams.n_layer()) { + case 11: type = LLM_TYPE_0_8B; break; + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_zonos3::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output_res_score = create_tensor(tn(LLM_TENSOR_OUTPUT_RES_SCORE, "weight"), {n_embd}, 0); + // no output head, the logits are unused; reuse the embedding table so a sampler can still run + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + + const int64_t d_inner = hparams.ssm_d_inner; + const int64_t head_dim = hparams.ssm_d_state; + const int64_t n_gdn_head = hparams.ssm_dt_rank; + const int64_t n_latent = hparams.moe_latent_size; + const int64_t n_ff_exp = hparams.n_ff_exp(); + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + layer.attn_res_score = create_tensor(tn(LLM_TENSOR_ATTN_RES_SCORE, "weight", i), {n_embd}, 0); + layer.ffn_res_score = create_tensor(tn(LLM_TENSOR_FFN_RES_SCORE, "weight", i), {n_embd}, 0); + + if (hparams.is_recr(i)) { + // split from the fused GDN in_proj at conversion, same layout as qwen35 + layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, 3 * d_inner}, 0); + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, d_inner}, 0); // z gate + layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", i), {hparams.ssm_d_conv, 3 * d_inner}, 0); + layer.ssm_dt = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {n_gdn_head}, 0); + layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), {n_gdn_head}, 0); + layer.ssm_alpha = create_tensor(tn(LLM_TENSOR_SSM_ALPHA, "weight", i), {n_embd, n_gdn_head}, 0); + layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_gdn_head}, 0); + layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {head_dim}, 0); + layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", i), {d_inner, n_embd}, 0); + } else { + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0); + // per-head scale of the normalized Q, the QK norm itself has no weight + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k, n_head}, 0); + // per-head output gate, same name as the GDN z gate + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head}, 0); + } + + if (i < (int) hparams.n_layer_dense_lead) { + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + } else { + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0); + layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", i), {n_embd, n_latent}, 0); + layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", i), {n_latent, n_embd}, 0); + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_latent, n_ff_exp, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_latent, n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_latent, n_expert}, 0); + } + } +} + +std::unique_ptr llama_model_zonos3::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(params); +} + +llama_model_zonos3::graph::graph(const llm_graph_params & params) : + llm_build_delta_net_base(params) { + const float eps = hparams.f_norm_rms_eps; + + ggml_tensor * cur; + + ggml_tensor * inpL = build_inp_embd(model.tok_embd); + + auto * inp = build_inp_mem_hybrid(); + + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + // the summed row embedding goes through a weightless norm + inpL = ggml_rms_norm(ctx0, inpL, eps); + cb(inpL, "emb_norm", -1); + attn_res_push(inpL); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + cur = attn_res_route(layer.attn_res_score, il); + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + if (hparams.is_recr(il)) { + cur = build_layer_gdn(inp->get_recr(), cur, il); + } else { + cur = build_layer_attn(inp->get_attn(), cur, inp_pos, il); + } + cb(cur, "attn_out", il); + attn_res_push(cur); + + cur = attn_res_route(layer.ffn_res_score, il); + cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_layer_ffn(cur, il); + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + attn_res_push(cur); + } + + cur = attn_res_route(model.output_res_score, -1); + if (inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + + cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur, model.output_s); + cb(cur, "result_output", -1); + res->t_logits = cur; +} + +void llama_model_zonos3::graph::attn_res_push(ggml_tensor * cur) { + ggml_tensor * src = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens); + bank = bank ? ggml_concat(ctx0, bank, src, 1) : src; +} + +// softmax over the bank sources, scored by query . rms_norm(source); the sum uses the raw sources +ggml_tensor * llama_model_zonos3::graph::attn_res_route(ggml_tensor * query, int il) { + const int64_t n_src = bank->ne[1]; + + ggml_tensor * scores = ggml_rms_norm(ctx0, bank, hparams.f_norm_rms_eps); + scores = ggml_mul(ctx0, scores, query); + scores = ggml_sum_rows(ctx0, scores); // [1, n_src, n_tokens] + scores = ggml_reshape_2d(ctx0, scores, n_src, n_tokens); + + ggml_tensor * probs = ggml_soft_max(ctx0, scores); + cb(probs, "attn_res_probs", il); + + return ggml_dsv4_hc_pre(ctx0, bank, probs); +} + +ggml_tensor * llama_model_zonos3::graph::build_layer_attn( + llm_graph_input_attn_kv * inp, + ggml_tensor * cur, + ggml_tensor * inp_pos, + int il) { + const auto & layer = model.layers[il]; + + const int64_t n_embd_head = hparams.n_embd_head_v(); + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + + // the reference QK norm has no weight and a fixed eps + const float qk_norm_eps = 1e-6f; + + auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il); + + ggml_tensor * gate = build_lora_mm(layer.wqkv_gate, cur); + gate = ggml_sigmoid(ctx0, ggml_reshape_3d(ctx0, gate, 1, n_head, n_tokens)); + cb(gate, "attn_gate", il); + + Qcur = ggml_rms_norm(ctx0, Qcur, qk_norm_eps); + Qcur = ggml_mul(ctx0, Qcur, layer.attn_q_norm); + Kcur = ggml_rms_norm(ctx0, Kcur, qk_norm_eps); + cb(Qcur, "Qcur_normed", il); + cb(Kcur, "Kcur_normed", il); + + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + cur = build_attn(inp, + nullptr, nullptr, nullptr, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + cb(cur, "attn_pregate", il); + + cur = ggml_reshape_3d(ctx0, cur, n_embd_head, n_head, n_tokens); + cur = ggml_mul(ctx0, cur, gate); + cur = ggml_reshape_2d(ctx0, cur, n_embd_head * n_head, n_tokens); + cb(cur, "attn_gated", il); + + return build_lora_mm(layer.wo, cur, layer.wo_s); +} + +// copied from qwen35 build_layer_attn_linear, with the reference L2 norm eps +ggml_tensor * llama_model_zonos3::graph::build_layer_gdn( + llm_graph_input_rs * inp, + ggml_tensor * cur, + int il) { + const auto & layer = model.layers[il]; + const auto * mctx_cur = inp->mctx; + + const int64_t d_inner = hparams.ssm_d_inner; + const int64_t n_seqs = ubatch.n_seqs; + const int64_t head_k_dim = hparams.ssm_d_state; + const int64_t num_k_heads = hparams.ssm_n_group; + const int64_t num_v_heads = hparams.ssm_dt_rank; + const int64_t head_v_dim = d_inner / num_v_heads; + const int64_t n_seq_tokens = ubatch.n_seq_tokens; + + GGML_ASSERT(n_seqs != 0); + GGML_ASSERT(ubatch.equal_seqs()); + GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs); + GGML_ASSERT(num_k_heads == num_v_heads); + + ggml_tensor * qkv_mixed = build_lora_mm(layer.wqkv, cur, layer.wqkv_s); + qkv_mixed = ggml_reshape_3d(ctx0, qkv_mixed, qkv_mixed->ne[0], n_seq_tokens, n_seqs); + cb(qkv_mixed, "linear_attn_qkv_mixed", il); + + ggml_tensor * z = build_lora_mm(layer.wqkv_gate, cur, layer.wqkv_gate_s); + cb(z, "z", il); + + ggml_tensor * beta = build_lora_mm(layer.ssm_beta, cur, layer.ssm_beta_s); + beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs); + beta = ggml_sigmoid(ctx0, beta); + cb(beta, "beta_sigmoid", il); + + ggml_tensor * alpha = build_lora_mm(layer.ssm_alpha, cur, layer.ssm_alpha_s); + alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs); + + ggml_tensor * alpha_softplus = ggml_softplus(ctx0, ggml_add(ctx0, alpha, layer.ssm_dt)); + ggml_tensor * gate = ggml_mul(ctx0, alpha_softplus, layer.ssm_a); // -A_log.exp() * softplus + gate = ggml_reshape_4d(ctx0, gate, 1, num_v_heads, n_seq_tokens, n_seqs); + cb(gate, "gate", il); + + ggml_tensor * conv_states_all = mctx_cur->get_r_l(il); + ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il); + + ggml_tensor * conv_kernel = layer.ssm_conv1d; + const int64_t conv_kernel_size = conv_kernel->ne[0]; + const int64_t conv_channels = d_inner + 2 * num_k_heads * head_k_dim; + + ggml_tensor * conv_input = build_conv_state(inp, conv_states_all, qkv_mixed, conv_kernel_size, conv_channels, il); + + ggml_tensor * state = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs); + state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs); + + ggml_tensor * conv_out = ggml_silu(ctx0, ggml_ssm_conv(ctx0, conv_input, conv_kernel)); + cb(conv_out, "conv_output_silu", il); + + const int64_t qkv_dim = head_k_dim * num_k_heads * 2 + head_v_dim * num_v_heads; + const int64_t nb1_qkv = ggml_row_size(conv_out->type, qkv_dim); + + ggml_tensor * q_conv = ggml_view_4d(ctx0, conv_out, head_k_dim, num_k_heads, n_seq_tokens, n_seqs, + ggml_row_size(conv_out->type, head_k_dim), nb1_qkv, nb1_qkv * n_seq_tokens, 0); + ggml_tensor * k_conv = ggml_view_4d(ctx0, conv_out, head_k_dim, num_k_heads, n_seq_tokens, n_seqs, + ggml_row_size(conv_out->type, head_k_dim), nb1_qkv, nb1_qkv * n_seq_tokens, + ggml_row_size(conv_out->type, head_k_dim * num_k_heads)); + ggml_tensor * v_conv = ggml_view_4d(ctx0, conv_out, head_v_dim, num_v_heads, n_seq_tokens, n_seqs, + ggml_row_size(conv_out->type, head_v_dim), nb1_qkv, nb1_qkv * n_seq_tokens, + ggml_row_size(conv_out->type, 2 * head_k_dim * num_k_heads)); + + // the reference L2 norm is x * rsqrt(sum(x^2) + 1e-6) + q_conv = build_gdn_l2_norm(ctx0, q_conv, 1e-6f); + k_conv = build_gdn_l2_norm(ctx0, k_conv, 1e-6f); + cb(q_conv, "q_conv_predelta", il); + cb(k_conv, "k_conv_predelta", il); + cb(v_conv, "v_conv_predelta", il); + + ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il); + + // gated norm: rms_norm(output) * silu(z), per head + ggml_tensor * z_4d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs); + output = build_norm(output, layer.ssm_norm, nullptr, LLM_NORM_RMS, il); + output = ggml_mul(ctx0, output, ggml_silu(ctx0, z_4d)); + + output = ggml_reshape_2d(ctx0, output, head_v_dim * num_v_heads, n_seq_tokens * n_seqs); + cb(output, "linear_attn_gated", il); + + return build_lora_mm(layer.ssm_out, output, layer.ssm_out_s); +} + +ggml_tensor * llama_model_zonos3::graph::build_layer_ffn(ggml_tensor * cur, int il) { + const auto & layer = model.layers[il]; + + if (layer.ffn_gate_inp == nullptr) { + cur = build_ffn(cur, + layer.ffn_up, nullptr, layer.ffn_up_s, + layer.ffn_gate, nullptr, layer.ffn_gate_s, + layer.ffn_down, nullptr, layer.ffn_down_s, + nullptr, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + return cur; + } + + // the router reads the full-width input, the experts the latent one + ggml_tensor * logits = build_lora_mm(layer.ffn_gate_inp, ggml_rms_norm(ctx0, cur, hparams.f_norm_rms_eps)); + cb(logits, "ffn_moe_logits", il); + + ggml_tensor * latent = build_lora_mm(layer.ffn_latent_down, cur); + cb(latent, "ffn_latent_in", il); + + cur = build_moe_ffn(latent, + nullptr, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + logits); + cb(cur, "ffn_moe_out", il); + + cur = build_lora_mm(layer.ffn_latent_up, cur); + cb(cur, "ffn_out", il); + return cur; +} diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv index 89cafb247..efb8718e7 100644 --- a/tests/fusion/MTL.csv +++ b/tests/fusion/MTL.csv @@ -348,3 +348,8 @@ xing4_0 ,0 ,any ,MUL+ADD , 2 xing4_0 ,0 ,any ,RMS_NORM+MUL , 9 xing4_0 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 xverse ,0 ,any ,RMS_NORM+MUL , 5 +zonos3 ,0 ,any ,GATED_DELTA_NET+CPY , 1 +zonos3 ,0 ,any ,MUL+ADD , 2 +zonos3 ,0 ,any ,RMS_NORM+MUL , 12 +zonos3 ,0 ,any ,RMS_NORM+SCALE , 2 +zonos3 ,0 ,any ,SSM_CONV+UNARY , 1 diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp index d52f1af9c..3f86d1535 100644 --- a/tests/test-jinja.cpp +++ b/tests/test-jinja.cpp @@ -3,6 +3,9 @@ #include #include #include +#include +#include +#include #include "chat-tts-norm.h" #include "json.h" @@ -42,6 +45,7 @@ static void test_caps(testing & t); static void test_string_parts(testing & t); static void test_fuzzing(testing & t); static void test_tts_norm_patterns(testing & t); +static void test_tts_template_zonos3(testing & t); static void test_shared_render(testing & t); @@ -81,6 +85,7 @@ int main(int argc, char *argv[]) { t.test("array methods", test_array_methods); t.test("object methods", test_object_methods); t.test("tts norm patterns", test_tts_norm_patterns); + t.test("tts template zonos3", test_tts_template_zonos3); if (!g_python_mode) { t.test("shared render", test_shared_render); t.test("hasher", test_hasher); @@ -2979,6 +2984,72 @@ static void test_tts_norm_patterns(testing & t) { } } +// the ZONOS3 "tts" template spells out numbers in en, fr, de, es, it, pt, ja and zh, other languages pass through +static void test_tts_template_zonos3(testing & t) { + std::string path = __FILE__; + const size_t pos = path.find_last_of("/\\"); + path = (pos == std::string::npos ? std::string(".") : path.substr(0, pos)) + "/../models/templates/Zyphra-ZONOS3-Mini-tts.jinja"; + std::ifstream fs(path, std::ios_base::binary); + if (!t.assert_true("template file is readable: " + path, fs.is_open())) { + return; + } + const std::string tmpl((std::istreambuf_iterator(fs)), std::istreambuf_iterator()); + + const std::vector> cases = { + { "en", "I have 3 cats and 21 dogs.", "I have three cats and twenty-one dogs." }, + { "en", "1,234,567 people, 1,2345 and 12345", "one million two hundred thirty-four thousand five hundred sixty-seven people, one,two thousand three hundred forty-five and twelve thousand three hundred forty-five" }, + { "en", "$1, $0.99, $5.32 and $2.5 billion", "one dollar, ninety-nine cents, five dollars and thirty-two cents and two point five billion dollars" }, + { "en", "\xe2\x82\xac" "20 and \xc2\xa3" "3.50", "twenty euros and three pounds and fifty pence" }, + { "en", "3.5% then 10%", "three point five percent then ten percent" }, + { "en", "1st, 22nd, 103rd, 12th", "first, twenty-second, one hundred third, twelfth" }, + { "en", "in 1990, since 1905, by 2005, in 2000, from 1900, the 1990s, 1234 people", + "in nineteen ninety, since nineteen oh five, by two thousand five, in two thousand, from nineteen hundred, the nineteen nineties, one thousand two hundred thirty-four people" }, + { "en", "at 3:30, 10:05 or 7:00, not 25:00", "at three thirty, ten oh five or seven o'clock, not twenty-five:zero zero" }, + { "en", "pi is 3.14, it hit -5 today, range 10-20", "pi is three point one four, it hit minus five today, range ten-twenty" }, + { "en", "agent 007 ran 5km with an mp3", "agent zero zero seven ran five km with an mp three" }, + { "en", "caf\xc3\xa9 42.", "caf\xc3\xa9 forty-two." }, + { "", "no language 42", "no language forty-two" }, + { "en", "no digits here", "no digits here" }, + { "ko", "3\xea\xb0\x9c", "3\xea\xb0\x9c" }, + // other languages + { "fr_fr", "J'ai 21 chats, 71 chiens, 80 oiseaux et 91 souris.", + "J'ai vingt et un chats, soixante et onze chiens, quatre-vingts oiseaux et quatre-vingt-onze souris." }, + { "fr", "200 ans, 2 345 678 habitants et 80 000 euros.", + "deux cents ans, deux millions trois cent quarante-cinq mille six cent soixante-dix-huit habitants et quatre-vingt mille euros." }, + { "fr", "Ça coûte 5,32 €, hausse de 12,05 %.", + "Ça coûte cinq euros et trente-deux centimes, hausse de douze virgule zéro cinq pour cent." }, + { "fr", "Le 1er mai, la 1re fois, le 21e siècle, à 15h30, à 1h ou à 21:05, -5 degrés.", + "Le premier mai, la première fois, le vingt et unième siècle, à quinze heures trente, à une heure ou à vingt et une heures cinq, moins cinq degrés." }, + { "de", "21 Katzen, 101 Hunde, 1.234 Bücher, 1.000.000 Einwohner, im Jahr 1990.", + "einundzwanzig Katzen, einhunderteins Hunde, eintausendzweihundertvierunddreißig Bücher, eine Million Einwohner, im Jahr neunzehnhundertneunzig." }, + { "fr", "2 000 000 €, 3 000 000 $ et 2 500 000 €.", + "deux millions d'euros, trois millions de dollars et deux millions cinq cent mille euros." }, + { "es", "2.000.000 € y 1.000.000 $.", "dos millones de euros y un millón de dólares." }, + { "de", "Im Jahr 1990 waren 1500 Leute da.", "Im Jahr neunzehnhundertneunzig waren eintausendfünfhundert Leute da." }, + { "de", "5,32 € oder 1 €, plus 3,5 % um 15:30 Uhr, am 3.10.2024.", + "fünf Euro und zweiunddreißig Cent oder ein Euro, plus drei Komma fünf Prozent um fünfzehn Uhr dreißig, am drei zehn zweitausendvierundzwanzig." }, + { "es", "100, 101, 21.000 personas, 2.500.000 habitantes, 21 €, a las 1:00, el 3.º y la 2ª vez.", + "cien, ciento uno, veintiún mil personas, dos millones quinientos mil habitantes, veintiún euros, a las una en punto, el tercero y la segunda vez." }, + { "it", "21, 23, 28, 180, 3.003, 1.000.000 di euro, 5,32 €, all'1:00, l'11º e il 23º.", + "ventuno, ventitré, ventotto, centottanta, tremilatré, un milione di euro, cinque euro e trentadue centesimi, all'una, l'undicesimo e il ventitreesimo." }, + { "pt_br", "101, 1.200, 1.234, R$ 5,32, às 2:00, o 1º e a 2ª vez.", + "cento e um, mil e duzentos, mil duzentos e trinta e quatro, cinco reais e trinta e dois centavos, às duas horas, o primeiro e a segunda vez." }, + { "ja", "3個、1000円、10000人、123456789人、1.5キロ、30%、$5、午後3:30、-5度、2024年。", + "三個、千円、一万人、一億二千三百四十五万六千七百八十九人、一点五キロ、三十パーセント、五ドル、午後三時三十分、マイナス五度、二千二十四年。" }, + { "cmn", "10005人,115人,2000人,20000人,316,2:30,2002年,-5度,3.5%,$23。", + "一万零五人,一百一十五人,两千人,两万人,三百一十六,两点三十分,二零零二年,负五度,百分之三点五,二十三美元。" }, + }; + for (const auto & c : cases) { + // same variables as common_chat_get_tts_prompt() + const json vars = { + {"messages", json::array({{{"role", "user"}, {"content", c[1]}}})}, + {"language", c[0]}, + {"voice", ""}, + }; + test_template(t, c[0] + ": " + c[1], tmpl, vars, c[2], true); + } +} + static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm) { t.test(name, [&tmpl, &expect, tts_norm](testing & t) { std::string error; diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 32e77e883..147d5b536 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -785,6 +785,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, std::vector{n_ff, n_ff / 2, n_ff, n_ff}); } + if (arch == LLM_ARCH_ZONOS3) { + ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd / 2); + ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, n_ff); + } + ms.add_kv(LLM_KV_POSNET_EMBEDDING_LENGTH, n_embd); ms.add_kv(LLM_KV_POSNET_BLOCK_COUNT, n_layer); ms.add_kv(LLM_KV_CONVNEXT_EMBEDDING_LENGTH, n_embd); @@ -793,7 +798,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const ms.add_kv(LLM_KV_XIELU_ALPHA_P, 1.0f); ms.add_kv(LLM_KV_XIELU_BETA, 1.0f); ms.add_kv(LLM_KV_XIELU_EPS, 1.0e-7f); - ms.add_kv(LLM_KV_SSM_INNER_SIZE, arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_CLOVER_MOE ? 256 : 2*n_embd); + ms.add_kv(LLM_KV_SSM_INNER_SIZE, arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_CLOVER_MOE || arch == LLM_ARCH_ZONOS3 ? 256 : 2*n_embd); ms.add_kv(LLM_KV_SSM_CONV_KERNEL, uint32_t(4)); ms.add_kv(LLM_KV_SSM_STATE_SIZE, uint32_t(128)); ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head); @@ -1077,6 +1082,7 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_MUSE_SPARK: case LLM_ARCH_KOLIBRI1: case LLM_ARCH_METIS16: + case LLM_ARCH_ZONOS3: case LLM_ARCH_INKLING2: return true; default: diff --git a/tests/test-server-component.cpp b/tests/test-server-component.cpp index 580203be8..2ef8b04f5 100644 --- a/tests/test-server-component.cpp +++ b/tests/test-server-component.cpp @@ -825,9 +825,9 @@ static void test_speech_convert(testing & t) { t.assert_true ("audio streams", out.at("stream").get()); t.assert_true ("rejects sse", rejected("stream_format", "sse")); - // speed - t.assert_true ("speed 1.0", !rejected("speed", 1.0)); - t.assert_true ("rejects speed 1.5", rejected("speed", 1.5)); + // speed is forwarded, the model decides whether it is supported + out = convert_speech_to_tts(with("speed", 1.5)); + t.assert_equal("speed forwarded", 1.5, out.at("speed").get()); // instructions are ignored t.assert_true ("instructions", !rejected("instructions", "whisper")); diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index fd54e2e72..32a17ef61 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -83,6 +83,8 @@ add_library(mtmd models/qwen3tts-spkenc.cpp models/qwen3tts-gen.cpp models/drama3-gen.cpp + models/zonos3-gen.cpp + models/zonos3-spkenc.cpp models/pockettts-seanet.cpp models/pockettts-spkenc.cpp models/pockettts-gen.cpp diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md index 8feb7a647..b6ed5a577 100644 --- a/tools/mtmd/README-dev.md +++ b/tools/mtmd/README-dev.md @@ -103,6 +103,7 @@ Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system i - In most cases, you can hard-code model configs in the model graph class, or in `mtmd_hparams` - If some values need to be exposed to the `mtmd_helper` layer, hard-code them in `mtmd_helper` and distinguish by pipeline and `mtmd_gen_audio_info::model_variant` if necessary - Do NOT add new GGUF metadata or new fields to `mtmd_gen_audio_info` unless you can prove that you absolutely need them + - If the model requires special preprocessing of the input text prompt (prompt format, text normalization), put it in `tokenizer.chat_template.tts` as a jinja program. It receives the text as `messages[0].content`, plus `language` and `voice`. See `conversion/lfm3.py` (prompt format) and `models/templates/Zyphra-ZONOS3-Mini-tts.jinja` (text normalization) as examples. For text normalization, use `tts_norm_scan()` / `tts_norm_sub()` (see `docs/tts-norm-patterns.md`) and add test cases to `tests/test-jinja.cpp`, which also checks them against the python reference with `-py` 4. Make sure most of the changes happen inside `mtmd-helper-gen.cpp`. A good PR looks like this: - 10-20% changes is to add new backbone (text) model and conversion - 60% changes inside `mtmd-helper-gen.cpp` diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 58a83378a..05494c71f 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -341,6 +341,27 @@ struct mtmd_graph_mimo_audio : mtmd_graph { ggml_cgraph * build() override; }; +// zonos3 speaker encoder: MECT-B2 + ASTP, then the speaker projection of the backbone +struct mtmd_graph_zonos3_spkenc : mtmd_graph { + mtmd_graph_zonos3_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} + ggml_cgraph * build() override; + + ggml_tensor * W(const std::string & name) const; + bool has(const std::string & name) const; + ggml_tensor * chan(ggml_tensor * v) const; + ggml_tensor * bn(ggml_tensor * x, const std::string & p) const; + ggml_tensor * conv(ggml_tensor * x, const std::string & p, int s_t, int s_f, bool bias, int pad) const; + ggml_tensor * grouped3x3(ggml_tensor * x, const std::string & p, int s_t, int s_f) const; + ggml_tensor * res_block(ggml_tensor * x, const std::string & q, int s_f, int s_t) const; + ggml_tensor * linear(ggml_tensor * x, const std::string & p) const; + ggml_tensor * layer_norm(ggml_tensor * x, const std::string & p) const; + ggml_tensor * flatten(ggml_tensor * x) const; + ggml_tensor * unflatten(ggml_tensor * x, int64_t F, int64_t C) const; + ggml_tensor * attention(ggml_tensor * h, const std::string & p) const; + ggml_tensor * moe_ffn(ggml_tensor * h, const std::string & p) const; + ggml_tensor * pos_conv(ggml_tensor * h, const std::string & p) const; +}; + struct mtmd_graph_qwen3tts_spkenc : mtmd_graph { mtmd_graph_qwen3tts_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; @@ -536,6 +557,47 @@ struct mtmd_graph_drama3_gen : mtmd_graph { ggml_tensor * inp_pos, ggml_tensor * k_slot, ggml_tensor * v_slot, int pos); }; +// +// zonos3 generation: +// GEN_CODE = 10 codebook heads, or the backbone row of a sampled frame +// GEN_WAV = ZAC codec decoder up to the iSTFT spectra +// +struct mtmd_graph_zonos3_gen : mtmd_graph { + mtmd_graph_zonos3_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, bool all_known, int n_frames) + : mtmd_graph(ctx, img), gen_process(gen_process), all_known(all_known), n_frames(n_frames) {} + ggml_cgraph * build() override; + + static constexpr int n_codebooks = 10; + static constexpr int n_cb_vocab = 4096; + static constexpr float softcap = 15.0f; + + // ZAC decoder, fixed by the codec config + static constexpr int n_stages = 5; + static constexpr int stage_ctx[n_stages] = { 150, 300, 480, 480, 480 }; // attention window + static constexpr int head_dim = 64; + static constexpr float ln_eps = 1e-5f; + static constexpr float convnext_eps = 1e-6f; + static constexpr int hop = 184; // at 44160 Hz + static constexpr int n_fft[2] = { 736, 2208 }; + static constexpr int rs_orig = 736; // 44160 -> 44100 Hz + static constexpr int rs_new = 735; + static constexpr int samples_per_frame = 2940; + + mtmd_subctx_gen_process gen_process; + bool all_known; // GEN_CODE: only the backbone row + int n_frames; // GEN_WAV + + std::map state_in; + std::vector> state_out; + + ggml_cgraph * build_gen_code(); + ggml_cgraph * build_gen_wav(); + ggml_tensor * convnext(ggml_tensor * x, const mtmd_zonos3_wav::convnext & b, const std::string & state_name); + ggml_tensor * moss_layer(ggml_tensor * x, const mtmd_zonos3_wav::layer & l, int s, int il, ggml_tensor * base); +}; + +std::vector list_zonos3_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); + struct mtmd_graph_soprano : mtmd_graph { mtmd_graph_soprano(mtmd_subcontext * ctx, const mtmd_image_f32 & img, int n_frames) : mtmd_graph(ctx, img), n_frames(n_frames) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/models/zonos3-gen.cpp b/tools/mtmd/models/zonos3-gen.cpp new file mode 100644 index 000000000..d306f9858 --- /dev/null +++ b/tools/mtmd/models/zonos3-gen.cpp @@ -0,0 +1,207 @@ +#include "models.h" + +// GEN_CODE: hidden state -> logits of the 10 codebooks (out_feats), or codes -> backbone row (graph output) +ggml_cgraph * mtmd_graph_zonos3_gen::build_gen_code() { + ggml_tensor * embd_w = model.gen_code_out_embd_w; // [n_embd, n_vocab * n_codebooks] + const int64_t n_vocab = embd_w->ne[1] / n_codebooks; + + if (!all_known) { + ggml_tensor * h = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd); + ggml_set_name(h, "inp_raw"); + ggml_set_input(h); + + ggml_tensor * logits = ggml_mul_mat(ctx0, model.gen_code_head_w, h); + logits = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, logits, 1.0f / softcap)), softcap); + logits = ggml_reshape_2d(ctx0, logits, n_vocab, n_codebooks); + ggml_set_name(logits, "out_feats"); + ggml_set_output(logits); + ggml_build_forward_expand(gf, logits); + } + + ggml_tensor * inp_codes = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_codebooks); + ggml_set_name(inp_codes, "inp_codes"); + ggml_set_input(inp_codes); + + ggml_tensor * embd = ggml_get_rows(ctx0, embd_w, inp_codes); // [n_embd, n_codebooks] + embd = ggml_sum_rows(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, embd))); + embd = ggml_reshape_2d(ctx0, embd, n_mmproj_embd, 1); + cb(embd, "gen_embd", -1); + ggml_build_forward_expand(gf, embd); + + return gf; +} + +std::vector list_zonos3_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { + GGML_UNUSED(hparams); + const auto & z = model.zonos3; + std::vector slots; + slots.push_back({"pos", 1, 1}); + for (size_t i = 0; i < z.sem.size(); i++) { + slots.push_back({"sem_conv_" + std::to_string(i), z.sem[i].dw_w->ne[0] - 1, z.sem[i].dw_w->ne[1]}); + } + for (size_t s = 0; s < z.stages.size(); s++) { + const int64_t d = z.stages[s].layers.empty() ? 0 : z.stages[s].layers[0].wo_w->ne[1]; + for (size_t il = 0; il < z.stages[s].layers.size(); il++) { + const std::string sfx = std::to_string(s) + "_" + std::to_string(il); + slots.push_back({"tfm_k_" + sfx, d, mtmd_graph_zonos3_gen::stage_ctx[s] - 1}); + slots.push_back({"tfm_v_" + sfx, d, mtmd_graph_zonos3_gen::stage_ctx[s] - 1}); + } + } + for (size_t i = 0; i < z.head.size(); i++) { + slots.push_back({"head_conv_" + std::to_string(i), z.head[i].dw_w->ne[0] - 1, z.head[i].dw_w->ne[1]}); + } + return slots; +} + +static ggml_tensor * zonos3_layer_norm(ggml_context * ctx, ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, float eps) { + return ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, x, eps), w), b); +} + +// causal ConvNeXt block, the conv history is kept in the state +ggml_tensor * mtmd_graph_zonos3_gen::convnext(ggml_tensor * x, const mtmd_zonos3_wav::convnext & b, const std::string & state_name) { + const int64_t C = x->ne[0]; + const int64_t T = x->ne[1]; + const int64_t K = b.dw_w->ne[0]; + + ggml_tensor * hist = state_in.at(state_name); // [K - 1, C] + ggml_tensor * xt = ggml_concat(ctx0, hist, ggml_cont(ctx0, ggml_transpose(ctx0, x)), 0); // [K - 1 + T, C] + state_out.push_back({state_name, ggml_cont(ctx0, ggml_view_2d(ctx0, xt, K - 1, C, xt->nb[1], (size_t) T * xt->nb[0]))}); + + ggml_tensor * h = ggml_ssm_conv(ctx0, ggml_reshape_3d(ctx0, xt, K - 1 + T, C, 1), b.dw_w); // [C, T, 1] + h = ggml_add(ctx0, ggml_reshape_2d(ctx0, h, C, T), b.dw_b); + h = zonos3_layer_norm(ctx0, h, b.norm_w, b.norm_b, convnext_eps); + h = ggml_gelu_erf(ctx0, ggml_add(ctx0, ggml_mul_mat(ctx0, b.pw1_w, h), b.pw1_b)); + h = ggml_add(ctx0, ggml_mul_mat(ctx0, b.pw2_w, h), b.pw2_b); + return ggml_add(ctx0, x, h); +} + +// windowed causal MOSS layer, adapted from qwen3tts code2wav tfm_layer_forward() +ggml_tensor * mtmd_graph_zonos3_gen::moss_layer(ggml_tensor * x, const mtmd_zonos3_wav::layer & l, int s, int il, ggml_tensor * base) { + const int64_t d = x->ne[0]; + const int64_t N = x->ne[1]; + const int64_t n_head = d / head_dim; + const int64_t W = stage_ctx[s]; + const int64_t prefix = W - 1; + const int64_t n_kv = prefix + N; + const std::string sfx = std::to_string(s) + "_" + std::to_string(il); + + ggml_tensor * h = zonos3_layer_norm(ctx0, x, l.norm1_w, l.norm1_b, ln_eps); + ggml_tensor * qkv = ggml_mul_mat(ctx0, l.qkv_w, h); // [3d, N] + auto section = [&](int i) { + return ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, head_dim, n_head, N, ggml_row_size(qkv->type, head_dim), qkv->nb[1], + ggml_row_size(qkv->type, i * d))); + }; + + ggml_tensor * pos = ggml_cast(ctx0, ggml_add(ctx0, ggml_arange(ctx0, 0.0f, (float) N, 1.0f), base), GGML_TYPE_I32); + ggml_tensor * q = ggml_rope_ext(ctx0, section(0), pos, nullptr, head_dim, GGML_ROPE_TYPE_NORMAL, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + ggml_tensor * k = ggml_rope_ext(ctx0, section(1), pos, nullptr, head_dim, GGML_ROPE_TYPE_NORMAL, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + ggml_tensor * v = section(2); + + ggml_tensor * k_full = ggml_concat(ctx0, state_in.at("tfm_k_" + sfx), ggml_reshape_2d(ctx0, k, d, N), 1); // [d, n_kv] + ggml_tensor * v_full = ggml_concat(ctx0, state_in.at("tfm_v_" + sfx), ggml_reshape_2d(ctx0, v, d, N), 1); + state_out.push_back({"tfm_k_" + sfx, ggml_cont(ctx0, ggml_view_2d(ctx0, k_full, d, prefix, k_full->nb[1], (size_t) N * k_full->nb[1]))}); + state_out.push_back({"tfm_v_" + sfx, ggml_cont(ctx0, ggml_view_2d(ctx0, v_full, d, prefix, v_full->nb[1], (size_t) N * v_full->nb[1]))}); + + // key j is visible to query i iff 0 <= (prefix + i) - j < W and j >= prefix - base (cold start) + ggml_tensor * pos_k = ggml_reshape_2d(ctx0, ggml_arange(ctx0, 0.0f, (float) n_kv, 1.0f), n_kv, 1); + ggml_tensor * pos_q = ggml_reshape_2d(ctx0, ggml_arange(ctx0, (float) prefix, (float) (prefix + N), 1.0f), 1, N); + ggml_tensor * diff = ggml_sub(ctx0, ggml_repeat_4d(ctx0, pos_q, n_kv, N, 1, 1), pos_k); // [n_kv, N] + ggml_tensor * keep = ggml_mul(ctx0, ggml_step(ctx0, ggml_scale_bias(ctx0, diff, 1.0f, 0.5f)), + ggml_step(ctx0, ggml_scale_bias(ctx0, diff, -1.0f, (float) W - 0.5f))); + keep = ggml_mul(ctx0, keep, ggml_step(ctx0, ggml_scale_bias(ctx0, ggml_add(ctx0, pos_k, base), 1.0f, 0.5f - (float) prefix))); + ggml_tensor * mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), n_kv, N, 1, 1); // 0 = keep, -inf = masked + if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) { + mask = ggml_cast(ctx0, mask, GGML_TYPE_F16); + } + + ggml_tensor * attn = build_attn(l.wo_w, nullptr, + ggml_reshape_4d(ctx0, q, head_dim, n_head, N, 1), + ggml_reshape_4d(ctx0, k_full, head_dim, n_head, n_kv, 1), + ggml_reshape_4d(ctx0, v_full, head_dim, n_head, n_kv, 1), + mask, 1.0f / sqrtf((float) head_dim), il); + x = ggml_add(ctx0, x, ggml_mul(ctx0, attn, l.attn_scale)); + + h = zonos3_layer_norm(ctx0, x, l.norm2_w, l.norm2_b, ln_eps); + h = ggml_mul_mat(ctx0, l.ffn2_w, ggml_gelu_erf(ctx0, ggml_mul_mat(ctx0, l.ffn1_w, h))); + return ggml_add(ctx0, x, ggml_mul(ctx0, h, l.ffn_scale)); +} + +ggml_cgraph * mtmd_graph_zonos3_gen::build_gen_wav() { + const auto & z = model.zonos3; + const int64_t T = n_frames; + + for (const auto & slot : list_zonos3_gen_state_slots(hparams, model)) { + ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1); + ggml_set_name(t, ("state_in_" + slot.name).c_str()); + ggml_set_input(t); + state_in[slot.name] = t; + } + + // codebook-major, residual codes are rows of the merged table + ggml_tensor * inp_codes = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, T * n_codebooks); + ggml_set_name(inp_codes, "inp_codes"); + ggml_set_input(inp_codes); + auto codes_of = [&](int c) { + return ggml_view_1d(ctx0, inp_codes, T, (size_t) c * T * ggml_element_size(inp_codes)); + }; + + // 1 for a known residual code, 0 for a missing one (early decode of a frame), codebook-major + ggml_tensor * inp_keep = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, T * (n_codebooks - 1)); + ggml_set_name(inp_keep, "inp_keep"); + ggml_set_input(inp_keep); + + ggml_tensor * x = ggml_add(ctx0, ggml_get_rows(ctx0, z.sem_table, codes_of(0)), z.sem_bias); // [semantic_dim, T] + for (size_t i = 0; i < z.sem.size(); i++) { + x = convnext(x, z.sem[i], "sem_conv_" + std::to_string(i)); + } + x = ggml_add(ctx0, ggml_mul_mat(ctx0, z.sem_out_w, x), z.sem_out_b); // [latent, T] + for (int c = 1; c < n_codebooks; c++) { + ggml_tensor * bias = ggml_view_1d(ctx0, z.res_bias, z.res_bias->ne[0], (size_t) (c - 1) * z.res_bias->nb[1]); + ggml_tensor * keep = ggml_view_2d(ctx0, inp_keep, 1, T, ggml_element_size(inp_keep), (size_t) (c - 1) * T * ggml_element_size(inp_keep)); + x = ggml_add(ctx0, x, ggml_mul(ctx0, ggml_add(ctx0, ggml_get_rows(ctx0, z.res_table, codes_of(c)), bias), keep)); + } + cb(x, "zac_latent", -1); + + ggml_tensor * frames = ggml_reshape_1d(ctx0, state_in.at("pos"), 1); + int cpf = 1; // positions per frame + for (int s = 0; s < n_stages; s++) { + const auto & st = z.stages[s]; + x = ggml_mul_mat(ctx0, st.in_w, x); + ggml_tensor * base = ggml_scale(ctx0, frames, (float) cpf); + for (size_t il = 0; il < st.layers.size(); il++) { + x = moss_layer(x, st.layers[il], s, (int) il, base); + } + if (st.out_w) { + x = ggml_mul_mat(ctx0, st.out_w, x); + } + if (s + 1 < n_stages) { + // patch up x2: out[ch, 2t + j] = in[2ch + j, t] + const int64_t C = x->ne[0], N = x->ne[1]; + x = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, x, 2, C / 2, N), 1, 0, 2, 3)); + x = ggml_reshape_2d(ctx0, x, C / 2, 2 * N); + cpf *= 2; + } + cb(x, "zac_stage_out", s); + } + state_out.push_back({"pos", ggml_scale_bias(ctx0, state_in.at("pos"), 1.0f, (float) T)}); + + for (size_t i = 0; i < z.head.size(); i++) { + x = convnext(x, z.head[i], "head_conv_" + std::to_string(i)); + } + x = zonos3_layer_norm(ctx0, x, z.head_norm_w, z.head_norm_b, convnext_eps); + ggml_tensor * spec = ggml_add(ctx0, ggml_mul_mat(ctx0, z.head_out_w, x), z.head_out_b); // [n_fft0 + 2 + n_fft1 + 2, 16T] + ggml_set_name(spec, "out_feats"); + ggml_set_output(spec); + ggml_build_forward_expand(gf, spec); + + for (auto & slot : state_out) { + ggml_set_name(slot.second, ("state_out_" + slot.first).c_str()); + ggml_set_output(slot.second); + ggml_build_forward_expand(gf, slot.second); + } + return gf; +} + +ggml_cgraph * mtmd_graph_zonos3_gen::build() { + return gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV ? build_gen_wav() : build_gen_code(); +} diff --git a/tools/mtmd/models/zonos3-spkenc.cpp b/tools/mtmd/models/zonos3-spkenc.cpp new file mode 100644 index 000000000..94a388cd7 --- /dev/null +++ b/tools/mtmd/models/zonos3-spkenc.cpp @@ -0,0 +1,221 @@ +#include "models.h" + +// zonos3 speaker encoder: MECT-B2 + ASTP pooling (ownmect_s2), then the speaker projection of the backbone +// ported from the reference ggml runtime (spk-encoder.cpp) + +namespace { + +constexpr int kHidden = 64; +constexpr int kFeat = 2560; +constexpr int kHeads = 4; +constexpr int kExperts = 4; +constexpr int kPosK = 113; + +// per block: cnn depth, stride f, stride t +struct mect_block { int cnn, s_f, s_t; }; +const std::vector> & mect_b2() { + static const std::vector> cfg = { + { {3, 1, 1}, {3, 1, 1} }, + { {3, 2, 1}, {3, 1, 2}, {3, 1, 2}, {3, 1, 2} }, + { {3, 2, 1}, {3, 1, 2}, {3, 1, 2}, {3, 1, 2}, {3, 1, 2} }, + { {3, 2, 1}, {2, 1, 2}, {2, 1, 2} }, + }; + return cfg; +} + +} // namespace + +ggml_tensor * mtmd_graph_zonos3_spkenc::W(const std::string & name) const { + return model.zonos3_spk.at("a.spk." + name); +} + +bool mtmd_graph_zonos3_spkenc::has(const std::string & name) const { + return model.zonos3_spk.count("a.spk." + name) > 0; +} + +// [C] as a per-channel broadcast over a [T, F, C] activation +ggml_tensor * mtmd_graph_zonos3_spkenc::chan(ggml_tensor * v) const { + return ggml_reshape_4d(ctx0, v, 1, 1, v->ne[0], 1); +} + +ggml_tensor * mtmd_graph_zonos3_spkenc::bn(ggml_tensor * x, const std::string & p) const { + return ggml_add(ctx0, ggml_mul(ctx0, x, chan(W(p + ".scale"))), chan(W(p + ".shift"))); +} + +ggml_tensor * mtmd_graph_zonos3_spkenc::conv(ggml_tensor * x, const std::string & p, int s_t, int s_f, bool bias, int pad) const { + ggml_tensor * y = ggml_conv_2d_direct(ctx0, W(p + ".weight"), x, s_t, s_f, pad, pad, 1, 1); + return bias ? ggml_add(ctx0, y, chan(W(p + ".bias"))) : y; +} + +// grouped 3x3 conv as one depthwise conv per input slot of a group +ggml_tensor * mtmd_graph_zonos3_spkenc::grouped3x3(ggml_tensor * x, const std::string & p, int s_t, int s_f) const { + int n_slot = 0; + while (has(p + ".w" + std::to_string(n_slot))) { + n_slot++; + } + const int64_t T = x->ne[0]; + const int64_t F = x->ne[1]; + const int64_t IC = x->ne[2]; + const int64_t OC = W(p + ".w0")->ne[3]; + const int64_t G = IC / n_slot; + const int64_t B = OC / G; + ggml_tensor * x4 = ggml_reshape_4d(ctx0, x, T, F, n_slot, G); + ggml_tensor * y = nullptr; + for (int j = 0; j < n_slot; j++) { + ggml_tensor * xj = x; + if (n_slot > 1 || B > 1) { + ggml_tensor * slot = ggml_view_4d(ctx0, x4, T, F, 1, G, x4->nb[1], x4->nb[2], x4->nb[3], (size_t) j * x4->nb[2]); + xj = ggml_reshape_3d(ctx0, ggml_repeat_4d(ctx0, slot, T, F, B, G), T, F, OC); + } + ggml_tensor * t = ggml_conv_2d_dw_direct(ctx0, W(p + ".w" + std::to_string(j)), xj, s_t, s_f, 1, 1, 1, 1); + y = y ? ggml_add(ctx0, y, t) : t; + } + return y; +} + +ggml_tensor * mtmd_graph_zonos3_spkenc::res_block(ggml_tensor * x, const std::string & q, int s_f, int s_t) const { + ggml_tensor * t = grouped3x3(x, q + "conv1", s_t, s_f); + t = ggml_relu(ctx0, bn(conv(t, q + "pw1", 1, 1, true, 0), q + "bn1")); + t = grouped3x3(t, q + "conv2", 1, 1); + t = ggml_relu(ctx0, bn(conv(t, q + "pw2", 1, 1, true, 0), q + "bn2")); + ggml_tensor * sc = x; + if (has(q + "sc.weight")) { + sc = bn(conv(x, q + "sc", s_t, s_f, false, 0), q + "sc.bn"); + } + return ggml_add(ctx0, t, sc); +} + +ggml_tensor * mtmd_graph_zonos3_spkenc::linear(ggml_tensor * x, const std::string & p) const { + return ggml_add(ctx0, ggml_mul_mat(ctx0, W(p + ".weight"), x), W(p + ".bias")); +} + +ggml_tensor * mtmd_graph_zonos3_spkenc::layer_norm(ggml_tensor * x, const std::string & p) const { + return ggml_add(ctx0, ggml_mul(ctx0, ggml_norm(ctx0, x, 1e-6f), W(p + ".weight")), W(p + ".bias")); +} + +// [T, F, C] -> [F*C, T], k = f*C + c +ggml_tensor * mtmd_graph_zonos3_spkenc::flatten(ggml_tensor * x) const { + const int64_t T = x->ne[0], F = x->ne[1], C = x->ne[2]; + return ggml_reshape_2d(ctx0, ggml_cont(ctx0, ggml_permute(ctx0, x, 2, 1, 0, 3)), F * C, T); +} + +ggml_tensor * mtmd_graph_zonos3_spkenc::unflatten(ggml_tensor * x, int64_t F, int64_t C) const { + return ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, x, C, F, x->ne[1]), 2, 1, 0, 3)); +} + +ggml_tensor * mtmd_graph_zonos3_spkenc::attention(ggml_tensor * h, const std::string & p) const { + const int64_t T = h->ne[1]; + const int hd = kHidden / kHeads; + ggml_tensor * q = ggml_scale(ctx0, linear(h, p + "attn.q"), 1.0f / sqrtf((float) hd)); + ggml_tensor * k = linear(h, p + "attn.k"); + ggml_tensor * v = linear(h, p + "attn.v"); + auto heads = [&](ggml_tensor * z) { // [64, T] -> [hd, T, n_head] + return ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, z, hd, kHeads, T), 0, 2, 1, 3)); + }; + ggml_tensor * qp = heads(q); + ggml_tensor * kp = heads(k); + ggml_tensor * vt = ggml_cont(ctx0, ggml_permute(ctx0, heads(v), 1, 0, 2, 3)); // [T, hd, n_head] + // queries in blocks, so long clips do not need a T x T score matrix per head + ggml_tensor * o = nullptr; + for (int64_t a = 0; a < T; a += 512) { + const int64_t nq = std::min(512, T - a); + ggml_tensor * qb = ggml_view_3d(ctx0, qp, hd, nq, kHeads, qp->nb[1], qp->nb[2], (size_t) a * qp->nb[1]); + ggml_tensor * score = ggml_soft_max(ctx0, ggml_mul_mat(ctx0, kp, qb)); // [T, nq, n_head] + ggml_tensor * ob = ggml_mul_mat(ctx0, vt, score); // [hd, nq, n_head] + ob = ggml_reshape_2d(ctx0, ggml_cont(ctx0, ggml_permute(ctx0, ob, 0, 2, 1, 3)), kHidden, nq); + o = o ? ggml_concat(ctx0, o, ob, 1) : ob; + } + return linear(o, p + "attn.o"); +} + +// soft MoE: all experts run, weighted by the router +ggml_tensor * mtmd_graph_zonos3_spkenc::moe_ffn(ggml_tensor * h, const std::string & p) const { + ggml_tensor * r = ggml_soft_max(ctx0, ggml_mul_mat(ctx0, W(p + "moe.gate.weight"), h)); // [n_expert, T] + ggml_tensor * e = ggml_gelu(ctx0, ggml_mul_mat(ctx0, W(p + "moe.experts.weight"), h)); // [64 * n_expert, T] + ggml_tensor * mix = nullptr; + for (int i = 0; i < kExperts; i++) { + ggml_tensor * ei = ggml_view_2d(ctx0, e, kHidden, e->ne[1], e->nb[1], (size_t) i * kHidden * ggml_element_size(e)); + ggml_tensor * ri = ggml_view_2d(ctx0, r, 1, r->ne[1], r->nb[1], (size_t) i * ggml_element_size(r)); + ggml_tensor * t = ggml_mul(ctx0, ei, ri); + mix = mix ? ggml_add(ctx0, mix, t) : t; + } + return linear(ggml_gelu(ctx0, mix), p + "ffn_out"); +} + +ggml_tensor * mtmd_graph_zonos3_spkenc::pos_conv(ggml_tensor * h, const std::string & p) const { + const int64_t T = h->ne[1]; + ggml_tensor * ht = ggml_cont(ctx0, ggml_transpose(ctx0, h)); // [T, 64] + ht = ggml_pad_ext(ctx0, ht, kPosK / 2, kPosK / 2, 0, 0, 0, 0, 0, 0); // [T + K - 1, 64] + ggml_tensor * acc = ggml_ssm_conv(ctx0, ggml_reshape_3d(ctx0, ht, ht->ne[0], kHidden, 1), W(p + "pos.dw.weight")); // [64, T, 1] + acc = ggml_add(ctx0, ggml_reshape_2d(ctx0, acc, kHidden, T), W(p + "pos.dw.bias")); + acc = ggml_add(ctx0, ggml_mul(ctx0, acc, W(p + "pos.bn.scale")), W(p + "pos.bn.shift")); + acc = ggml_gelu_erf(ctx0, acc); + return linear(acc, p + "pos.pw"); +} + +ggml_cgraph * mtmd_graph_zonos3_spkenc::build() { + const int64_t T = img.nx(); + const int64_t n_mel = img.ny(); + + ggml_tensor * inp = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, T, n_mel, 1, 1); + ggml_set_name(inp, "inp_raw"); + ggml_set_input(inp); + + ggml_tensor * x = bn(conv(inp, "fe.head.conv", 1, 1, true, 1), "fe.head.bn"); // [T, F, C] + ggml_tensor * xf = flatten(x); + std::vector feats; + const auto & stages = mect_b2(); + for (size_t s = 0; s < stages.size(); s++) { + for (size_t k = 0; k < stages[s].size(); k++) { + const auto & b = stages[s][k]; + const std::string p = "fe.b" + std::to_string(s + 1) + "." + std::to_string(k) + "."; + ggml_tensor * y = x; + for (int r = 0; r < b.cnn; r++) { + y = res_block(y, p + "cnn." + std::to_string(r) + ".", r == 0 ? b.s_f : 1, r == 0 ? b.s_t : 1); + } + if (has(p + "reduce.weight")) { + y = bn(conv(y, p + "reduce", 1, 1, true, 0), p + "reduce.bn"); + } + const int64_t F2 = y->ne[1], C2 = y->ne[2]; + ggml_tensor * x2 = flatten(y); // [2560, T'] + ggml_tensor * h = layer_norm(linear(x2, p + "rproj"), p + "rproj.ln"); + h = ggml_add(ctx0, h, pos_conv(h, p)); + h = layer_norm(ggml_add(ctx0, h, attention(h, p)), p + "ln1"); + h = layer_norm(ggml_add(ctx0, h, moe_ffn(h, p)), p + "ln2"); + ggml_tensor * e = ggml_add(ctx0, linear(h, p + "eproj"), x2); + if (b.s_t > 1) { + // nearest upsampling back to the input frames + const int64_t Tp = e->ne[1]; + e = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, e, kFeat, 1, Tp), kFeat, b.s_t, Tp, 1); + e = ggml_view_2d(ctx0, ggml_reshape_2d(ctx0, e, kFeat, (int64_t) b.s_t * Tp), kFeat, T, e->nb[1], 0); + } + xf = ggml_add(ctx0, e, xf); + x = unflatten(xf, F2, C2); + } + feats.push_back(xf); + } + + ggml_tensor * dense = W("fe.dense_w"); // [2560, n_stage] + ggml_tensor * fused = nullptr; + for (size_t s = 0; s < feats.size(); s++) { + ggml_tensor * t = ggml_mul(ctx0, feats[s], ggml_view_2d(ctx0, dense, kFeat, 1, dense->nb[1], s * dense->nb[1])); + fused = fused ? ggml_add(ctx0, fused, t) : t; + } + + // ASTP: attention-weighted mean and std over time + ggml_tensor * att = linear(ggml_tanh(ctx0, linear(fused, "pool.l1")), "pool.l2"); // [2560, T] + ggml_tensor * alpha = ggml_soft_max(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, att))); // [T, 2560] + ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, fused)); // [T, 2560] + ggml_tensor * mean = ggml_sum_rows(ctx0, ggml_mul(ctx0, alpha, xt)); // [1, 2560] + ggml_tensor * sq = ggml_sum_rows(ctx0, ggml_mul(ctx0, alpha, ggml_mul(ctx0, xt, xt))); + ggml_tensor * var = ggml_sub(ctx0, sq, ggml_mul(ctx0, mean, mean)); + ggml_tensor * stdv = ggml_sqrt(ctx0, ggml_clamp(ctx0, var, 1e-7f, 3.0e38f)); + ggml_tensor * pooled = ggml_concat(ctx0, ggml_reshape_2d(ctx0, mean, kFeat, 1), ggml_reshape_2d(ctx0, stdv, kFeat, 1), 0); + pooled = ggml_add(ctx0, ggml_mul(ctx0, pooled, W("bn.scale")), W("bn.shift")); + ggml_tensor * spk = ggml_mul_mat(ctx0, W("linear.weight"), pooled); // [192, 1], not normalized + cb(spk, "spk_embd", -1); + + ggml_tensor * out = linear(spk, "proj"); // speaker row of the prompt + ggml_build_forward_expand(gf, out); + return gf; +} diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index 33c26c31f..c6d751b41 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -1176,45 +1176,102 @@ static float mtmd_audio_lufs(const float * samples, size_t n_samples, int sample // mtmd_audio_preprocessor_chatterbox_ref // -void mtmd_audio_preprocessor_chatterbox_ref::initialize() { - // kaldi fbank: povey window, kaldi mel banks over the 512-point spectrum, the nyquist bin left out - { - const int frame_len = 400; - const int n_fft = 512; - const int n_bins = n_fft / 2 + 1; - const int n_mel = hparams.n_mel_bins; - const double sr = (double) hparams.audio_sample_rate; +// torchaudio kaldi.fbank at 16 kHz, the nyquist bin is left out of the mel banks +static constexpr int kaldi_frame_len = 400; +static constexpr int kaldi_hop = 160; +static constexpr int kaldi_n_fft = 512; - fbank_cache.fill_sin_cos_table(n_fft); - fbank_cache.hann_window.resize(frame_len); - for (int i = 0; i < frame_len; i++) { - fbank_cache.hann_window[(size_t) i] = (float) pow(0.5 - 0.5 * cos(2.0 * M_PI * i / (frame_len - 1)), 0.85); - } +static void mtmd_audio_kaldi_fbank_init(mtmd_audio_cache & cache, int n_mel, double sr, bool povey) { + const int n_bins = kaldi_n_fft / 2 + 1; - auto mel = [](double f) { return 1127.0 * log(1.0 + f / 700.0); }; - const double mel_lo = mel(20.0); - const double mel_hi = mel(sr / 2.0); - const double delta = (mel_hi - mel_lo) / (n_mel + 1); - const double bin_hz = sr / n_fft; + cache.fill_sin_cos_table(kaldi_n_fft); + cache.hann_window.resize(kaldi_frame_len); + for (int i = 0; i < kaldi_frame_len; i++) { + const double c = cos(2.0 * M_PI * i / (kaldi_frame_len - 1)); + cache.hann_window[(size_t) i] = (float) (povey ? pow(0.5 - 0.5 * c, 0.85) : 0.54 - 0.46 * c); + } - auto & filters = fbank_cache.filters; - filters.n_mel = n_mel; - filters.n_fft = n_fft; - filters.data.assign((size_t) n_mel * n_bins, 0.0f); - for (int m = 0; m < n_mel; m++) { - const double left = mel_lo + m * delta; - const double center = left + delta; - const double right = center + delta; - for (int i = 0; i < n_bins - 1; i++) { - const double f = mel(bin_hz * i); - if (f > left && f < right) { - const double w = f <= center ? (f - left) / (center - left) - : (right - f) / (right - center); - filters.data[(size_t) m * n_bins + i] = (float) w; - } + auto mel = [](double f) { return 1127.0 * log(1.0 + f / 700.0); }; + const double mel_lo = mel(20.0); + const double mel_hi = mel(sr / 2.0); + const double delta = (mel_hi - mel_lo) / (n_mel + 1); + const double bin_hz = sr / kaldi_n_fft; + + auto & filters = cache.filters; + filters.n_mel = n_mel; + filters.n_fft = kaldi_n_fft; + filters.data.assign((size_t) n_mel * n_bins, 0.0f); + for (int m = 0; m < n_mel; m++) { + const double left = mel_lo + m * delta; + const double center = left + delta; + const double right = center + delta; + for (int i = 0; i < n_bins - 1; i++) { + const double f = mel(bin_hz * i); + if (f > left && f < right) { + const double w = f <= center ? (f - left) / (center - left) + : (right - f) / (right - center); + filters.data[(size_t) m * n_bins + i] = (float) w; } } } +} + +// snip edges, dc removal, preemphasis 0.97, log mel energies mean subtracted over time; out is mel major +static bool mtmd_audio_kaldi_fbank(const mtmd_audio_cache & cache, const float * pcm, size_t n_samples, int n_mel, float scale, + mtmd_audio_mel & out) { + if ((int) n_samples < kaldi_frame_len) { + return false; + } + const int n_frames = 1 + ((int) n_samples - kaldi_frame_len) / kaldi_hop; + + out.n_len = n_frames; + out.n_len_org = n_frames; + out.n_mel = n_mel; + out.data.assign((size_t) n_mel * n_frames, 0.0f); + + const std::vector & win = cache.hann_window; + std::vector fft_in((size_t) kaldi_n_fft * 2), fft_out((size_t) kaldi_n_fft * 8); + std::vector e; + std::vector x(kaldi_frame_len); + for (int fr = 0; fr < n_frames; fr++) { + for (int i = 0; i < kaldi_frame_len; i++) { + x[(size_t) i] = pcm[(size_t) fr * kaldi_hop + i] * scale; + } + + double mean = 0.0; + for (int i = 0; i < kaldi_frame_len; i++) { + mean += x[i]; + } + mean /= kaldi_frame_len; + + std::fill(fft_in.begin(), fft_in.end(), 0.0f); + fft_in[0] = (float) ((x[0] - mean) * (1.0 - 0.97) * win[0]); + for (int i = 1; i < kaldi_frame_len; i++) { + fft_in[(size_t) i] = (float) (((x[i] - mean) - 0.97 * (x[i - 1] - mean)) * win[(size_t) i]); + } + mtmd_audio_mel_energies(cache, fft_in, fft_out, false, e); + for (int m = 0; m < n_mel; m++) { + out.data[(size_t) m * n_frames + fr] = (float) log(std::max(e[(size_t) m], (double) FLT_EPSILON)); + } + } + + for (int m = 0; m < n_mel; m++) { + float * row = out.data.data() + (size_t) m * n_frames; + double mean = 0.0; + for (int fr = 0; fr < n_frames; fr++) { + mean += row[fr]; + } + mean /= n_frames; + for (int fr = 0; fr < n_frames; fr++) { + row[fr] -= (float) mean; + } + } + return true; +} + +void mtmd_audio_preprocessor_chatterbox_ref::initialize() { + // kaldi fbank of CAMPPlus: povey window + mtmd_audio_kaldi_fbank_init(fbank_cache, hparams.n_mel_bins, (double) hparams.audio_sample_rate, true); // s3 tokenizer: librosa slaney mel, 128 bands s3tok_cache.fill_sin_cos_table(400); @@ -1253,59 +1310,12 @@ bool mtmd_audio_preprocessor_chatterbox_ref::process(const float * } } - // entry 0: torchaudio kaldi.fbank of CAMPPlus (snip edges, dc removal, preemphasis 0.97), mean subtracted over time + // entry 0: torchaudio kaldi.fbank of CAMPPlus { - const int frame_len = 400; - const int hop = 160; - const int n_fft = 512; - const int n_mel = hparams.n_mel_bins; - - if ((int) pcm.size() < frame_len) { + mtmd_audio_mel out; + if (!mtmd_audio_kaldi_fbank(fbank_cache, pcm.data(), pcm.size(), hparams.n_mel_bins, 1.0f, out)) { return false; } - const int n_frames = 1 + ((int) pcm.size() - frame_len) / hop; - - mtmd_audio_mel out; - out.n_len = n_frames; - out.n_len_org = n_frames; - out.n_mel = n_mel; - out.data.assign((size_t) n_mel * n_frames, 0.0f); - - const std::vector & win = fbank_cache.hann_window; - std::vector fft_in((size_t) n_fft * 2), fft_out((size_t) n_fft * 8); - std::vector e; - for (int fr = 0; fr < n_frames; fr++) { - const float * x = pcm.data() + (size_t) fr * hop; - - double mean = 0.0; - for (int i = 0; i < frame_len; i++) { - mean += x[i]; - } - mean /= frame_len; - - std::fill(fft_in.begin(), fft_in.end(), 0.0f); - fft_in[0] = (float) ((x[0] - mean) * (1.0 - 0.97) * win[0]); - for (int i = 1; i < frame_len; i++) { - fft_in[(size_t) i] = (float) (((x[i] - mean) - 0.97 * (x[i - 1] - mean)) * win[(size_t) i]); - } - mtmd_audio_mel_energies(fbank_cache, fft_in, fft_out, false, e); - for (int m = 0; m < n_mel; m++) { - out.data[(size_t) m * n_frames + fr] = (float) log(std::max(e[(size_t) m], (double) FLT_EPSILON)); - } - } - - for (int m = 0; m < n_mel; m++) { - float * row = out.data.data() + (size_t) m * n_frames; - double mean = 0.0; - for (int fr = 0; fr < n_frames; fr++) { - mean += row[fr]; - } - mean /= n_frames; - for (int fr = 0; fr < n_frames; fr++) { - row[fr] -= (float) mean; - } - } - output.push_back(std::move(out)); } @@ -1883,6 +1893,197 @@ bool mtmd_audio_preprocessor_gemma4ua::process(const float * sam return true; } +// +// mtmd_audio_preprocessor_zonos3_spk +// + +void mtmd_audio_preprocessor_zonos3_spk::initialize() { + mtmd_audio_kaldi_fbank_init(cache, hparams.n_mel_bins, (double) hparams.audio_sample_rate, false); +} + +bool mtmd_audio_preprocessor_zonos3_spk::process(const float * samples, size_t n_samples, std::vector & output, + bool /*shape_only*/) const { + // the reference uses the first 60 s, on the int16 sample scale + const size_t n = std::min(n_samples, (size_t) hparams.audio_sample_rate * 60); + mtmd_audio_mel out; + if (!mtmd_audio_kaldi_fbank(cache, samples, n, hparams.n_mel_bins, 32768.0f, out)) { + return false; + } + output.clear(); + output.push_back(std::move(out)); + return true; +} + +// +// mtmd_audio_zonos3_synth implementation +// + +mtmd_audio_zonos3_synth::mtmd_audio_zonos3_synth(const std::vector & mixer, const std::vector & resample, + int n_fft0, int n_fft1, int hop, int rs_orig, int rs_new) + : mixer(mixer), fir_kernel((int) (mixer.size() / 2)), hop(hop), rs_kernel(resample), rs_orig(rs_orig), rs_new(rs_new) { + GGML_ASSERT(mixer.size() % 2 == 0 && fir_kernel % 2 == 1); + GGML_ASSERT(resample.size() % (size_t) rs_new == 0); + const int rs_len = (int) (resample.size() / rs_new); + GGML_ASSERT(rs_len >= rs_orig && (rs_len - rs_orig) % 2 == 0); + rs_width = (rs_len - rs_orig) / 2; + + const int n_fft[2] = { n_fft0, n_fft1 }; + const int h = (fir_kernel - 1) / 2; + for (int i = 0; i < 2; i++) { + branch & b = br[i]; + b.n_fft = n_fft[i]; + b.cache.fill_sin_cos_table(b.n_fft); + b.cache.fill_hann_window(b.n_fft, true); + b.ifft_in.assign((size_t) b.n_fft * 2 * 4, 0.0f); + b.ifft_out.assign((size_t) b.n_fft * 2 * 4, 0.0f); + b.mix_in.assign((size_t) h, 0.0f); + b.mix_avail = h; + } +} + +// the first n_fft / 2 samples are the centering pad +void mtmd_audio_zonos3_synth::ola_finalize(branch & b, int64_t pf) { + const int64_t pad = b.n_fft / 2; + const int64_t n = std::min(pf - b.ola_base, (int64_t) b.ola.size()); + if (n <= 0) { + return; + } + for (int64_t j = 0; j < n; j++) { + if (b.ola_base + j < pad) { + continue; + } + b.mix_in.push_back(b.env[(size_t) j] > 1e-11f ? b.ola[(size_t) j] / b.env[(size_t) j] : 0.0f); + b.mix_avail++; + } + b.ola.erase(b.ola.begin(), b.ola.begin() + n); + b.env.erase(b.env.begin(), b.env.begin() + n); + b.ola_base += n; +} + +void mtmd_audio_zonos3_synth::mix_emit(std::vector & out) { + const int64_t avail = std::min(br[0].mix_avail, br[1].mix_avail); + const int64_t m1 = avail - fir_kernel + 1; + if (m1 <= mix_next) { + return; + } + const size_t n = (size_t) (m1 - mix_next); + std::vector y(n, 0.0f); + for (int c = 0; c < 2; c++) { + const float * w = mixer.data() + (size_t) c * fir_kernel; + const float * s = br[c].mix_in.data() + (mix_next - mix_base); + for (int k = 0; k < fir_kernel; k++) { + for (size_t i = 0; i < n; i++) { + y[i] += w[k] * s[i + k]; + } + } + } + mix_next = m1; + for (int c = 0; c < 2; c++) { + br[c].mix_in.erase(br[c].mix_in.begin(), br[c].mix_in.begin() + (mix_next - mix_base)); + } + mix_base = mix_next; + + rs_buf.insert(rs_buf.end(), y.begin(), y.end()); + rs_in += (int64_t) n; + resample(false, out); +} + +// on flush, the input is zero-extended and the output is ceil(new * rs_in / orig) samples +void mtmd_audio_zonos3_synth::resample(bool flush, std::vector & out) { + const int KL = 2 * rs_width + rs_orig; + const int64_t total = flush ? ((int64_t) rs_new * rs_in + rs_orig - 1) / rs_orig : -1; + std::vector xw(KL); + for (;; rs_block++) { + const int64_t x0 = rs_block * rs_orig - rs_width; + if (flush ? rs_block * rs_new >= total : x0 + KL > rs_in) { + break; + } + for (int k = 0; k < KL; k++) { + const int64_t j = x0 + k; + xw[k] = (j < 0 || j >= rs_in) ? 0.0f : rs_buf[(size_t) (j - rs_base)]; + } + const int64_t n_emit = flush ? std::min(rs_new, total - rs_block * rs_new) : rs_new; + for (int64_t i = 0; i < n_emit; i++) { + const float * ki = rs_kernel.data() + (size_t) i * KL; + float acc = 0.0f; + for (int k = 0; k < KL; k++) { + acc += ki[k] * xw[k]; + } + out.push_back(acc); + } + } + // drop the samples no later block reads + const int64_t keep_from = std::min(rs_in, std::max(0, rs_block * rs_orig - rs_width)); + if (keep_from > rs_base) { + rs_buf.erase(rs_buf.begin(), rs_buf.begin() + (keep_from - rs_base)); + rs_base = keep_from; + } +} + +void mtmd_audio_zonos3_synth::push(const float * spec, int64_t n_cols, std::vector & out) { + const int64_t n_rows = (br[0].n_fft + 2) + (br[1].n_fft + 2); + const float max_log_mag = logf(100.0f); + for (int i = 0; i < 2; i++) { + branch & b = br[i]; + const int n = b.n_fft; + const int bins = n / 2 + 1; + const int64_t row0 = i == 0 ? 0 : br[0].n_fft + 2; + for (int64_t c = 0; c < n_cols; c++) { + const float * logmag = spec + c * n_rows + row0; + const float * phase = logmag + bins; + // irfft ignores the imaginary parts of DC and Nyquist + for (int k = 0; k < bins; k++) { + const float a = expf(std::min(logmag[k], max_log_mag)); + b.ifft_in[2 * k + 0] = a * cosf(phase[k]); + b.ifft_in[2 * k + 1] = (k == 0 || k == n / 2) ? 0.0f : a * sinf(phase[k]); + } + for (int k = 1; k < n - bins + 1; k++) { + b.ifft_in[2 * (n - k) + 0] = b.ifft_in[2 * k + 0]; + b.ifft_in[2 * (n - k) + 1] = -b.ifft_in[2 * k + 1]; + } + ifft(b.cache, b.ifft_in.data(), n, b.ifft_out.data()); + + const int64_t t = n_frames + c; + const size_t need = (size_t) (t * hop + n - b.ola_base); + if (b.ola.size() < need) { + b.ola.resize(need, 0.0f); + b.env.resize(need, 0.0f); + } + float * ola = b.ola.data() + (t * hop - b.ola_base); + float * env = b.env.data() + (t * hop - b.ola_base); + const float * win = b.cache.hann_window.data(); + for (int j = 0; j < n; j++) { + ola[j] += win[j] * b.ifft_out[2 * j]; + env[j] += win[j] * win[j]; + } + } + } + n_frames += n_cols; + for (auto & b : br) { + ola_finalize(b, n_frames * hop); + } + mix_emit(out); +} + +void mtmd_audio_zonos3_synth::flush(std::vector & out) { + if (done || n_frames == 0) { + return; + } + done = true; + // the mixer has h zeros of right padding + const int h = (fir_kernel - 1) / 2; + const int64_t L = n_frames * hop; + for (auto & b : br) { + ola_finalize(b, b.ola_base + (int64_t) b.ola.size()); + const int64_t keep = h + L - mix_base; + b.mix_in.resize((size_t) keep, 0.0f); + b.mix_in.resize((size_t) (keep + h), 0.0f); + b.mix_avail = mix_base + keep + h; + } + mix_emit(out); + resample(true, out); +} + // // mtmd_audio_streaming_istft implementation // diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index 7251f82db..937451c8a 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -243,6 +243,16 @@ struct mtmd_audio_preprocessor_chatterbox_ref : mtmd_audio_preprocessor { mtmd_audio_cache ve_cache; // entry 4 }; +// zonos3 speaker encoder front-end: kaldi fbank with a hamming window [80 x n_frames] (mel major) +struct mtmd_audio_preprocessor_zonos3_spk : mtmd_audio_preprocessor { + mtmd_audio_preprocessor_zonos3_spk(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} + void initialize() override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; + + private: + mtmd_audio_cache cache; +}; + struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor { mtmd_audio_preprocessor_parakeet(const mtmd_model * model) : mtmd_audio_preprocessor(model) { } void initialize() override; @@ -261,6 +271,51 @@ struct mtmd_audio_preprocessor_nemotron_transcribe : mtmd_audio_preprocessor_par bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; }; +// +// zonos3 iSTFT head, streaming: 2 iSTFT branches, FIR mixer, 44160 -> 44100 Hz resampler +// ported from the reference ggml runtime (zac.cpp) +// +struct mtmd_audio_zonos3_synth { + // mixer [2, fir_kernel], resample [rs_new, rs_orig + 2 * width] + mtmd_audio_zonos3_synth(const std::vector & mixer, const std::vector & resample, + int n_fft0, int n_fft1, int hop, int rs_orig, int rs_new); + + // spec: n_cols frames of [log magnitude, phase] of each branch + void push(const float * spec, int64_t n_cols, std::vector & out); + void flush(std::vector & out); + + private: + struct branch { + int n_fft; + mtmd_audio_cache cache; + std::vector ola, env; // from ola_base on + int64_t ola_base = 0; + std::vector mix_in; // from mix_base on + int64_t mix_avail = 0; + std::vector ifft_in, ifft_out; + }; + + void ola_finalize(branch & b, int64_t pf); + void mix_emit(std::vector & out); + void resample(bool flush, std::vector & out); + + branch br[2]; + std::vector mixer; + int fir_kernel; + int hop; + int64_t n_frames = 0; + int64_t mix_base = 0; + int64_t mix_next = 0; + bool done = false; + + std::vector rs_kernel; + int rs_orig, rs_new, rs_width; + std::vector rs_buf; // from rs_base on + int64_t rs_base = 0; + int64_t rs_in = 0; + int64_t rs_block = 0; +}; + // // streaming ISTFT - converts spectrogram frames back to audio one frame at a time // diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp index 01c4c0ac6..ac656032d 100644 --- a/tools/mtmd/mtmd-graph.cpp +++ b/tools/mtmd/mtmd-graph.cpp @@ -960,6 +960,10 @@ std::unique_ptr mtmd_get_graph_builder(mtmd_subcontext * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_ZONOS3_SPKENC: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_POCKETTTS_SPKENC: { builder = std::make_unique(ctx, img); @@ -1015,6 +1019,16 @@ std::unique_ptr mtmd_get_graph_builder(mtmd_subcontext * ctx, const } builder = std::make_unique(ctx, img, gen_process, all_known, n_frames); } break; + case PROJECTOR_TYPE_ZONOS3_GEN: + { + const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; + const int64_t n_cb = mtmd_graph_zonos3_gen::n_codebooks; + // GEN_CODE with codes: only the backbone row of that frame is needed + const bool all_known = params && params->codes && gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; + const int n_frames = params && params->codes && gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV + ? (int) (params->codes->size() / n_cb) : 1; + builder = std::make_unique(ctx, img, gen_process, all_known, n_frames); + } break; case PROJECTOR_TYPE_DRAMA3_GEN: { const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index 73e7e1b27..8a5757c1e 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -107,6 +107,8 @@ public: virtual int32_t flush() = 0; // continuing a sequence (inp->n_past > 0) with the model's own text/audio switching virtual bool supports_continue() const { return false; } + // mtmd_helper_gen_audio_inp::speed and loudness other than the defaults + virtual bool supports_delivery() const { return false; } virtual bool next_is_text() const { return false; } std::vector logit_bias; // see mtmd_helper_gen_audio_get_logit_bias() @@ -2418,6 +2420,535 @@ private: bool wav_header_sent = false; }; +// ZONOS3: one backbone row per frame of 10 codebooks, codebook j lags j frames +class zonos3_gen_audio_pipeline : public mtmd_gen_audio_pipeline { +public: + using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; + + bool supports_delivery() const override { return true; } + + void reset() override { + seq_id = 0; + pos = 0; + segments.clear(); + seg_idx = 0; + seg_pos = 0; + rows.clear(); + step = 0; + eos_row = -1; + countdown = 0; + done = false; + n_decoded = 0; + n_codec_samples = 0; + flushed = false; + gen_state.reset(); + pcm_sent = 0; + wav_header_sent = false; + audio_pcm.clear(); + h_state_buf.clear(); + out_buf.clear(); + } + + int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { + reset(); + seq_id = inp->seq_id; + out_type = inp->out_type; + stream = inp->stream; + + // settings of the reference, only the temperature can be changed + const mtmd_gen_inp def = mtmd_gen_inp_default(mctx); + temp = inp->temp >= 0.0f ? inp->temp : def.temp; + top_k = def.top_k; + top_p = def.top_p; + rep_pen = def.penalty_repeat; + rep_win = def.penalty_last_n; + rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed); + + if (!init_gen_state(inp->seed)) { + return 1; + } + + // prompt: [voice rows] [condition rows] BOS text EOS [silence rows] + // the speaker CFG of the reference is omitted: it adds too much code and maintenance cost + if (inp->speaker_ref) { + segment seg; + if (!encode_speaker(inp->speaker_ref, seg.embd)) { + return 1; + } + segments.push_back(std::move(seg)); + } + + // condition rows: the bucket of each value over the checkpoint edges, or unknown + auto bucket = [](const std::vector & edges, float v) { + return std::to_string(std::upper_bound(edges.begin(), edges.end(), v) - edges.begin()); + }; + const std::string loudness = inp->loudness != 0.0f ? bucket(loudness_edges, inp->loudness) : "unk"; + const std::string rate = inp->speed != 1.0f ? bucket(rate_edges, inp->speed * typical_rate) : "unk"; + segment text; + for (const std::string & name : { "<|loudness_" + loudness + "|>", std::string("<|fmax_unk|>"), "<|speaking_rate_" + rate + "|>" }) { + const llama_token t = find_special_token(vocab, name); + if (t == LLAMA_TOKEN_NULL) { + LOG_ERR("mtmd_helper_gen_audio: missing ZONOS3 token %s in vocab\n", name.c_str()); + return 1; + } + text.tokens.push_back(t); + } + const std::string prompt(inp->prompt, inp->prompt_len); + std::vector ids(prompt.size() + 16); + const int32_t n_ids = llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(), ids.data(), (int32_t) ids.size(), true, false); + if (n_ids <= 2) { + LOG_ERR("mtmd_helper_gen_audio: empty prompt or tokenization failed\n"); + return 1; + } + text.tokens.insert(text.tokens.end(), ids.begin(), ids.begin() + n_ids); + segments.push_back(std::move(text)); + + // codec silence, delayed + segment silence; + for (int k = 0; k < n_prefill; k++) { + std::vector row(n_cb, audio_pad); + for (int cb = 0; cb <= k && cb < n_cb; cb++) { + row[cb] = silence_frame[cb]; + } + const float * embd = embed_row(row); + if (!embd) { + return 1; + } + silence.embd.insert(silence.embd.end(), embd, embd + n_embd); + rows.insert(rows.end(), row.begin(), row.end()); + } + segments.push_back(std::move(silence)); + + size_t n_prompt = 0; + for (const auto & seg : segments) { + n_prompt += seg.n_rows(n_embd); + } + if (n_prompt >= llama_n_ctx_seq(lctx)) { + LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n"); + return 1; + } + if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) { + LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n"); + return 1; + } + return 0; + } + + int32_t step_prompt(int32_t n_batch) override { + GGML_ASSERT(n_batch > 0); + if (seg_idx >= segments.size()) { + return 0; + } + const segment & seg = segments[seg_idx]; + const int n_rows = (int) seg.n_rows(n_embd); + const int n = std::min(n_batch, n_rows - seg_pos); + const bool last = seg_idx + 1 == segments.size() && seg_pos + n == n_rows; + + int ret; + if (!seg.tokens.empty()) { + llama_batch batch = llama_batch_init(n, 0, 1); + for (int i = 0; i < n; i++) { + batch.token [i] = seg.tokens[(size_t) (seg_pos + i)]; + batch.pos [i] = pos + i; + batch.n_seq_id[i] = 1; + batch.seq_id [i][0] = seq_id; + batch.logits [i] = last && i == n - 1; + } + batch.n_tokens = n; + ret = llama_decode(lctx, batch); + llama_batch_free(batch); + } else { + decode_embd_batch batch(seg.embd.data() + (size_t) seg_pos * n_embd, n, 1, n_embd); + batch.set_position_normal(pos, seq_id); + batch.logits[n - 1] = last; + ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n)); + } + if (ret != 0) { + LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); + return -1; + } + pos += n; + seg_pos += n; + if (seg_pos == n_rows) { + seg_idx++; + seg_pos = 0; + } + + int32_t n_left = 0; + for (size_t i = seg_idx; i < segments.size(); i++) { + n_left += (int32_t) segments[i].n_rows(n_embd) - (i == seg_idx ? seg_pos : 0); + } + if (n_left == 0) { + const float * h = llama_get_embeddings_ith(lctx, -1); + h_state_buf.assign(h, h + n_embd); + } + return n_left; + } + + int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { + GGML_UNUSED(sampled); + GGML_UNUSED(h_state_in); + *h_state_out = nullptr; + *out_stop = false; + if (done) { + *out_stop = true; + return 0; + } + + mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; + inp.embd = h_state_buf.data(); + mtmd_gen_out out{}; + if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || out.n_feats != (size_t) n_cb * n_vocab) { + LOG_ERR("mtmd_helper_gen_audio: codebook heads failed\n"); + return 1; + } + + std::vector frame(n_cb); + for (int cb = 0; cb < n_cb; cb++) { + frame[cb] = sample(out.feats + (size_t) cb * n_vocab, cb); + } + // codebooks whose delayed stream has not started yet are pad + const int r = n_prefill + step; + for (int cb = r + 1; cb < n_cb; cb++) { + frame[cb] = audio_pad; + } + rows.insert(rows.end(), frame.begin(), frame.end()); + step++; + + // after EOA, n_cb + 1 more rows flush the delay + int max_eoa = -1; + for (int cb = 0; cb < n_cb; cb++) { + if (frame[cb] == audio_eoa) { + max_eoa = cb; + } + } + if (eos_row < 0 && max_eoa >= 0) { + eos_row = std::max(0, r - max_eoa); + countdown = n_cb + 1; + } + if ((countdown > 0 && --countdown == 0) || step >= max_frames || (uint32_t) pos >= llama_n_ctx_seq(lctx)) { + done = true; + *out_stop = true; + return 0; + } + + const float * next = embed_row(frame); + if (!next) { + return 1; + } + decode_embd_batch batch(next, 1, 1, n_embd); + batch.set_position_normal(pos, seq_id); + batch.logits[0] = 1; + if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) { + LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); + return 1; + } + pos++; + const float * h = llama_get_embeddings_ith(lctx, -1); + h_state_buf.assign(h, h + n_embd); + *h_state_out = h_state_buf.data(); + + if (stream) { + return decode_stream() ? 0 : 1; + } + return 0; + } + + int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { + *out_sample_rate = info.sample_rate; + + if (!stream) { + if (flush() != 0) { + return 1; + } + if (out_n_samples) { + *out_n_samples = (int64_t) audio_pcm.size(); + } + if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { + *out_data = (const char *) audio_pcm.data(); + *out_data_len = audio_pcm.size() * sizeof(float); + return 0; + } + out_buf.clear(); + if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { + LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); + return 1; + } + *out_data = out_buf.data(); + *out_data_len = out_buf.size(); + return 0; + } + + // streaming: only the audio since the previous call + const size_t n_new = audio_pcm.size() - pcm_sent; + if (out_n_samples) { + *out_n_samples = (int64_t) n_new; + } + if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { + *out_data = (const char *) (audio_pcm.data() + pcm_sent); + *out_data_len = n_new * sizeof(float); + pcm_sent = audio_pcm.size(); + return 0; + } + out_buf.clear(); + if (!wav_header_sent) { + write_wav16_header(out_buf, UINT32_MAX, info.sample_rate); + wav_header_sent = true; + } + append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); + pcm_sent = audio_pcm.size(); + *out_data = out_buf.data(); + *out_data_len = out_buf.size(); + return 0; + } + + int32_t flush() override { + return decode_wav(true) ? 0 : 1; + } + +private: + static constexpr int n_cb = 10; + static constexpr int n_vocab = 4098; // codes, EOA, pad + static constexpr int audio_eoa = 4096; + static constexpr int audio_pad = 4097; + + // condition_edges of params.json; speed 1 is this many UTF-8 bytes per second, the middle of the rate buckets + const std::vector loudness_edges = { -29.778f, -26.947f, -25.047f, -23.3f, -21.743f, -20.386f, -19.186f, -17.851f, -15.967f }; + const std::vector rate_edges = { + 6.546f, 8.182f, 9.273f, 10.31f, 11.054f, 11.667f, 12.193f, 12.656f, 13.083f, 13.476f, 13.846f, 14.196f, 14.53f, 14.887f, 15.193f, + 15.517f, 15.865f, 16.208f, 16.579f, 16.972f, 17.411f, 17.898f, 18.434f, 19.069f, 19.818f, 20.816f, 22.286f, 24.722f, 29.25f, + }; + static constexpr float typical_rate = 15.0f; + + // digital silence of the ZAC codec (registries/silence_frame_*.json) + static constexpr int32_t silence_frame[n_cb] = { 1918, 746, 3544, 234, 1267, 2914, 3944, 2849, 326, 1337 }; + static constexpr int n_prefill = 3; + + // -3 dB of headroom + static constexpr int samples_per_frame = 2940; + static constexpr float out_gain = 0.70794578f; + + static constexpr int rep_codebooks = 6; + static constexpr int max_frames = 2000; + static constexpr size_t stream_frames = 8; + + struct segment { + std::vector tokens; // either tokens or embd rows + std::vector embd; + size_t n_rows(int n) const { return tokens.empty() ? embd.size() / (size_t) n : tokens.size(); } + }; + + const float * embed_row(const std::vector & codes) { + mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; + inp.codes = const_cast(codes.data()); + inp.n_codes = codes.size(); + mtmd_gen_out out{}; + if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { + LOG_ERR("mtmd_helper_gen_audio: codebook embedding failed\n"); + return nullptr; + } + return out.embd; + } + + // same as the reference sampler (servers/ggml/src/zonos3-sampler.cpp) + int32_t sample(const float * in, int cb) { + std::vector l(in, in + n_vocab); + l[audio_pad] = -INFINITY; + + if (rep_pen != 1.0f && cb < rep_codebooks) { + const size_t n_rows = rows.size() / n_cb - n_prefill; + const size_t n_win = std::min(n_rows, (size_t) std::max(rep_win, 0)); + std::vector seen(audio_eoa, false); + for (size_t f = n_prefill + n_rows - n_win; f < n_prefill + n_rows; f++) { + const int32_t c = rows[f * n_cb + (size_t) cb]; + if (c >= 0 && c < audio_eoa && !seen[(size_t) c]) { + seen[(size_t) c] = true; + l[c] = l[c] > 0.0f ? l[c] / rep_pen : l[c] * rep_pen; + } + } + } + + if (temp <= 0.0f) { + return (int32_t) (std::max_element(l.begin(), l.end()) - l.begin()); + } + for (auto & v : l) { + v /= temp; + } + + float kth = -INFINITY; + if (top_k > 0 && top_k < n_vocab) { + std::vector tmp(l); + std::nth_element(tmp.begin(), tmp.begin() + (top_k - 1), tmp.end(), std::greater()); + kth = tmp[(size_t) top_k - 1]; + } + std::vector idx; + for (int32_t i = 0; i < n_vocab; i++) { + if (l[i] >= kth && l[i] > -INFINITY) { + idx.push_back(i); + } + } + float lmax = -INFINITY; + for (int32_t i : idx) { + lmax = std::max(lmax, l[i]); + } + std::vector p(idx.size()); + double sum = 0.0; + for (size_t j = 0; j < idx.size(); j++) { + p[j] = std::exp((double) (l[idx[j]] - lmax)); + sum += p[j]; + } + std::vector ord(idx.size()); + for (size_t j = 0; j < ord.size(); j++) { + ord[j] = j; + } + std::sort(ord.begin(), ord.end(), [&](size_t a, size_t b) { return p[a] > p[b]; }); + if (top_p > 0.0f && top_p < 1.0f) { + double before = 0.0; + for (size_t j : ord) { + const double pj = p[j]; + if (before > top_p * sum) { + p[j] = 0.0; + } + before += pj; + } + } + double kept = 0.0; + for (double v : p) { + kept += v; + } + double u = std::uniform_real_distribution(0.0, kept)(rng); + for (size_t j = 0; j < p.size(); j++) { + u -= p[j]; + if (u <= 0.0 && p[j] > 0.0) { + return idx[j]; + } + } + return idx[ord[0]]; + } + + // frames with all codebooks known + size_t n_ready() const { + const size_t n_rows = rows.size() / n_cb; + size_t n = n_rows >= (size_t) n_cb ? n_rows - (n_cb - 1) : 0; + if (eos_row >= 0) { + n = std::min(n, (size_t) eos_row); + } + return n; + } + + // streaming, as the reference does for the first audio: the silence frames are decoded with silence codes only, + // then the first generated frame from its first code alone; later chunks of complete frames grow 2, 4, 8 + bool decode_stream() { + if (n_decoded == 0) { + const size_t n_rows = rows.size() / n_cb; + if (n_rows <= (size_t) n_prefill || rows[(size_t) n_prefill * n_cb] >= audio_eoa) { + return true; + } + std::vector codes; + for (int k = 0; k < n_prefill; k++) { + codes.insert(codes.end(), silence_frame, silence_frame + n_cb); + } + for (int cb = 0; cb < n_cb; cb++) { + const size_t r = (size_t) n_prefill + cb; + const int32_t c = r < n_rows ? rows[r * n_cb + cb] : -1; + codes.push_back(c >= audio_eoa ? -1 : c); + } + if (!run_codec(codes.data(), codes.size())) { + return false; + } + n_decoded = n_prefill + 1; + return true; + } + const size_t n_chunk = std::min(stream_frames, n_decoded - n_prefill + 1); + if (n_ready() >= n_decoded + n_chunk) { + return decode_wav(false); + } + return true; + } + + // the silence frames are decoded too, as left context + bool decode_wav(bool final) { + const size_t n_frames = n_ready(); + std::vector codes; + while (n_decoded < n_frames) { + const size_t n = std::min(n_frames - n_decoded, (size_t) 64); + codes.resize(n * n_cb); + for (size_t f = 0; f < n; f++) { + for (int cb = 0; cb < n_cb; cb++) { + // EOA and pad have no codec row + const int32_t c = rows[(n_decoded + f + cb) * n_cb + cb]; + codes[f * n_cb + cb] = std::clamp(c, 0, audio_eoa - 1); + } + } + if (!run_codec(codes.data(), codes.size())) { + return false; + } + n_decoded += n; + } + if (final && done && n_decoded > 0 && !flushed) { + flushed = true; + return run_codec(nullptr, 0); + } + return true; + } + + bool run_codec(const int32_t * codes, size_t n_codes) { + mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; + inp.codes = const_cast(codes); + inp.n_codes = n_codes; + mtmd_gen_out out{}; + if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { + LOG_ERR("mtmd_helper_gen_audio: codec decode failed\n"); + return false; + } + // drop the audio of the silence frames, then the output gain and soft limiter of the reference + const float knee = std::pow(10.0f, -1.0f / 20.0f); + for (size_t i = 0; i < out.n_samples; i++, n_codec_samples++) { + if (n_codec_samples < (size_t) n_prefill * samples_per_frame) { + continue; + } + const float x = out.audio[i] * out_gain; + const float a = std::fabs(x); + audio_pcm.push_back(a <= knee ? x : std::copysign(knee + (1.0f - knee) * std::tanh((a - knee) / (1.0f - knee)), x)); + } + return true; + } + + std::vector segments; + size_t seg_idx = 0; + int seg_pos = 0; + + std::vector rows; // delayed rows: silence rows, then the generated ones + int step = 0; + int eos_row = -1; + int countdown = 0; + bool done = false; + size_t n_decoded = 0; + size_t n_codec_samples = 0; + bool flushed = false; + + float temp = 0.75f; + int32_t top_k = 80; + float top_p = 0.69f; + float rep_pen = 2.4f; + int32_t rep_win = 96; + std::mt19937 rng; + + llama_seq_id seq_id = 0; + llama_pos pos = 0; + mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; + bool stream = false; + + std::vector h_state_buf; + std::vector audio_pcm; + std::vector out_buf; + size_t pcm_sent = 0; + bool wav_header_sent = false; +}; + static std::unique_ptr make_pipeline(llama_context * lctx, mtmd_context * mctx) { switch (mtmd_gen_audio_get_info(mctx).type) { case MTMD_GEN_AUDIO_TYPE_QWEN3TTS: @@ -2432,6 +2963,8 @@ static std::unique_ptr make_pipeline(llama_context * lc return std::unique_ptr(new chatterbox_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_DRAMA3: return std::unique_ptr(new drama3_gen_audio_pipeline(lctx, mctx)); + case MTMD_GEN_AUDIO_TYPE_ZONOS3: + return std::unique_ptr(new zonos3_gen_audio_pipeline(lctx, mctx)); default: return nullptr; } @@ -2463,6 +2996,8 @@ struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) { inp.top_p = 1.0f; inp.temp = -1.0f; // model default inp.seed = UINT32_MAX; // random + inp.speed = 1.0f; + inp.loudness = 0.0f; inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; return inp; } @@ -2476,6 +3011,14 @@ int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_ LOG_ERR("mtmd_helper_gen_audio: the pipeline cannot continue a sequence\n"); return 1; } + if (!std::isfinite(inp->speed) || inp->speed <= 0.0f || !std::isfinite(inp->loudness) || inp->loudness > 0.0f) { + LOG_ERR("mtmd_helper_gen_audio: speed must be > 0 and loudness < 0 LUFS (0 = model default)\n"); + return 2; + } + if ((inp->speed != 1.0f || inp->loudness != 0.0f) && !ctx->pipeline->supports_delivery()) { + LOG_ERR("mtmd_helper_gen_audio: the model does not support speed or loudness\n"); + return 2; + } ctx->pipeline->logit_bias.clear(); return ctx->pipeline->set_input(inp); } diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index 1bf3ee2ba..5ca414906 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -291,6 +291,10 @@ struct mtmd_helper_gen_audio_inp { float temp; // audio frame temperature if the model supports it, < 0 = model default, 0 = greedy uint32_t seed; // UINT32_MAX for random (default: random) + // delivery (only for models that support it); other models reject non-default values + float speed; // speaking speed relative to a typical pace, > 0 (default: 1.0 = the model's own pace) + float loudness; // target loudness in LUFS, < 0 (default: 0.0 = the model's own loudness) + enum mtmd_helper_gen_audio_outtype out_type; }; @@ -304,6 +308,7 @@ MTMD_API void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx); MTMD_API void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx); +// returns 0 on success, 2 if the input is invalid for the model (e.g. unsupported speed or loudness), 1 on other errors MTMD_API int32_t mtmd_helper_gen_audio_set_input( mtmd_helper_gen_audio * ctx, const struct mtmd_helper_gen_audio_inp * inp); diff --git a/tools/mtmd/mtmd-impl.h b/tools/mtmd/mtmd-impl.h index e2157f767..0d287dcdd 100644 --- a/tools/mtmd/mtmd-impl.h +++ b/tools/mtmd/mtmd-impl.h @@ -106,6 +106,7 @@ #define KEY_GEN_AUDIO_PROJ_TYPE "clip.gen.audio.projector_type" // for models with mixed modalities // name of the weight variant, for settings that are not in the checkpoint #define KEY_GEN_AUDIO_VARIANT "clip.gen.audio.model_variant" +#define KEY_GEN_AUDIO_DEFAULT_VOICE "clip.gen.audio.default_voice" // sampling defaults of the generated audio, like general.sampling.* for text #define KEY_GEN_AUDIO_TEMP "clip.gen.audio.sampling.temp" #define KEY_GEN_AUDIO_TOP_K "clip.gen.audio.sampling.top_k" @@ -572,6 +573,8 @@ enum projector_type { PROJECTOR_TYPE_SOPRANO, PROJECTOR_TYPE_LFM3A_GEN, PROJECTOR_TYPE_DRAMA3_GEN, + PROJECTOR_TYPE_ZONOS3_GEN, + PROJECTOR_TYPE_ZONOS3_SPKENC, PROJECTOR_TYPE_MUSE_GLIMMER, PROJECTOR_TYPE_SENSENOVA_U1, PROJECTOR_TYPE_COHERE2V, @@ -660,6 +663,8 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_SOPRANO, "soprano"}, { PROJECTOR_TYPE_LFM3A_GEN, "lfm3a_gen"}, { PROJECTOR_TYPE_DRAMA3_GEN, "drama3_gen"}, + { PROJECTOR_TYPE_ZONOS3_GEN, "zonos3_gen"}, + { PROJECTOR_TYPE_ZONOS3_SPKENC, "zonos3_spkenc"}, { PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"}, { PROJECTOR_TYPE_SENSENOVA_U1, "sensenova_u1"}, { PROJECTOR_TYPE_COHERE2V, "cohere2v"}, diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp index c4e5bbd54..c4c52a4d5 100644 --- a/tools/mtmd/mtmd-model.cpp +++ b/tools/mtmd/mtmd-model.cpp @@ -214,6 +214,7 @@ struct mtmd_model_loader { hparams.image_size = 0; hparams.patch_size = 1; get_string(KEY_GEN_AUDIO_VARIANT, hparams.gen_model_variant, false); + get_string(KEY_GEN_AUDIO_DEFAULT_VOICE, hparams.gen_default_voice, false); } else { GGML_ASSERT(false && "unknown modality"); @@ -967,6 +968,15 @@ struct mtmd_model_loader { "%s: mimo_audio: %s must be > 0\n", __func__, KEY_A_LOCAL_GROUP_SIZE)); } } break; + case PROJECTOR_TYPE_ZONOS3_SPKENC: + { + // kaldi fbank front-end + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 512; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + hparams.warmup_audio_size = 1000; + } break; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: { // ECAPA-TDNN speaker encoder, mel front-end uses the Slaney default (fmin=0, fmax=sr/2) @@ -1032,6 +1042,10 @@ struct mtmd_model_loader { // matches the reference decoder's sliding_window (speech_tokenizer/config.json) hparams.wav_tfm_swa = 72; } break; + case PROJECTOR_TYPE_ZONOS3_GEN: + { + hparams.audio_sample_rate = 44100; + } break; case PROJECTOR_TYPE_DRAMA3_GEN: { // Fish DAC decoder, codec.pth has no config @@ -1479,6 +1493,7 @@ struct mtmd_model_loader { const bool has_standard_layers = ( model.proj_type != PROJECTOR_TYPE_GEMMA3NV && model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC && + model.proj_type != PROJECTOR_TYPE_ZONOS3_SPKENC && model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN && model.proj_type != PROJECTOR_TYPE_SOPRANO && model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO && @@ -2574,6 +2589,118 @@ struct mtmd_model_loader { // code2wav: RVQ codes -> raw PCM, lives in the same ctx as code_predictor load_code2wav(true); } break; + case PROJECTOR_TYPE_ZONOS3_GEN: + { + model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight")); + model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); + if (model.gen_code_out_embd_w->ne[1] % mtmd_graph_zonos3_gen::n_codebooks != 0 || + model.gen_code_head_w->ne[1] != model.gen_code_out_embd_w->ne[1]) { + throw std::runtime_error(mtmd_format("%s: zonos3: unexpected codebook table shape\n", __func__)); + } + + // voice.: the voice rows of the prompt + for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { + const char * name = gguf_get_tensor_name(ctx_gguf.get(), i); + if (strncmp(name, "voice.", 6) != 0) { + continue; + } + const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name); + if (t->type != GGML_TYPE_F32 || ggml_n_dims(t) > 2 || t->ne[0] != model.gen_code_out_embd_w->ne[0]) { + throw std::runtime_error(mtmd_format("%s: invalid voice tensor %s\n", __func__, name)); + } + auto & embd = model.gen_voices[name + 6].rows; + embd.resize(ggml_nelements(t)); + fin.seekg(tensor_offset.at(name), std::ios::beg); + if (!fin.read(reinterpret_cast(embd.data()), ggml_nbytes(t))) { + throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name)); + } + } + LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size()); + + // ZAC codec decoder + auto & z = model.zonos3; + const std::string p = "a.gen.wav"; + auto has = [&](const std::string & name) { + return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0; + }; + auto load_convnext = [&](const std::string & q) { + mtmd_zonos3_wav::convnext b; + b.dw_w = get_tensor(q + ".dw.weight"); + b.dw_b = get_tensor(q + ".dw.bias"); + b.norm_w = get_tensor(q + ".norm.weight"); + b.norm_b = get_tensor(q + ".norm.bias"); + b.pw1_w = get_tensor(q + ".pw1.weight"); + b.pw1_b = get_tensor(q + ".pw1.bias"); + b.pw2_w = get_tensor(q + ".pw2.weight"); + b.pw2_b = get_tensor(q + ".pw2.bias"); + return b; + }; + z.sem_table = get_tensor(p + ".sem.table"); + z.sem_bias = get_tensor(p + ".sem.bias"); + for (int i = 0; has(mtmd_format("%s.sem.%d.dw.weight", p.c_str(), i)); i++) { + z.sem.push_back(load_convnext(p + ".sem." + std::to_string(i))); + } + z.sem_out_w = get_tensor(p + ".sem.out.weight"); + z.sem_out_b = get_tensor(p + ".sem.out.bias"); + z.res_table = get_tensor(p + ".res.table"); + z.res_bias = get_tensor(p + ".res.bias"); + for (int s = 0; has(mtmd_format("%s.stage.%d.in.weight", p.c_str(), s)); s++) { + const std::string q = p + ".stage." + std::to_string(s); + mtmd_zonos3_wav::stage st; + st.in_w = get_tensor(q + ".in.weight"); + st.out_w = get_tensor(q + ".out.weight", false); + for (int il = 0; has(mtmd_format("%s.blk.%d.qkv.weight", q.c_str(), il)); il++) { + const std::string b = q + ".blk." + std::to_string(il); + mtmd_zonos3_wav::layer l; + l.norm1_w = get_tensor(b + ".norm1.weight"); + l.norm1_b = get_tensor(b + ".norm1.bias"); + l.qkv_w = get_tensor(b + ".qkv.weight"); + l.wo_w = get_tensor(b + ".wo.weight"); + l.attn_scale = get_tensor(b + ".attn_scale"); + l.norm2_w = get_tensor(b + ".norm2.weight"); + l.norm2_b = get_tensor(b + ".norm2.bias"); + l.ffn1_w = get_tensor(b + ".ffn1.weight"); + l.ffn2_w = get_tensor(b + ".ffn2.weight"); + l.ffn_scale = get_tensor(b + ".ffn_scale"); + st.layers.push_back(l); + } + z.stages.push_back(std::move(st)); + } + for (int i = 0; has(mtmd_format("%s.head.%d.dw.weight", p.c_str(), i)); i++) { + z.head.push_back(load_convnext(p + ".head." + std::to_string(i))); + } + z.head_norm_w = get_tensor(p + ".head.norm.weight"); + z.head_norm_b = get_tensor(p + ".head.norm.bias"); + z.head_out_w = get_tensor(p + ".head.out.weight"); + z.head_out_b = get_tensor(p + ".head.out.bias"); + if (z.stages.size() != mtmd_graph_zonos3_gen::n_stages || z.sem.empty() || z.head.empty() || + z.head_out_w->ne[1] != mtmd_graph_zonos3_gen::n_fft[0] + mtmd_graph_zonos3_gen::n_fft[1] + 4) { + throw std::runtime_error(mtmd_format("%s: zonos3: unexpected codec decoder layout\n", __func__)); + } + + // read on the host + auto read_host = [&](const std::string & name, std::vector & dst) -> const ggml_tensor * { + const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name.c_str()); + if (!t || t->type != GGML_TYPE_F32 || ggml_n_dims(t) != 2) { + throw std::runtime_error(mtmd_format("%s: invalid or missing tensor %s\n", __func__, name.c_str())); + } + dst.resize(ggml_nelements(t)); + fin.seekg(tensor_offset.at(name), std::ios::beg); + if (!fin.read(reinterpret_cast(dst.data()), ggml_nbytes(t))) { + throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name.c_str())); + } + return t; + }; + const ggml_tensor * mixer = read_host(p + ".mixer.weight", z.mixer); + const ggml_tensor * rs = read_host(p + ".resample.weight", z.resample); + if (mixer->ne[1] != 2 || rs->ne[1] != mtmd_graph_zonos3_gen::rs_new || + rs->ne[0] < mtmd_graph_zonos3_gen::rs_orig || (rs->ne[0] - mtmd_graph_zonos3_gen::rs_orig) % 2 != 0) { + throw std::runtime_error(mtmd_format("%s: zonos3: unexpected mixer or resampler shape\n", __func__)); + } + z.fir_kernel = mixer->ne[0]; + z.rs_new = rs->ne[1]; + z.rs_kernel_len = rs->ne[0]; + } break; case PROJECTOR_TYPE_DRAMA3_GEN: { // generic a.gen.code.blk.* tensors are loaded by the loop above @@ -2847,6 +2974,19 @@ struct mtmd_model_loader { } } } break; + case PROJECTOR_TYPE_ZONOS3_SPKENC: + { + // the graph reads the tensors by name + for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { + const char * name = gguf_get_tensor_name(ctx_gguf.get(), i); + if (strncmp(name, "a.spk.", 6) == 0) { + model.zonos3_spk[name] = get_tensor(name); + } + } + if (!model.zonos3_spk.count("a.spk.proj.weight") || !model.zonos3_spk.count("a.spk.linear.weight")) { + throw std::runtime_error(mtmd_format("%s: zonos3: missing speaker encoder tensors\n", __func__)); + } + } break; case PROJECTOR_TYPE_CHATTERBOX_SPKENC: { auto & c = model.cbx; @@ -3756,6 +3896,10 @@ mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params model->hparams.custom_image_max_tokens = params.image_max_tokens; loader.load_hparams(*model, modality); loader.load_tensors(*model, buft, params.no_alloc); + const std::string & def = model->hparams.gen_default_voice; + if (!def.empty() && model->gen_voices.find(def) == model->gen_voices.end()) { + throw std::runtime_error(mtmd_format("%s: default voice \"%s\" is not a predefined voice\n", __func__, def.c_str())); + } return model; }; @@ -4163,6 +4307,7 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const { n_patches = ((img->nx() + ws - 1) / ws) * (ws / ds); } break; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + case PROJECTOR_TYPE_ZONOS3_SPKENC: { // pooling gives one speaker embedding, whatever the clip length is n_patches = 1; @@ -4175,6 +4320,7 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const { case PROJECTOR_TYPE_QWEN3TTS_GEN: case PROJECTOR_TYPE_LFM3A_GEN: case PROJECTOR_TYPE_DRAMA3_GEN: + case PROJECTOR_TYPE_ZONOS3_GEN: { // one hidden-state vector fed back to the talker per call n_patches = 1; @@ -4356,9 +4502,12 @@ int mtmd_model::n_mmproj_embd() const { return mm_2_w->ne[1]; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: return mm_fc_w->ne[2]; + case PROJECTOR_TYPE_ZONOS3_SPKENC: + return zonos3_spk.at("a.spk.proj.weight")->ne[1]; case PROJECTOR_TYPE_QWEN3TTS_GEN: case PROJECTOR_TYPE_LFM3A_GEN: case PROJECTOR_TYPE_DRAMA3_GEN: + case PROJECTOR_TYPE_ZONOS3_GEN: return gen_code_out_embd_w->ne[0]; case PROJECTOR_TYPE_POCKETTTS_SPKENC: return spk_proj_w->ne[1]; diff --git a/tools/mtmd/mtmd-model.h b/tools/mtmd/mtmd-model.h index b27b1c0d8..d04314d93 100644 --- a/tools/mtmd/mtmd-model.h +++ b/tools/mtmd/mtmd-model.h @@ -200,6 +200,7 @@ struct mtmd_hparams { // name of the weight variant, some pipelines tune themselves on it std::string gen_model_variant; + std::string gen_default_voice; // sampling defaults of the generated audio: greedy, no top-k, no repetition penalty float gen_temp = 0.0f; @@ -869,6 +870,60 @@ struct mtmd_drama3_layer { ggml_tensor * out_norm_w = nullptr; }; +// zonos3 ZAC codec decoder (a.gen.wav.*) +struct mtmd_zonos3_wav { + struct convnext { + ggml_tensor * dw_w = nullptr; // [K, C] + ggml_tensor * dw_b = nullptr; + ggml_tensor * norm_w = nullptr; + ggml_tensor * norm_b = nullptr; + ggml_tensor * pw1_w = nullptr; + ggml_tensor * pw1_b = nullptr; + ggml_tensor * pw2_w = nullptr; + ggml_tensor * pw2_b = nullptr; + }; + struct layer { + ggml_tensor * norm1_w = nullptr; + ggml_tensor * norm1_b = nullptr; + ggml_tensor * qkv_w = nullptr; + ggml_tensor * wo_w = nullptr; + ggml_tensor * attn_scale = nullptr; + ggml_tensor * norm2_w = nullptr; + ggml_tensor * norm2_b = nullptr; + ggml_tensor * ffn1_w = nullptr; + ggml_tensor * ffn2_w = nullptr; + ggml_tensor * ffn_scale = nullptr; + }; + struct stage { + ggml_tensor * in_w = nullptr; + ggml_tensor * out_w = nullptr; // nullptr in the last stage + std::vector layers; + }; + + ggml_tensor * sem_table = nullptr; // folded through its out_proj + ggml_tensor * sem_bias = nullptr; + std::vector sem; + ggml_tensor * sem_out_w = nullptr; + ggml_tensor * sem_out_b = nullptr; + ggml_tensor * res_table = nullptr; // merged + ggml_tensor * res_bias = nullptr; // [latent, n_codebooks - 1] + + std::vector stages; + + std::vector head; + ggml_tensor * head_norm_w = nullptr; + ggml_tensor * head_norm_b = nullptr; + ggml_tensor * head_out_w = nullptr; // both iSTFT branches + ggml_tensor * head_out_b = nullptr; + + // read on the host + std::vector mixer; // [2, fir_kernel] + std::vector resample; // [rs_new, rs_kernel_len] + int64_t fir_kernel = 0; + int64_t rs_new = 0; + int64_t rs_kernel_len = 0; +}; + struct mtmd_vocos { ggml_tensor * input_w = nullptr; ggml_tensor * input_b = nullptr; @@ -1148,6 +1203,7 @@ struct mtmd_model { // qwen3tts code2wav: RVQ codes -> raw PCM mtmd_code2wav c2w; std::vector drama3_layers; + mtmd_zonos3_wav zonos3; // pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path) mtmd_seanet seanet; @@ -1172,6 +1228,7 @@ struct mtmd_model { // chatterbox tensors read on the host, see cbx_read_tensor() std::map cbx_tensors; + std::map zonos3_spk; // a.spk.* // predefined voices, no rows if the voice is named in the prompt (lfm3a) std::map gen_voices; diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp index 798d875bb..ed8800fce 100644 --- a/tools/mtmd/mtmd-subcontext.cpp +++ b/tools/mtmd/mtmd-subcontext.cpp @@ -392,6 +392,7 @@ static std::vector list_gen_state_slots(const mtmd_hparams & hpa case PROJECTOR_TYPE_DRAMA3_GEN: return list_drama3_state_slots(hparams, model); case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model); case PROJECTOR_TYPE_LFM3A_GEN: return list_lfm3a_gen_state_slots(hparams, model); + case PROJECTOR_TYPE_ZONOS3_GEN: return list_zonos3_gen_state_slots(hparams, model); default: return {}; } } @@ -1676,6 +1677,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { case PROJECTOR_TYPE_GEMMA4UA: case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + case PROJECTOR_TYPE_ZONOS3_SPKENC: { // do nothing } break; @@ -1689,6 +1691,54 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { segfix[(size_t) S - 1] = 100.0f / (float) last; set_input_f32("inp_segfix", segfix); } break; + case PROJECTOR_TYPE_ZONOS3_GEN: + if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { + // frame-major codes -> codebook-major rows; a residual code of -1 is missing and left out + const int64_t n_cb = mtmd_graph_zonos3_gen::n_codebooks; + const int64_t n_codes = mtmd_graph_zonos3_gen::n_cb_vocab; + GGML_ASSERT(params->codes != nullptr && params->codes->size() % n_cb == 0); + const int64_t n_frames = (int64_t) params->codes->size() / n_cb; + std::vector rows(params->codes->size()); + std::vector keep((size_t) n_frames * (n_cb - 1), 1.0f); + for (int64_t t = 0; t < n_frames; t++) { + for (int64_t c = 0; c < n_cb; c++) { + int32_t code = (*params->codes)[t * n_cb + c]; + if (code == -1 && c > 0) { + keep[(c - 1) * n_frames + t] = 0.0f; + code = 0; + } + if (code < 0 || code >= n_codes) { + LOG_ERR("%s: code out of range (%d, codebook size %lld)\n", __func__, code, (long long) n_codes); + return false; + } + rows[c * n_frames + t] = code + (int32_t) (c == 0 ? 0 : (c - 1) * n_codes); + } + } + set_input_i32("inp_codes", rows); + set_input_f32("inp_keep", keep); + set_gen_state_in(); + } else { + const int64_t n_cb = mtmd_graph_zonos3_gen::n_codebooks; + const int64_t n_vocab = model.gen_code_out_embd_w->ne[1] / n_cb; + // no codes: logits only, the row of codes 0 is ignored + std::vector codes(n_cb, 0); + if (params->codes) { + if (params->codes->size() != (size_t) n_cb) { + LOG_ERR("%s: expected %lld codes, got %zu\n", __func__, (long long) n_cb, params->codes->size()); + return false; + } + codes = *params->codes; + } + for (int64_t i = 0; i < n_cb; i++) { + if (codes[i] < 0 || codes[i] >= n_vocab) { + LOG_ERR("%s: code out of range (%d, vocab %lld)\n", __func__, codes[i], (long long) n_vocab); + return false; + } + codes[i] += (int32_t) (i * n_vocab); + } + set_input_i32("inp_codes", codes); + } + break; case PROJECTOR_TYPE_LFM3A_GEN: { const int64_t n_cb = model.gen_code_embd_w->ne[2]; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 63a8e3954..0042aeb42 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1286,6 +1286,10 @@ struct mtmd_context { { audio_preproc = std::make_unique(models.a.get()); } break; + case PROJECTOR_TYPE_ZONOS3_SPKENC: + { + audio_preproc = std::make_unique(models.a.get()); + } break; case PROJECTOR_TYPE_POCKETTTS_SPKENC: { audio_preproc = std::make_unique(models.a.get()); @@ -2462,6 +2466,10 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { info.type = MTMD_GEN_AUDIO_TYPE_DRAMA3; info.sample_rate = 44100; break; + case PROJECTOR_TYPE_ZONOS3_GEN: + info.type = MTMD_GEN_AUDIO_TYPE_ZONOS3; + info.sample_rate = 44100; + break; default: info.type = MTMD_GEN_AUDIO_TYPE_NONE; break; @@ -2822,8 +2830,23 @@ const char * mtmd_gen_audio_voice_name(const mtmd_context * ctx, int32_t idx) { if (!ctx->ctx_gen_a || idx < 0) { return nullptr; } - const auto & voices = ctx->ctx_gen_a->model.gen_voices; - return idx < (int32_t) voices.size() ? std::next(voices.begin(), idx)->first.c_str() : nullptr; + const auto & model = ctx->ctx_gen_a->model; + const auto & voices = model.gen_voices; + if (idx >= (int32_t) voices.size()) { + return nullptr; + } + // the default voice goes first, the others keep their order + const auto def = voices.find(model.hparams.gen_default_voice); + if (def != voices.end() && idx == 0) { + return def->first.c_str(); + } + int32_t i = def != voices.end() ? 1 : 0; + for (auto it = voices.begin(); it != voices.end(); ++it) { + if (it != def && i++ == idx) { + return it->first.c_str(); + } + } + return nullptr; } mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * name) { @@ -2871,6 +2894,14 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { inp.top_p = 0.9f; inp.temp = 1.0f; break; + case PROJECTOR_TYPE_ZONOS3_GEN: + // sampled in the helper, defaults of registries/generation.json + inp.top_k = 80; + inp.top_p = 0.69f; + inp.temp = 0.75f; + inp.penalty_repeat = 2.4f; + inp.penalty_last_n = 96; + break; case PROJECTOR_TYPE_LFM3A_GEN: { // MaskGIT sampling runs in the helper, with the settings of the checkpoint @@ -2902,6 +2933,9 @@ struct mtmd_gen_state { // lfm3a: the codec graph yields STFT frames, the ISTFT runs on the host std::unique_ptr istft; + // zonos3: the iSTFT head runs on the host + std::unique_ptr zonos3_synth; + ~mtmd_gen_state() { delete gen_state; } @@ -2941,6 +2975,60 @@ void mtmd_gen_state_reset(mtmd_gen_state * state) { const auto * hparams = &state->sctx->model.hparams; state->istft = std::make_unique(hparams->audio_n_fft, hparams->audio_hop_len); } + state->zonos3_synth.reset(); +} + +// zonos3 GEN_WAV: n_codes == 0 flushes and resets the state +static int32_t mtmd_gen_wav_zonos3(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) { + mtmd_subcontext * sctx = ctx->ctx_gen_a.get(); + const auto & z = sctx->model.zonos3; + using g = mtmd_graph_zonos3_gen; + + if (!state->zonos3_synth) { + state->zonos3_synth = std::make_unique(z.mixer, z.resample, g::n_fft[0], g::n_fft[1], g::hop, g::rs_orig, g::rs_new); + } + auto & audio = state->out_audio; + audio.clear(); + + if (!inp->codes || inp->n_codes == 0) { + state->zonos3_synth->flush(audio); + mtmd_gen_state_reset(state); + } else { + if (inp->n_codes % g::n_codebooks != 0) { + LOG_ERR("%s: expected whole frames of %d codes\n", __func__, g::n_codebooks); + return 1; + } + std::vector in_codes(inp->codes, inp->codes + inp->n_codes); + std::vector spec; + + // no hidden-state input, the batch entry is an unused placeholder + mtmd_image_f32 dummy; + dummy.set_size({1, 1}, false, true); + dummy.cpy_buf(std::vector(1, 0.0f)); + + mtmd_image_f32_batch batch; + batch.is_audio = true; + batch.entries.push_back(std::move(dummy)); + + mtmd_subctx_encode_params params; + params.imgs = &batch; + params.n_threads = ctx->n_threads; + params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV; + params.codes = &in_codes; + params.out_feats = &spec; + params.gen_state = state->gen_state; + if (!sctx->encode(¶ms)) { + LOG_ERR("%s: encode failed (codec)\n", __func__); + return 1; + } + const int64_t n_rows = (g::n_fft[0] + 2) + (g::n_fft[1] + 2); + GGML_ASSERT(spec.size() % (size_t) n_rows == 0); + state->zonos3_synth->push(spec.data(), (int64_t) (spec.size() / n_rows), audio); + } + + out->audio = audio.data(); + out->n_samples = audio.size(); + return 0; } // lfm3a GEN_WAV: ISTFT of the codec output, n_codes == 0 flushes and resets the state @@ -3042,6 +3130,9 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_WAV && sctx->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN) { return mtmd_gen_wav_lfm3a(ctx, state, inp, out); } + if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_WAV && sctx->model.proj_type == PROJECTOR_TYPE_ZONOS3_GEN) { + return mtmd_gen_wav_zonos3(ctx, state, inp, out); + } if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE) { const size_t n_embd = (size_t) sctx->model.n_mmproj_embd(); diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index e14f345a6..b0d673dbe 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -472,6 +472,7 @@ enum mtmd_gen_audio_type { MTMD_GEN_AUDIO_TYPE_LFM3A, MTMD_GEN_AUDIO_TYPE_CHATTERBOX, MTMD_GEN_AUDIO_TYPE_DRAMA3, + MTMD_GEN_AUDIO_TYPE_ZONOS3, }; struct mtmd_gen_audio_info { @@ -485,7 +486,7 @@ MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * // read a named tensor of the gen audio model as F32, returns the element count (0 if not found), out may be null to query the size MTMD_API size_t mtmd_gen_audio_read_tensor(mtmd_context * ctx, const char * name, float * out, size_t n_max); -// name of the idx-th predefined voice, nullptr when idx is out of range +// name of the idx-th predefined voice, nullptr when idx is out of range; idx 0 is the default voice MTMD_API const char * mtmd_gen_audio_voice_name(const mtmd_context * ctx, int32_t idx); // predefined voice with this name as a voice bitmap, to pass as speaker reference @@ -516,6 +517,7 @@ struct mtmd_gen_inp { // for MTMD_GEN_PROCESS_TYPE_GEN_WAV // pass either codes (discrete) or feats (continuous), depending on the pipeline // lfm3a GEN_CODE also takes codes: the codes revealed so far, -1 for a masked codebook + // zonos3 GEN_WAV: -1 for a residual code not known yet, the frame is decoded without it int32_t * codes; size_t n_codes; const float * feats; diff --git a/tools/server/README.md b/tools/server/README.md index 753530a86..13d41d884 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -874,6 +874,10 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For the list `seed`: RNG seed for both the backbone sampler and the codec/vocoder (`-1` = random, the default). Set to a fixed value for reproducible output. +`speed`: Speaking speed relative to a typical pace, `> 0` (default `1.0`, the model's own pace). Only for models that support it, others reject a value other than `1.0` with a 400 error. + +`loudness`: Target loudness in LUFS, `< 0` (default `0.0`, the model's own loudness). Only for models that support it, others reject a value other than `0.0` with a 400 error. + `n_predict`: Max number of audio frames to generate. Defaults to `512`; generation normally stops earlier once the model emits an end-of-speech token. `response_format`: `wav` (default), `pcm` (raw `float32` little-endian mono samples, no header; the response `Content-Type` carries the sample rate, e.g. `audio/pcm;rate=24000;encoding=float;bits=32`) or `pcm16` (same, as signed 16-bit samples, e.g. `audio/pcm;rate=24000;encoding=signed-int;bits=16`). @@ -1668,7 +1672,7 @@ Generates speech from text, for models that can generate audio. The request is c `stream_format`: `audio` streams the response as it becomes available, see `stream` in `/tts` for the WAV header it writes. `sse` is rejected. The response is not streamed when omitted. -`speed`: Only `1.0` is accepted. +`speed`: Speaking speed, see `speed` in `/tts`. Only models that support it accept a value other than `1.0`. `loudness` of `/tts` can be used as well. `instructions`: Ignored. diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index 4e4622baf..af1e42dbd 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -799,10 +799,6 @@ json convert_speech_to_tts(const json & inp_body) { throw std::invalid_argument("Only 'audio' stream_format is supported for speech"); } - if (json_value(inp_body, "speed", 1.0) != 1.0) { - throw std::invalid_argument("'speed' is not supported for speech"); - } - if (!json_value(inp_body, "instructions", std::string()).empty()) { SRV_WRN("%s", "'instructions' is not supported for speech and is ignored\n"); } diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 7c2c35c35..28e01c69b 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1951,7 +1951,12 @@ private: // mtmd_helper always re-eval the whole prompt slot.prompt_clear(); task.tts_inp.data.seq_id = slot.id; - if (slot.tts.ctx.set_input(task.tts_inp.get()) != 0) { + const int32_t ret = slot.tts.ctx.set_input(task.tts_inp.get()); + if (ret == 2) { + send_error(task, "invalid TTS input: the model may not support \"speed\" or \"loudness\", see the server log", ERROR_TYPE_INVALID_REQUEST); + return false; + } + if (ret != 0) { send_error(task, "failed to process TTS prompt", ERROR_TYPE_SERVER); return false; } @@ -5650,7 +5655,7 @@ static json parse_tts_body(const server_http_req & req) { body[k] = std::stoll(body[k].get()); } } - for (const char * k : { "top_p", "temperature", "repeat_penalty", "speed" }) { + for (const char * k : { "top_p", "temperature", "repeat_penalty", "speed", "loudness" }) { if (body.contains(k) && body[k].is_string()) { body[k] = std::stof(body[k].get()); } diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp index 84e872f21..e72617d6d 100644 --- a/tools/server/server-schema.cpp +++ b/tools/server/server-schema.cpp @@ -555,6 +555,12 @@ std::vector> make_tts_schema(server_task & task) { add((new field_num("seed", task.tts_inp.data.seed)) ->set_desc("RNG seed for the backbone sampler and the codec/vocoder (-1 = random)")); + add((new field_num("speed", task.tts_inp.data.speed)) + ->set_desc("Speaking speed relative to a typical pace, for models that support it (1.0 = model default)")); + + add((new field_num("loudness", task.tts_inp.data.loudness)) + ->set_desc("Target loudness in LUFS, for models that support it (0.0 = model default)")); + add((new field_num("n_predict", task.params.n_predict)) ->set_hard_limits(-1, INT32_MAX) ->add_alias("max_tokens")