From 307e3f1551c2c3c6c0560c454ab46bb64baeb2df Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Tue, 17 Nov 2026 22:25:44 +0100 Subject: [PATCH] mtmd: add kokoro tts support (#32413) * jinja: impl scope-chain key lookup * add test for call_statement lambda * chat: add tts_dict for kokoro G2P template * more testing * value_float_t range check * no json-in-meta, add llama_model_meta_arr_str/_n * mtmd: add kokoro tts support * detailed guide about G2P and text norm port * refine G2P * clean up * migrate batch api * nits --- conversion/__init__.py | 2 + conversion/kokoro.py | 419 +++++++++++++ docs/multimodal/tts.md | 60 ++ gguf-py/gguf/constants.py | 19 + models/templates/hexgrad-Kokoro-82M-tts.jinja | 585 ++++++++++++++++++ src/llama-arch.cpp | 3 + src/llama-arch.h | 2 + src/llama-model.cpp | 4 + src/models/kokoro.cpp | 145 +++++ src/models/models.h | 16 + tests/CMakeLists.txt | 2 +- tests/test-llama-archs.cpp | 3 + tests/test-mtmd-impl.cpp | 181 ++++++ tools/mtmd/CMakeLists.txt | 1 + tools/mtmd/README-dev.md | 24 +- tools/mtmd/models/kokoro.cpp | 515 +++++++++++++++ tools/mtmd/models/models.h | 41 ++ tools/mtmd/mtmd-graph.cpp | 7 + tools/mtmd/mtmd-helper-gen.cpp | 250 +++++++- tools/mtmd/mtmd-impl.h | 2 + tools/mtmd/mtmd-model.cpp | 21 + tools/mtmd/mtmd-subcontext.cpp | 124 ++++ tools/mtmd/mtmd-subcontext.h | 23 + tools/mtmd/mtmd.cpp | 48 +- tools/mtmd/mtmd.h | 3 + tools/server/README.md | 4 +- tools/server/server-context.cpp | 4 +- tools/tts/tts.cpp | 8 +- 28 files changed, 2508 insertions(+), 8 deletions(-) create mode 100644 conversion/kokoro.py create mode 100644 models/templates/hexgrad-Kokoro-82M-tts.jinja create mode 100644 src/models/kokoro.cpp create mode 100644 tools/mtmd/models/kokoro.cpp diff --git a/conversion/__init__.py b/conversion/__init__.py index 0105099eb..9fc95f797 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -283,6 +283,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "Qwen3NextForCausalLM": "qwen", "Qwen3OmniMoeForConditionalGeneration": "qwen3vl", "PocketTTSModel": "pockettts", + "KokoroModel": "kokoro", "Zonos3Model": "zonos3", "YuE2ForCausalLM": "yue2", "MiniMaxMusic3ForConditionalGeneration": "minimax_music3", @@ -423,6 +424,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "Qwen3ASRForConditionalGeneration": "qwen3vl", "Qwen3OmniMoeForConditionalGeneration": "qwen3vl", "PocketTTSModel": "pockettts", + "KokoroModel": "kokoro", "SopranoModel": "soprano", "Qwen3TTSForConditionalGeneration": "qwen3tts", "Qwen3VLForConditionalGeneration": "qwen3vl", diff --git a/conversion/kokoro.py b/conversion/kokoro.py new file mode 100644 index 000000000..945eccdaa --- /dev/null +++ b/conversion/kokoro.py @@ -0,0 +1,419 @@ +from __future__ import annotations + +import importlib +import importlib.util +import json +import math +import re +from collections import Counter, defaultdict +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path +from typing import Any, Callable, Iterable + +import torch +from torch import Tensor + +from .base import ModelBase, TextModel, MmprojModel, gguf, logger + +# Kokoro (StyleTTS2 with an iSTFTNet decoder), https://huggingface.co/hexgrad/Kokoro-82M +# text model: the PL-BERT phoneme encoder (ALBERT), one token per phoneme character +# mmproj: bert_encoder, prosody predictor, text encoder, decoder and the voices +# the text to phonemes step is the "tts" chat template, its dictionaries come from the misaki and nltk python packages + +DEFAULT_VOICE = "af_heart" + +MODULE_PREFIX = { + "bert_encoder": "a.gen.bert_enc.", + "predictor": "a.gen.pred.", + "text_encoder": "a.gen.text.", + "decoder": "a.gen.dec.", +} + + +def _checkpoint(dir_model: Path) -> Path | None: + paths = sorted(dir_model.glob("kokoro-*.pth")) + return paths[0] if paths else None + + +def _is_kokoro_checkpoint(dir_model: Path) -> bool: + if _checkpoint(dir_model) is None or not (dir_model / "config.json").is_file(): + return False + with open(dir_model / "config.json", encoding="utf-8") as f: + config = json.load(f) + return "plbert" in config and "istftnet" in config and "vocab" in config + + +@ModelBase.register_hparams_loader(_is_kokoro_checkpoint) +def _load_kokoro_hparams(dir_model: Path) -> dict[str, Any]: + logger.info("gguf: detected Kokoro checkpoint") + with open(dir_model / "config.json", encoding="utf-8") as f: + config = json.load(f) + plbert = config["plbert"] + return { + "architectures": ["KokoroModel"], + "model_type": "kokoro", + "num_hidden_layers": plbert["num_hidden_layers"], + "hidden_size": plbert["hidden_size"], + "intermediate_size": plbert["intermediate_size"], + "num_attention_heads": plbert["num_attention_heads"], + "max_position_embeddings": plbert["max_position_embeddings"], + "vocab_size": config["n_token"], + "layer_norm_eps": 1e-12, # AlbertConfig default + "kokoro": config, + # the mmproj tensors are named explicitly, they do not go through the block tensor map + "audio_config": {"num_hidden_layers": 0}, + } + + +def _load_modules(dir_model: Path, modules: Iterable[str]) -> dict[str, dict[str, Tensor]]: + path = _checkpoint(dir_model) + assert path is not None + state = torch.load(path, map_location="cpu", mmap=True, weights_only=True) + # every module was saved from a DataParallel wrapper + return {m: {k.removeprefix("module."): v for k, v in state[m].items()} for m in modules} + + +def _g2p_fallback(lexicons: list[dict[str, Any]]) -> dict[str, Any]: + # model for the words missing from the lexicon: n-grams over units "letter:sounds", learned from the lexicon + # key "<0 to 4 units before> " -> {sounds: probability}; "" pads the start of a word, the letter "$" is its end + # the "tts" template decodes it with a beam search + order, min_count, stress = 5, 3, "\u02c8\u02cc" + vowels = set("AIOQWYaiu\u00e6\u0251\u0252\u0254\u0259\u025b\u025c\u026a\u028a\u028c\u1d7b\u1d4a") + + def sounds(ps: str) -> list[str]: + # a stress mark goes with the sound after it, a length mark with the one before + out: list[str] = [] + pre = "" + for c in ps: + if c in stress: + pre += c + elif c == "\u02d0" and out: + out[-1] += c + else: + out.append(pre + c) + pre = "" + return out + + def penalty(letter: str, chunk: str) -> float: + # a consonant letter says consonants only; a vowel letter says a vowel, with at most a glide + flat = [c for c in chunk if c not in stress and c != "\u02d0"] + n_vowel = sum(c in vowels for c in flat) + if letter in "bcdfgjkmnpqstvxz" and n_vowel > 0: + return -8.0 + if letter in "aeiou" and flat and (n_vowel == 0 or any(c not in vowels and c not in "jw" for c in flat)): + return -8.0 + return 0.0 + + words: dict[str, list[str]] = {} + for lexicon in lexicons: + for word, v in lexicon.items(): + ps = v["DEFAULT"] if isinstance(v, dict) else v + # same spelling normalization as the template: doubled consonants and "ck" say one sound + w = re.sub(r"([b-df-hj-np-tv-z])\1", r"\1", word.lower()).replace("ck", "k") + if ps and " " not in ps and len(w) >= 2 and re.fullmatch(r"[a-z']+", w) and w not in words: + t = sounds(ps) + if len(t) <= 2 * len(w): + words[w] = t + + def align(w: str, t: list[str], cost: dict[str, dict[str, float]]) -> list[str] | None: + # best split of the sounds over the letters, each letter takes 0, 1 or 2 of them + n_l, n_t = len(w), len(t) + two = [t[j] + t[j + 1] for j in range(n_t - 1)] + best = [[-1e18] * (n_t + 1) for _ in range(n_l + 1)] + back = [[0] * (n_t + 1) for _ in range(n_l + 1)] + best[0][0] = 0.0 + for i in range(n_l): + letter = w[i] + row = cost.setdefault(letter, {}) + cur, nxt, bk = best[i], best[i + 1], back[i + 1] + # the sounds left must fit in the letters left + for j in range(max(0, n_t - 2 * (n_l - i)), n_t + 1): + base = cur[j] + if base < -1e17: + continue + for k, chunk in ((0, ""), (1, t[j] if j < n_t else None), (2, two[j] if j + 1 < n_t else None)): + if chunk is None: + break + c = row.get(chunk) + if c is None: + # not seen yet for this letter + c = row[chunk] = -12.0 + penalty(letter, chunk) + if base + c > nxt[j + k]: + nxt[j + k] = base + c + bk[j + k] = k + if best[n_l][n_t] < -1e17: + return None + out: list[str] = [] + j = n_t + for i in range(n_l, 0, -1): + k = back[i][j] + out.append("".join(t[j - k:j])) + j -= k + return out[::-1] + + # hard EM: align with the current letter -> sounds costs, count, repeat + # the costs are learned on a sample of the words, then every word is aligned once + def step(items: Iterable[tuple[str, list[str]]], cost: dict[str, dict[str, float]]) -> list[tuple[str, list[str]]]: + return [(w, a) for w, a in ((w, align(w, t, cost)) for w, t in items) if a is not None] + + sample = list(words.items())[::max(1, len(words) // 20000)] + cost: dict[str, dict[str, float]] = {} + for _ in range(5): + counts: dict[str, Counter[str]] = defaultdict(Counter) + for w, a in step(sample, cost): + for c, o in zip(w, a): + counts[c][o] += 1 + # small cost for a silent letter, so that a sound stays on its own letter + cost = {c: {o: math.log(n / sum(cnt.values())) - (0.5 if o == "" else 0.0) + penalty(c, o) for o, n in cnt.items()} + for c, cnt in counts.items()} + aligned = step(words.items(), cost) + + # counts of a unit after each of its contexts, from 4 units down to none + ngram: dict[str, Counter[str]] = defaultdict(Counter) + n_ctx: Counter[str] = Counter() + for w, a in aligned: + units = [""] * (order - 1) + [c + ":" + o for c, o in zip(w, a)] + ["$:"] + for i in range(order - 1, len(units)): + letter, _, o = units[i].partition(":") + for n in range(order): + ctx = " ".join(units[i - n:i]) + ngram[(ctx + " " if ctx else "") + letter][o] += 1 + n_ctx[ctx] += 1 + + # the probability is over all the units after the context, not only the ones of this letter + # this tells the search how likely the letter is after the sounds it chose, so it can drop a bad choice + model: dict[str, Any] = {} + for key, cnt in ngram.items(): + ctx, _, letter = key.rpartition(" ") + # the two shortest contexts keep all the readings that the letter has min_count times or more + kept = {o: float(f"{n / n_ctx[ctx]:.4g}") for o, n in cnt.most_common() + if n >= min_count or (key.count(" ") < 2 and ngram[letter][o] >= min_count)} + if kept: + model[key] = kept + return model + + +def _g2p_dicts() -> dict[str, dict[str, Any]] | None: + # dictionaries of the "tts" template: the misaki lexicon and the nltk averaged perceptron tagger + spec = importlib.util.find_spec("misaki") + if spec is None or spec.origin is None: + logger.warning("misaki is not installed, the model will take phonemes as prompt (pip install misaki nltk)") + return None + data = Path(spec.origin).parent / "data" + try: + nltk = importlib.import_module("nltk") + base = Path(str(nltk.data.find("taggers/averaged_perceptron_tagger_eng"))) / "averaged_perceptron_tagger_eng" + except (ImportError, LookupError): + logger.warning("the nltk tagger is not available, the model will take phonemes as prompt " + "(pip install nltk && python -m nltk.downloader averaged_perceptron_tagger_eng)") + return None + + def load(path: Path) -> Any: + with open(path, encoding="utf-8") as f: + return json.load(f) + + # tagger: classes sorted by label, feature -> {"c": weight} + # (index keys: jinja2's map(attribute=...) splits on "." and some labels are punctuation) + classes = sorted(load(Path(f"{base}.classes.json"))) + cid = {c: f"c{i}" for i, c in enumerate(classes)} + weights = load(Path(f"{base}.weights.json")) + lex = { + "gold": load(data / "us_gold.json"), + "silver": load(data / "us_silver.json"), + "gold_gb": load(data / "gb_gold.json"), + "silver_gb": load(data / "gb_silver.json"), + } + logger.info("gguf: learning the pronunciation of unknown words from the lexicon, this takes a few minutes") + # one process per variant + with ProcessPoolExecutor(max_workers=2) as pool: + lts, lts_gb = pool.map(_g2p_fallback, [[lex["gold"], lex["silver"]], [lex["gold_gb"], lex["silver_gb"]]]) + return { + **lex, + "lts": lts, + "lts_gb": lts_gb, + "pos_weights": {f: {cid[t]: w for t, w in tw.items()} for f, tw in weights.items()}, + "pos_tagdict": load(Path(f"{base}.tagdict.json")), + "pos_classes": {"": classes}, + } + + +@ModelBase.register("KokoroModel") +@ModelBase.example("hexgrad/Kokoro-82M") +class KokoroTextModel(TextModel): + model_arch = gguf.MODEL_ARCH.KOKORO + + _LAYER = "encoder.albert_layer_groups.0.albert_layers.0." + _TENSOR_MAP = { + "embeddings.LayerNorm": gguf.MODEL_TENSOR.TOKEN_EMBD_NORM, + "encoder.embedding_hidden_mapping_in": gguf.MODEL_TENSOR.TOKEN_EMBD_PROJ, + } + # the layers share their weights, stored as block 0 + _LAYER_TENSOR_MAP = { + "attention.query": gguf.MODEL_TENSOR.ATTN_Q, + "attention.key": gguf.MODEL_TENSOR.ATTN_K, + "attention.value": gguf.MODEL_TENSOR.ATTN_V, + "attention.dense": gguf.MODEL_TENSOR.ATTN_OUT, + "attention.LayerNorm": gguf.MODEL_TENSOR.ATTN_OUT_NORM, + "ffn": gguf.MODEL_TENSOR.FFN_UP, + "ffn_output": gguf.MODEL_TENSOR.FFN_DOWN, + "full_layer_layer_norm": gguf.MODEL_TENSOR.LAYER_OUT_NORM, + } + + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + del remote_hf_model_id + bert = _load_modules(self.dir_model, ["bert"])["bert"] + return {name: (lambda t=t: t) for name, t in bert.items()} + + def set_vocab(self): + config = self.hparams["kokoro"] + n_vocab = config["n_token"] + tokens: list[str] = [f"[UNUSED_{i}]" for i in range(n_vocab)] + toktypes = [gguf.TokenType.UNUSED] * n_vocab + for text, i in config["vocab"].items(): + tokens[i] = text + toktypes[i] = gguf.TokenType.NORMAL + # id 0 pads both ends of the phoneme sequence + tokens[0] = "" + toktypes[0] = gguf.TokenType.CONTROL + + # the phonemes are mapped to tokens one character at a time by the audio pipeline + self.gguf_writer.add_tokenizer_model("llama") + self.gguf_writer.add_tokenizer_pre("default") + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_scores([0.0] * n_vocab) + self.gguf_writer.add_token_types(toktypes) + self.gguf_writer.add_bos_token_id(0) + self.gguf_writer.add_eos_token_id(0) + self.gguf_writer.add_pad_token_id(0) + self.gguf_writer.add_add_bos_token(False) + self.gguf_writer.add_add_eos_token(False) + + dicts = _g2p_dicts() + if dicts is not None: + template_path = Path(__file__).parent.parent / "models" / "templates" / "hexgrad-Kokoro-82M-tts.jinja" + with open(template_path, encoding="utf-8") as f: + self.gguf_writer.add_chat_template([{"name": "tts", "template": f.read()}]) + self.gguf_writer.add_tts_dicts(dicts) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + self.gguf_writer.add_causal_attention(False) + self.gguf_writer.add_pooling_type(gguf.PoolingType.NONE) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + del bid + n_embd = self.hparams["hidden_size"] + + if name == "embeddings.word_embeddings.weight": + # zero-pad the rows to n_embd, llama.cpp wants token and embedding inputs of the same width + data_torch = torch.nn.functional.pad(data_torch, (0, n_embd - data_torch.shape[1])) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), data_torch) + return + if name == "embeddings.position_embeddings.weight": + # every token has type 0, fold its embedding into the positions + type0 = self.model_tensors["embeddings.token_type_embeddings.weight"]()[0] + yield (self.format_tensor_name(gguf.MODEL_TENSOR.POS_EMBD), data_torch + type0) + return + if name.startswith(("embeddings.token_type_embeddings.", "pooler.")): + return + + key, suffix = name.rsplit(".", 1) + if key in self._TENSOR_MAP: + yield (self.format_tensor_name(self._TENSOR_MAP[key], suffix="." + suffix), data_torch) + return + if key.startswith(self._LAYER): + tensor = self._LAYER_TENSOR_MAP[key.removeprefix(self._LAYER)] + yield (self.format_tensor_name(tensor, 0, suffix="." + suffix), data_torch) + return + raise ValueError(f"unexpected tensor {name}") + + +@ModelBase.register("KokoroModel") +@ModelBase.example("hexgrad/Kokoro-82M") +class KokoroMmprojModel(MmprojModel): + has_vision_encoder = False + has_audio_encoder = False + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._wnorm_g: dict[str, Tensor] = {} + self._wnorm_v: dict[str, Tensor] = {} + self._lstm_bias: dict[str, Tensor] = {} + + def get_audio_config(self) -> dict[str, Any] | None: + return self.global_config.get("audio_config") + + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + del remote_hf_model_id + tensors: dict[str, Callable[[], Tensor]] = {} + for module, state in _load_modules(self.dir_model, MODULE_PREFIX.keys()).items(): + for name, t in state.items(): + tensors[MODULE_PREFIX[module] + name] = (lambda t=t: t) + return tensors + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + # the names are final already + return item + + def set_gguf_parameters(self): + config = self.global_config["kokoro"] + self.gguf_writer.add_file_type(self.ftype) + self.gguf_writer.add_clip_has_gen_audio_encoder(True) + self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.KOKORO) + self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text) + self.gguf_writer.add_gen_audio_embedding_length(config["hidden_dim"]) + self.gguf_writer.add_gen_audio_feed_forward_length(config["hidden_dim"]) + self.gguf_writer.add_gen_audio_block_count(config["n_layer"]) + self.gguf_writer.add_gen_audio_head_count(1) + self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-5) + if (self.dir_model / "voices" / f"{DEFAULT_VOICE}.pt").is_file(): + self.gguf_writer.add_gen_audio_default_voice(DEFAULT_VOICE) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + del bid + # torch weight_norm: weight = g * v / |v| over dims 1..n + if name.endswith((".weight_g", ".weight_v")): + base = name[:-len(".weight_g")] + (self._wnorm_g if name.endswith("_g") else self._wnorm_v)[base] = data_torch + if base in self._wnorm_g and base in self._wnorm_v: + g = self._wnorm_g.pop(base).float() + v = self._wnorm_v.pop(base).float() + w = g * v / v.norm(dim=tuple(range(1, v.dim())), keepdim=True) + if base.endswith(".pool"): + # depthwise transposed conv [C, 1, 3], stored as one row per tap + w = w.squeeze(1).T.contiguous() + yield (base + ".weight", w) + return + # the two LSTM biases are always added + if ".bias_ih_l0" in name or ".bias_hh_l0" in name: + base = name.replace(".bias_ih_l0", ".bias_l0").replace(".bias_hh_l0", ".bias_l0") + if base in self._lstm_bias: + yield (base, self._lstm_bias.pop(base).float() + data_torch.float()) + else: + self._lstm_bias[base] = data_torch + return + # snake alphas are per channel + if ".alpha1." in name or ".alpha2." in name: + data_torch = data_torch.reshape(-1) + yield (name, data_torch) + + def tensor_force_quant(self, name, new_name, bid, n_dims): + del name, bid + # tensors read on the host stay F32 + if "lstm" in new_name or ".shared." in new_name or new_name.startswith(("a.gen.bert_enc.", "voice.")) \ + or ".duration_proj." in new_name or ".m_source." in new_name or ".pool." in new_name: + return gguf.GGMLQuantizationType.F32 + # CONV_TRANSPOSE_1D takes only F32 weights on some backends + if ".ups." in new_name: + return gguf.GGMLQuantizationType.F32 + # the conv ops have no BF16 kernels + if n_dims >= 2: + return gguf.GGMLQuantizationType.F16 + return gguf.GGMLQuantizationType.F32 + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + # a voice is a table of style vectors, the row is picked by the number of phonemes + for path in sorted((self.dir_model / "voices").glob("*.pt")): + pack = torch.load(path, map_location="cpu", weights_only=True) + yield (f"voice.{path.stem}", pack.squeeze(1).float()) diff --git a/docs/multimodal/tts.md b/docs/multimodal/tts.md index 5ebc10356..45e838da5 100644 --- a/docs/multimodal/tts.md +++ b/docs/multimodal/tts.md @@ -15,6 +15,7 @@ Supported models: - [Fish Audio S2 Pro](#fish-audio-s2-pro) - [YuE2](#yue2) (songs) - [MiniMax Music 3](#minimax-music-3) (songs) +- [Kokoro](#kokoro) The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)). @@ -363,3 +364,62 @@ python convert_hf_to_gguf.py MiniMax-Music3 --mmproj --outfile mmproj-minimax-mu ``` The mmproj holds the depth decoder and the DiT layers (in the `--outtype` type), and the condition encoder, the small DiT projections and the DAC decoder, always in F32. + +## Kokoro + +Available params: +- `--tts-voice` selects one of the voices shipped with the weights (default: `af_heart`); `llama-tts` lists them when the name is unknown. The first letter of a voice is its language: `a` American English, `b` British English, `e` Spanish, `f` French, `h` Hindi, `i` Italian, `p` Brazilian Portuguese, `j` Japanese, `z` Mandarin +- `--tts-lang` can be `en-us` or `en-gb`; when it is not given, the voices named `b*` speak British English and those named `a*` American English +- Note: `--tts-speaker-file` is not supported, the model has no voice cloning +- Note: sampling params are not used, the model is not autoregressive. `--seed` only changes the noise of the vocoder source +- `speed` (`llama-server` only) changes the speaking rate, e.g. `1.5`; `loudness` is not supported + +Example usage: + +```sh +llama-tts -m kokoro.gguf \ + -mm mmproj-kokoro.gguf \ + -p "Hello world" \ + --tts-voice am_michael \ + --output out.wav +``` + +The model speaks phonemes. Only English has a text to phonemes step (the model's `tts` chat template, with the pronunciation dictionary stored in the model): +- the pronunciation of a word missing from the dictionary (mostly rare names) is predicted from its spelling, by a model that the converter learns from the dictionary. The reference uses espeak-ng for these words; the prediction is about as accurate, but it is not the same, so such a word can sound different from the reference +- a text longer than 510 phonemes is split at sentence ends and spoken segment by segment + +**Other languages, or your own phonemes:** + +The prompt of a voice of another language must already be phonemes, in the symbols of the model (see `vocab` in its `config.json`). Run the text to phonemes step in your own code, for example with [espeak-ng](https://github.com/espeak-ng/espeak-ng), which the reference uses for Spanish, French, Hindi, Italian and Brazilian Portuguese: + +```sh +PHONEMES=$(echo "El gobierno anunció nuevas medidas." | espeak-ng -q --ipa -v es --stdin | tr '\n' ' ') +llama-tts -m kokoro.gguf \ + -mm mmproj-kokoro.gguf \ + -p "$PHONEMES" \ + --tts-voice ef_dora \ + --output out.wav +``` + +Notes: +- espeak-ng drops the punctuation, which the model uses for pauses: phonemize the text between the marks and put them back for better prosody +- the reference also replaces a few sounds by single symbols (e.g. `tʃ` by `ʧ`), see `espeak.py` of [misaki](https://github.com/hexgrad/misaki) +- with `llama-server`, send `"raw_prompt": true` to pass phonemes with any voice, including the English ones + +**Note for GGUF conversion:** + +Download `config.json`, `kokoro-v1_0.pth` and the `voices/*.pt` files you want from [Kokoro-82M](https://huggingface.co/hexgrad/Kokoro-82M), every voice in the directory is stored in the mmproj: + +```sh +python convert_hf_to_gguf.py path/to/Kokoro-82M --outfile kokoro.gguf +python convert_hf_to_gguf.py path/to/Kokoro-82M --mmproj --outfile mmproj-kokoro.gguf +``` + +The pronunciation dictionary and the part-of-speech tagger of the `tts` template come from the `misaki` and `nltk` python packages. The converter also learns the spelling-to-sound model for unknown words from that dictionary, which adds about 15 seconds: + +```sh +pip install misaki nltk +python -m nltk.downloader averaged_perceptron_tagger_eng +``` + +Without them the converter writes no `tts` template, and the prompt must be a phoneme string (as produced by misaki). diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index d15ca9a5f..d55e5a96f 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -770,6 +770,7 @@ class MODEL_ARCH(IntEnum): QWEN3TTS = auto() POCKETTTS = auto() ZONOS3 = auto() + KOKORO = auto() K2HORIZON = auto() KOLIBRI1 = auto() LIMITE = auto() @@ -794,6 +795,7 @@ class VISION_PROJECTOR_TYPE(IntEnum): class MODEL_TENSOR(IntEnum): TOKEN_EMBD = auto() TOKEN_EMBD_NORM = auto() + TOKEN_EMBD_PROJ = auto() MASKED_EMBD_CENTROIDS= auto() MASKED_EMBD_ORDERING = auto() TOKEN_TYPES = auto() @@ -1755,6 +1757,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.QWEN3TTS: "qwen3tts", MODEL_ARCH.POCKETTTS: "pockettts", MODEL_ARCH.ZONOS3: "zonos3", + MODEL_ARCH.KOKORO: "kokoro", MODEL_ARCH.K2HORIZON: "k2-horizon", MODEL_ARCH.KOLIBRI1: "kolibri1", MODEL_ARCH.LIMITE: "limite", @@ -1777,6 +1780,7 @@ VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = { TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.TOKEN_EMBD: "token_embd", MODEL_TENSOR.TOKEN_EMBD_NORM: "token_embd_norm", + MODEL_TENSOR.TOKEN_EMBD_PROJ: "token_embd_proj", MODEL_TENSOR.TOKEN_TYPES: "token_types", MODEL_TENSOR.MASKED_EMBD_CENTROIDS: "masked_embd_centroids", MODEL_TENSOR.MASKED_EMBD_ORDERING: "masked_embd_ordering", @@ -6862,6 +6866,20 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.KOKORO: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.TOKEN_EMBD_NORM, + MODEL_TENSOR.TOKEN_EMBD_PROJ, + MODEL_TENSOR.POS_EMBD, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_OUT_NORM, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.LAYER_OUT_NORM, + ], MODEL_ARCH.ZONOS3: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, @@ -7337,6 +7355,7 @@ class VisionProjectorType: MINIMAX_MUSIC3_GEN = "minimax_music3_gen" # audio generation: RVQ depth decoder + flow-matching DiT + DAC decoder FISH_S2_GEN = "fish_s2_gen" # audio generation: fast AR code predictor + Fish DAC decoder FISH_S2_SPKENC = "fish_s2_spkenc" # audio: Fish DAC encoder + RVQ, reference clip -> prompt rows + KOKORO = "kokoro" # audio generation: StyleTTS2 prosody predictor + iSTFTNet decoder HUNYUANVL = "hunyuanvl" PARAKEET = "parakeet" # audio MINIMAXM3 = "minimax_m3" diff --git a/models/templates/hexgrad-Kokoro-82M-tts.jinja b/models/templates/hexgrad-Kokoro-82M-tts.jinja new file mode 100644 index 000000000..7b129cbe1 --- /dev/null +++ b/models/templates/hexgrad-Kokoro-82M-tts.jinja @@ -0,0 +1,585 @@ +{#- Kokoro "tts" template: english text -> phonemes, a port of the english path of misaki (the reference G2P) + functions: tts_norm_scan(), tts_norm_sub(), tts_dict(), see docs/tts-norm-patterns.md + dictionaries, written by conversion/kokoro.py: + - gold / silver (US), gold_gb / silver_gb (british): the misaki lexicon, a value is a string or {DEFAULT, ...} + - pos_weights / pos_tagdict / pos_classes: the nltk averaged perceptron tagger + - lts / lts_gb: model for the words missing from the lexicon, see lts_search() + a voice of another language takes phonemes, its prompt passes through -#} +{%- set unk = unk if unk is defined else '' -%} +{#- british english: the voices named b*, or language en-gb; it has its own lexicon and a few suffix rules -#} +{%- set lang = (language if language is defined and language else '') | lower | replace('_', '-') -%} +{%- set GB = lang in ['en-gb', 'british'] or (not lang and voice is defined and voice and voice[:1] == 'b') -%} +{%- set GOLD = 'gold_gb' if GB else 'gold' -%} +{%- set SILVER = 'silver_gb' if GB else 'silver' -%} +{%- set LTS = 'lts_gb' if GB else 'lts' -%} +{%- set V = 'AIOQWYaiuæɑɒɔəɛɜɪʊʌᵻ' -%} +{%- set C = 'bdfhjklmnpstvwzðŋɡɹɾʃʒʤʧθ' -%} +{%- set NQP = ';:,.!?—…' -%} +{%- set TAUS = 'AIOWYiuæɑəɛɪɹʊʌ' -%} +{%- set SYMBOLS = {'%': 'percent', '&': 'and', '+': 'plus', '@': 'at'} -%} +{#- the lexicon has only ASCII keys: an accented letter is read as its base letter -#} +{%- set FOLD = [['[àáâãäåāă]', 'a'], ['[ÀÁÂÃÄÅĀĂ]', 'A'], ['[çćč]', 'c'], ['[ÇĆČ]', 'C'], ['[èéêëēėęě]', 'e'], ['[ÈÉÊËĒĖĘĚ]', 'E'], + ['[ìíîïīı]', 'i'], ['[ÌÍÎÏĪİ]', 'I'], ['[ñńň]', 'n'], ['[ÑŃŇ]', 'N'], ['[òóôõöøōő]', 'o'], ['[ÒÓÔÕÖØŌŐ]', 'O'], + ['[ùúûüūůű]', 'u'], ['[ÙÚÛÜŪŮŰ]', 'U'], ['[ýÿ]', 'y'], ['[ÝŸ]', 'Y'], ['[śšș]', 's'], ['[ŚŠȘ]', 'S'], ['[źżž]', 'z'], ['[ŹŻŽ]', 'Z'], + ['ł', 'l'], ['Ł', 'L'], ['ß', 'ss'], ['æ', 'ae'], ['Æ', 'AE'], ['œ', 'oe'], ['Œ', 'OE']] -%} +{%- set CURRENCIES = {'$': ['dollar', 'cent'], '£': ['pound', 'pence'], '€': ['euro', 'cent']} -%} + +{#- ---------- small helpers ---------- -#} +{%- macro full(s, pat) -%}{%- set segs = tts_norm_scan(s, '^(?:' ~ pat ~ ')$') -%}{{ '1' if segs | length == 1 and segs[0].rule else '' }}{%- endmacro -%} +{#- key of w in a lexicon, or ''; like misaki's grow_dictionary(), a lowercase entry also answers for Capitalized and vice versa -#} +{%- macro dkey(name, w) -%} +{%- if tts_dict(name, w) is not none -%}{{ w }} +{%- elif w | length > 1 -%} + {%- set l = w | lower -%}{%- set c = l | capitalize -%} + {%- if w == c and w != l and tts_dict(name, l) is not none -%}{{ l }} + {%- elif w == l and w != c and tts_dict(name, c) is not none -%}{{ c }} + {%- endif -%} +{%- endif -%} +{%- endmacro -%} +{%- macro has(w) -%}{{ '1' if dkey(GOLD, w) or dkey(SILVER, w) else '' }}{%- endmacro -%} +{#- default pronunciation in gold, or '' -#} +{%- macro gold(w) -%} +{%- set k = dkey(GOLD, w) -%} +{%- if k -%}{%- set e = tts_dict(GOLD, k) -%}{{ e.DEFAULT if e is mapping else e }}{%- endif -%} +{%- endmacro -%} + +{#- ---------- words missing from the lexicon ---------- -#} +{#- beam search over the letters with an n-gram model of units "letter:sounds" + tts_dict(LTS, "<0 to 4 units before> ") -> {sounds: probability}; "" pads the start, the letter "$" is the end + a unit takes its score from the longest context that has it, times 0.4 for each unit dropped + fixed is none, or the sounds of each letter without stress: then only the stress is searched, with exactly one primary stress + returns the sounds of each letter, joined by "|" -#} +{%- macro lts_search(word, fixed) -%} +{%- set bm = namespace(l=[{'p': 1.0, 'h': ['', '', '', ''], 'o': [], 's': false}]) -%} +{%- for ch in (tts_norm_scan(word, '.') | map(attribute='text') | list) + ['$'] -%} + {%- set i = loop.index0 -%} + {%- set nx = namespace(l=[]) -%} + {%- for hyp in bm.l -%} + {%- set st = namespace(f=1.0, seen=[]) -%} + {%- for n in range(4, -1, -1) -%} + {%- set d = tts_dict(LTS, (hyp.h[4 - n:] + [ch]) | join(' ')) -%} + {%- if d is not none -%} + {%- for o, p in d | items -%} + {%- if o not in st.seen -%} + {%- set _ = st.seen.append(o) -%} + {%- set prim = 'ˈ' in o -%} + {%- set match = fixed is none or ch == '$' or (o | replace('ˈ', '') | replace('ˌ', '')) == fixed[i] -%} + {%- if match and not (fixed is not none and prim and hyp.s) -%} + {%- set _ = nx.l.append({'p': hyp.p * st.f * p, 'h': (hyp.h + [ch ~ ':' ~ o])[-4:], 'o': hyp.o + [o], 's': hyp.s or prim}) -%} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- set st.f = st.f * 0.4 -%} + {%- endfor -%} + {%- endfor -%} + {%- set srt = nx.l | sort(attribute='p', reverse=true) -%} + {%- if fixed is none -%}{%- set bm.l = srt[:16] -%} + {%- else -%}{%- set bm.l = (srt | selectattr('s') | list)[:8] + (srt | rejectattr('s') | list)[:8] -%} + {%- endif -%} +{%- endfor -%} +{%- set done = (bm.l | selectattr('s') | list) or bm.l -%} +{%- if done -%}{{ (done | sort(attribute='p', reverse=true))[0].o[:-1] | join('|') }}{%- endif -%} +{%- endmacro -%} + +{#- pronunciation of a word that is not in the lexicon, '' if it cannot be read -#} +{%- macro lts(word) -%} +{%- set w = namespace(s=word | lower | replace('’', "'")) -%} +{#- doubled consonants and "ck" say one sound -#} +{%- for c in ['b', 'c', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'm', 'n', 'p', 'q', 'r', 's', 't', 'v', 'w', 'x', 'y', 'z'] -%}{%- set w.s = w.s | replace(c ~ c, c) -%}{%- endfor -%} +{%- set w.s = w.s | replace('ck', 'k') -%} +{%- if w.s | length >= 2 and full(w.s, "[a-z']+") -%} + {#- first the sounds, then the stress marks on those sounds -#} + {%- set first = lts_search(w.s, none) -%} + {%- if first or w.s | length == 0 -%} + {%- set sounds = first.split('|') | map('replace', 'ˈ', '') | map('replace', 'ˌ', '') | list -%} + {%- if sounds | length == w.s | length -%}{{ lts_search(w.s, sounds) | replace('|', '') }}{%- endif -%} + {%- endif -%} +{%- endif -%} +{%- endmacro -%} + +{#- a word missing from the lexicon: the predicted pronunciation; a word that cannot be read (e.g. "cpp") is spelled out -#} +{%- macro fallback(word) -%} +{%- set ps = lts(word) if tts_norm_scan(word, '[aeiouyAEIOUY]') | selectattr('rule') | list else '' -%} +{%- if ps and tts_norm_scan(ps, '[' ~ V ~ ']') | selectattr('rule') | list -%}{{ ps }} +{%- else -%}{{ get_nnp(word) or unk }} +{%- endif -%} +{%- endmacro -%} + +{%- macro parent_tag(tag) -%} +{%- if tag.startswith('VB') -%}VERB +{%- elif tag.startswith('NN') -%}NOUN +{%- elif tag.startswith('ADV') or tag.startswith('RB') -%}ADV +{%- elif tag.startswith('ADJ') or tag.startswith('JJ') -%}ADJ +{%- else -%}{{ tag }} +{%- endif -%} +{%- endmacro -%} + +{%- macro apply_stress(ps, stress) -%} +{%- if stress is none or not ps -%}{{ ps }} +{%- elif stress < -1 -%}{{ ps | replace('ˈ', '') | replace('ˌ', '') }} +{%- elif stress == -1 or ((stress == 0 or stress == -0.5) and 'ˈ' in ps) -%}{{ ps | replace('ˌ', '') | replace('ˈ', 'ˌ') }} +{%- elif (stress == 0 or stress == 0.5 or stress == 1) and 'ˈ' not in ps and 'ˌ' not in ps -%}{{ tts_norm_sub(ps, '^([^' ~ V ~ ']*)([' ~ V ~ '])', '%1ˌ%2') }} +{%- elif stress >= 1 and 'ˈ' not in ps and 'ˌ' in ps -%}{{ ps | replace('ˌ', 'ˈ') }} +{%- elif stress > 1 and 'ˈ' not in ps and 'ˌ' not in ps -%}{{ tts_norm_sub(ps, '^([^' ~ V ~ ']*)([' ~ V ~ '])', '%1ˈ%2') }} +{%- else -%}{{ ps }} +{%- endif -%} +{%- endmacro -%} + +{#- spell out letters: "NASA" -> N A S A, "e.g." -> E G -#} +{%- macro get_nnp(word) -%} +{%- set ns = namespace(ps='', ok=true) -%} +{%- for c in tts_norm_scan(word, '%a') -%} + {%- if c.rule -%} + {%- set p = gold(c.text | upper) -%} + {%- if not p -%}{%- set ns.ok = false -%}{%- else -%}{%- set ns.ps = ns.ps ~ p -%}{%- endif -%} + {%- endif -%} +{%- endfor -%} +{%- if ns.ok and ns.ps -%}{{ tts_norm_sub(apply_stress(ns.ps, 0), 'ˌ([^ˌ]*)$', 'ˈ%1') }}{%- endif -%} +{%- endmacro -%} + +{#- tag: Penn tag or '' ; fv = future vowel context: 'none' (end / punctuation follows), 'v', 'c', or '' (no context) -#} +{%- macro lookup(word, tag, stress, fv) -%} +{%- set ns = namespace(w=word, nnp=false, ps=none, t='') -%} +{%- if word == word | upper and not dkey(GOLD, word) -%}{%- set ns.w = word | lower -%}{%- set ns.nnp = tag == 'NNP' -%}{%- endif -%} +{%- set k = dkey(GOLD, ns.w) -%} +{%- if k -%} + {%- set e = tts_dict(GOLD, k) -%} + {%- if e is mapping -%} + {#- POS variants: misaki's ps.get(tag) with 'None' (no following vowel context) and parent tag fallback -#} + {%- if fv == 'none' and 'None' in e -%}{%- set ns.t = 'None' -%} + {%- elif tag and tag in e -%}{%- set ns.t = tag -%} + {%- else -%}{%- set ns.t = parent_tag(tag) -%} + {%- endif -%} + {#- a null variant means "no pronunciation for this tag": spell the word out -#} + {%- set ns.ps = (e[ns.t] or '') if ns.t and ns.t in e else e.DEFAULT -%} + {%- else -%}{%- set ns.ps = e -%} + {%- endif -%} +{%- elif not ns.nnp -%} + {%- set k = dkey(SILVER, ns.w) -%} + {%- set ns.ps = tts_dict(SILVER, k) if k else none -%} +{%- endif -%} +{%- set r = get_nnp(ns.w) if (not ns.ps or (ns.nnp and 'ˈ' not in ns.ps)) else '' -%} +{%- if r -%}{{ r }}{%- else -%}{{ apply_stress(ns.ps or '', stress) }}{%- endif -%} +{%- endmacro -%} + +{%- macro is_known(word) -%} +{%- if has(word) or word in SYMBOLS -%}1 +{%- elif not full(word, '[A-Za-z]+') -%} +{%- elif word | length == 1 -%}1 +{%- elif word == word | upper and dkey(GOLD, word | lower) -%}1 +{%- elif word[1:] == word[1:] | upper -%}1 +{%- endif -%} +{%- endmacro -%} + +{#- ---------- suffixes ---------- -#} +{%- macro sfx_s(stem) -%} +{%- if not stem -%} +{%- elif stem[-1:] in 'ptkfθ' -%}{{ stem }}s +{%- elif stem[-1:] in 'szʃʒʧʤ' -%}{{ stem }}{{ 'ɪ' if GB else 'ᵻ' }}z +{%- else -%}{{ stem }}z +{%- endif -%} +{%- endmacro -%} + +{%- macro stem_s(word, tag, stress, fv) -%} +{%- set n = word | length -%} +{%- if n < 3 or not word.endswith('s') -%} +{%- elif not word.endswith('ss') and is_known(word[:-1]) -%}{{ sfx_s(lookup(word[:-1], tag, stress, fv)) }} +{%- elif (word.endswith("'s") or (n > 4 and word.endswith('es') and not word.endswith('ies'))) and is_known(word[:-2]) -%}{{ sfx_s(lookup(word[:-2], tag, stress, fv)) }} +{%- elif n > 4 and word.endswith('ies') and is_known(word[:-3] ~ 'y') -%}{{ sfx_s(lookup(word[:-3] ~ 'y', tag, stress, fv)) }} +{%- endif -%} +{%- endmacro -%} + +{%- macro sfx_ed(stem) -%} +{%- if not stem -%} +{%- elif stem[-1:] in 'pkfθʃsʧ' -%}{{ stem }}t +{%- elif stem[-1:] == 'd' -%}{{ stem }}{{ 'ɪ' if GB else 'ᵻ' }}d +{%- elif stem[-1:] != 't' -%}{{ stem }}d +{%- elif GB or stem | length < 2 -%}{{ stem }}ɪd +{%- elif stem[-2:-1] in TAUS -%}{{ stem[:-1] }}ɾᵻd +{%- else -%}{{ stem }}ᵻd +{%- endif -%} +{%- endmacro -%} + +{%- macro stem_ed(word, tag, stress, fv) -%} +{%- set n = word | length -%} +{%- if n < 4 or not word.endswith('d') -%} +{%- elif not word.endswith('dd') and is_known(word[:-1]) -%}{{ sfx_ed(lookup(word[:-1], tag, stress, fv)) }} +{%- elif n > 4 and word.endswith('ed') and not word.endswith('eed') and is_known(word[:-2]) -%}{{ sfx_ed(lookup(word[:-2], tag, stress, fv)) }} +{%- endif -%} +{%- endmacro -%} + +{%- macro sfx_ing(stem) -%} +{%- if not stem -%} +{%- elif GB -%}{%- if stem[-1:] not in 'əː' -%}{{ stem }}ɪŋ{%- endif -%} +{%- elif stem | length > 1 and stem[-1:] == 't' and stem[-2:-1] in TAUS -%}{{ stem[:-1] }}ɾɪŋ +{%- else -%}{{ stem }}ɪŋ +{%- endif -%} +{%- endmacro -%} + +{%- macro stem_ing(word, tag, stress, fv) -%} +{%- set n = word | length -%} +{%- if n < 5 or not word.endswith('ing') -%} +{%- elif n > 5 and is_known(word[:-3]) -%}{{ sfx_ing(lookup(word[:-3], tag, stress, fv)) }} +{%- elif is_known(word[:-3] ~ 'e') -%}{{ sfx_ing(lookup(word[:-3] ~ 'e', tag, stress, fv)) }} +{%- elif n > 5 and tts_norm_scan(word, '(?:bb|cc|dd|gg|kk|ll|mm|nn|pp|rr|ss|tt|vv|xx|zz|ck)ing$') | selectattr('rule') | list and is_known(word[:-4]) -%}{{ sfx_ing(lookup(word[:-4], tag, stress, fv)) }} +{%- endif -%} +{%- endmacro -%} + +{#- ---------- words ---------- -#} +{#- misaki Lexicon.get_special_case(); fto = the next word is "to" (for "used to") -#} +{%- macro special(word, tag, stress, fv, fto) -%} +{%- if word in SYMBOLS -%}{{ lookup(SYMBOLS[word], '', none, fv) }} +{%- elif word == 'a' or word == 'A' -%}{{ 'ɐ' if tag == 'DT' else 'ˈA' }} +{%- elif word in ['am', 'Am', 'AM'] -%} + {%- if tag.startswith('NN') -%}{{ get_nnp(word) }} + {%- elif fv == 'none' or word != 'am' or (stress is not none and stress > 0) -%}{{ gold('am') }} + {%- else -%}ɐm{%- endif -%} +{%- elif word in ['an', 'An', 'AN'] -%}{{ get_nnp(word) if (word == 'AN' and tag.startswith('NN')) else 'ɐn' }} +{%- elif word == 'I' and tag == 'PRP' -%}ˌI +{%- elif word in ['by', 'By', 'BY'] and parent_tag(tag) == 'ADV' -%}bˈI +{%- elif word in ['to', 'To'] or (word == 'TO' and tag in ['TO', 'IN']) -%}{{ {'v': 'tʊ', 'c': 'tə'}.get(fv, gold('to')) }} +{%- elif word in ['in', 'In'] or (word == 'IN' and tag != 'NNP') -%}{{ 'ˈ' if (fv == 'none' or tag != 'IN') else '' }}ɪn +{%- elif word in ['the', 'The'] or (word == 'THE' and tag == 'DT') -%}{{ 'ði' if fv == 'v' else 'ðə' }} +{%- elif tag == 'IN' and full(word, '[Vv][Ss]%.?') -%}{{ lookup('versus', '', none, fv) }} +{%- elif word in ['used', 'Used', 'USED'] -%}{{ tts_dict(GOLD, 'used').VBD if (tag in ['VBD', 'JJ'] and fto) else gold('used') }} +{%- endif -%} +{%- endmacro -%} + +{%- macro get_word(word0, tag, stress, fv, fto) -%} +{%- set sp = special(word0, tag, stress, fv, fto) -%} +{%- if sp -%}{{ sp }}{%- else -%} +{%- set ns = namespace(word=word0) -%} +{%- set wl = word0 | lower -%} +{%- if word0 | length > 1 and full(word0 | replace("'", ''), '%a+') and word0 != wl and (tag != 'NNP' or word0 | length > 7) and not has(word0) + and (word0 == word0 | upper or word0[1:] == word0[1:] | lower) + and (has(wl) or stem_s(wl, tag, stress, fv) or stem_ed(wl, tag, stress, fv) or stem_ing(wl, tag, stress, fv)) -%} + {%- set ns.word = wl -%} +{%- endif -%} +{%- set w = ns.word -%} +{%- if is_known(w) -%}{{ lookup(w, tag, stress, fv) }} +{%- elif w.endswith("s'") and is_known(w[:-2] ~ "'s") -%}{{ lookup(w[:-2] ~ "'s", tag, stress, fv) }} +{%- elif w.endswith("'") and is_known(w[:-1]) -%}{{ lookup(w[:-1], tag, stress, fv) }} +{%- else -%} + {%- set r = stem_s(w, tag, stress, fv) -%} + {%- if not r -%}{%- set r = stem_ed(w, tag, stress, fv) -%}{%- endif -%} + {%- if not r -%}{%- set r = stem_ing(w, tag, 0.5 if stress is none else stress, fv) -%}{%- endif -%} + {{- r -}} +{%- endif -%} +{%- endif -%} +{%- endmacro -%} + +{%- macro lex_word(word0, tag, fv, fto=false) -%} +{%- set w = word0 | replace('‘', "'") | replace('’', "'") -%} +{%- set stress = none if w == w | lower else (2 if w == w | upper else 0.5) -%} +{{- get_word(w, tag, stress, fv, fto) -}} +{%- endmacro -%} + +{#- ---------- numbers (num2words-compatible word lists, "and" dropped like misaki) ---------- -#} +{%- set ONES = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'eleven', 'twelve', + 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'] -%} +{%- set TENS = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'] -%} +{%- set SCALES = [[1000000000000, 'trillion'], [1000000000, 'billion'], [1000000, 'million'], [1000, 'thousand']] -%} +{%- macro card(n) -%} +{%- if n < 20 -%}{{ ONES[n] }} +{%- elif n < 100 -%}{{ TENS[n // 10] }}{{ ' ' ~ ONES[n % 10] if n % 10 else '' }} +{%- elif n < 1000 -%}{{ ONES[n // 100] }} hundred{{ ' ' ~ card(n % 100) if n % 100 else '' }} +{%- else -%} + {%- set ns = namespace(done=false) -%} + {%- for sc in SCALES -%} + {%- if not ns.done and n >= sc[0] -%} + {{- card(n // sc[0]) }} {{ sc[1] }}{{ ' ' ~ card(n % sc[0]) if n % sc[0] else '' -}} + {%- set ns.done = true -%} + {%- endif -%} + {%- endfor -%} +{%- endif -%} +{%- endmacro -%} + +{%- macro ordw(n) -%} +{%- set ws = card(n).split(' ') -%} +{%- set last = ws[-1] -%} +{%- set IRR = {'one': 'first', 'two': 'second', 'three': 'third', 'five': 'fifth', 'eight': 'eighth', 'nine': 'ninth', 'twelve': 'twelfth'} -%} +{%- if last in IRR -%}{%- set last = IRR[last] -%}{%- elif last.endswith('y') -%}{%- set last = last[:-1] ~ 'ieth' -%}{%- else -%}{%- set last = last ~ 'th' -%}{%- endif -%} +{{- (ws[:-1] + [last]) | join(' ') -}} +{%- endmacro -%} + +{%- macro yearw(n) -%} +{%- set hi = n // 100 -%}{%- set lo = n % 100 -%} +{%- if hi == 0 or (hi % 10 == 0 and lo < 10) or hi >= 100 -%}{{ card(n) }} +{%- else -%}{{ card(hi) }} {{ 'hundred' if lo == 0 else ('oh ' ~ card(lo) if lo < 10 else card(lo)) }} +{%- endif -%} +{%- endmacro -%} + +{%- macro say(words) -%} +{%- set ns = namespace(out=[]) -%} +{%- for w in words.split(' ') -%} + {%- if w -%}{%- set ns.out = ns.out + [lookup(w, '', -2 if w == 'point' else none, '')] -%}{%- endif -%} +{%- endfor -%} +{{- ns.out | join(' ') -}} +{%- endmacro -%} + +{%- macro digits(s) -%}{%- set ns = namespace(out=[]) -%}{%- for c in tts_norm_scan(s, '%d') -%}{%- if c.rule -%}{%- set ns.out = ns.out + [ONES[c.text | int]] -%}{%- endif -%}{%- endfor -%}{{ ns.out | join(' ') }}{%- endmacro -%} + +{#- num: digits with optional ,### groups and .decimals ; sfx: st/nd/rd/th/s/'s/ed/ing ; cur: currency symbol or '' -#} +{%- macro get_number(neg, num, sfx, cur) -%} +{%- set plain = num | replace(',', '') -%} +{%- set ns = namespace(w='') -%} +{%- if sfx in ['st', 'nd', 'rd', 'th'] and '.' not in plain -%}{%- set ns.w = ordw(plain | int) -%} +{%- elif not neg and not cur and full(num, '%d%d%d%d') -%}{%- set ns.w = yearw(plain | int) -%} +{%- elif cur and cur in CURRENCIES and full(plain, '%d*(?:%.%d%d?)?') -%} + {%- set parts = plain.split('.') -%} + {%- set units = CURRENCIES[cur] -%} + {%- set a = parts[0] | int if parts[0] else 0 -%} + {%- set b = parts[1] | int if parts | length > 1 and parts[1] else 0 -%} + {%- set out = namespace(l=[]) -%} + {%- for pair in [[a, units[0]], [b, units[1]]] -%} + {%- if pair[0] or (loop.first and not b) -%} + {%- set uw = lookup(pair[1], '', none, '') if pair[0] == 1 or pair[1] == 'pence' else stem_s(pair[1] ~ 's', '', none, '') -%} + {%- set out.l = out.l + [say(card(pair[0])) ~ ' ' ~ uw] -%} + {%- endif -%} + {%- endfor -%} + {%- set ns.w = '@' ~ (out.l | join(' ' ~ lookup('and', '', none, '') ~ ' ')) -%} +{%- elif '.' in plain -%} + {%- set ip, fp = plain.split('.') -%} + {%- set ns.w = ((card(ip | int) ~ ' ') if ip else '') ~ 'point ' ~ digits(fp) -%} +{%- else -%}{%- set ns.w = card(plain | int) -%} +{%- endif -%} +{%- set r = ns.w[1:] if ns.w.startswith('@') else say(ns.w | replace('-', ' ')) -%} +{%- if neg -%}{%- set r = lookup('minus', '', none, '') ~ ' ' ~ r -%}{%- endif -%} +{%- if sfx in ['s', "'s"] -%}{{ sfx_s(r) }} +{%- elif sfx in ['ed', "'d"] -%}{{ sfx_ed(r) }} +{%- elif sfx == 'ing' -%}{{ sfx_ing(r) }} +{%- else -%}{{ r }} +{%- endif -%} +{%- endmacro -%} + +{#- ---------- context ---------- -#} +{%- macro next_fv(ps, fv) -%} +{%- set ns = namespace(r=fv, done=false) -%} +{%- for s in tts_norm_scan(ps, [['v', '[' ~ V ~ ']'], ['c', '[' ~ C ~ ']'], ['p', '[' ~ NQP ~ ']']]) -%} + {%- if s.rule and not ns.done -%}{%- set ns.r = 'none' if s.rule == 'p' else s.rule -%}{%- set ns.done = true -%}{%- endif -%} +{%- endfor -%} +{{- ns.r -}} +{%- endmacro -%} + +{#- ---------- POS tagger: nltk averaged perceptron, weights in the pos_* dictionaries ---------- -#} +{%- macro pos_norm(w) -%} +{%- if '-' in w and w[:1] != '-' -%}!HYPHEN +{%- elif full(w, '%d%d%d%d') -%}!YEAR +{%- elif full(w[:1], '%d') -%}!DIGITS +{%- else -%}{{ w | lower }} +{%- endif -%} +{%- endmacro -%} + +{%- set POS_CLASSES = tts_dict('pos_classes', '') -%} + +{#- features -> best label; nltk: max(classes, key=(score, label)); classes are sorted, so >= keeps the larger label on ties -#} +{%- macro pos_predict(feats) -%} +{%- set ns = namespace(ws=[], best='', best_s=none) -%} +{%- for f in feats -%} + {%- set w = tts_dict('pos_weights', f) -%} + {%- if w -%}{%- set ns.ws = ns.ws + [w] -%}{%- endif -%} +{%- endfor -%} +{%- for c in POS_CLASSES -%} + {%- set s = ns.ws | map(attribute='c' ~ loop.index0, default=0.0) | sum -%} + {%- if ns.best_s is none or s >= ns.best_s -%}{%- set ns.best = c -%}{%- set ns.best_s = s -%}{%- endif -%} +{%- endfor -%} +{{- ns.best -}} +{%- endmacro -%} + +{#- tokens: list of strings (PTB-style) -> space-separated tags -#} +{%- macro pos_tag(tokens) -%} +{%- set cn = namespace(l=['-START-', '-START2-']) -%} +{%- for t in tokens -%}{%- set cn.l = cn.l + [pos_norm(t)] -%}{%- endfor -%} +{%- set c = cn.l + ['-END-', '-END2-'] -%} +{%- set ns = namespace(prev='-START-', prev2='-START2-', out=[]) -%} +{%- for w in tokens -%} + {%- set i = loop.index0 + 2 -%} + {%- set tag = tts_dict('pos_tagdict', w) -%} + {%- if not tag -%} + {%- set tag = pos_predict([ + 'bias', + 'i suffix ' ~ w[-3:], + 'i pref1 ' ~ w[:1], + 'i-1 tag ' ~ ns.prev, + 'i-2 tag ' ~ ns.prev2, + 'i tag+i-2 tag ' ~ ns.prev ~ ' ' ~ ns.prev2, + 'i word ' ~ c[i], + 'i-1 tag+i word ' ~ ns.prev ~ ' ' ~ c[i], + 'i-1 word ' ~ c[i - 1], + 'i-1 suffix ' ~ c[i - 1][-3:], + 'i-2 word ' ~ c[i - 2], + 'i+1 word ' ~ c[i + 1], + 'i+1 suffix ' ~ c[i + 1][-3:], + 'i+2 word ' ~ c[i + 2], + ]) -%} + {%- endif -%} + {%- set ns.out = ns.out + [tag] -%} + {%- set ns.prev2 = ns.prev -%}{%- set ns.prev = tag -%} +{%- endfor -%} +{{- ns.out | join(' ') -}} +{%- endmacro -%} + +{%- macro weight(ps) -%}{{ (ps | length) + (tts_norm_scan(ps, '[AIOQWYʤʧ]') | selectattr('rule') | list | length) }}{%- endmacro -%} + +{#- hyphenated compound: per-part lookup (right to left), joined without space, misaki's stress demotion -#} +{%- macro compound(text, tag, fv) -%} +{%- if has(text) -%}{{ lex_word(text, tag, fv) }}{%- else -%} +{%- set ns = namespace(fv=fv, parts=[], bad=false) -%} +{%- for s in tts_norm_scan(text, '%a+(?:[\'’]%a+)*') | reverse -%} + {%- if s.rule -%} + {%- set p = lex_word(s.text, tag, ns.fv) -%} + {%- if not p -%}{%- set ns.bad = true -%}{%- endif -%} + {%- set ns.parts = [{'t': s.text, 'p': p}] + ns.parts -%} + {%- set ns.fv = next_fv(p, ns.fv) -%} + {%- endif -%} +{%- endfor -%} +{%- set n = ns.parts | length -%} +{%- set d = namespace(demote=[]) -%} +{#- a part is missing from the lexicon: read the compound part by part -#} +{%- if ns.bad -%} + {%- for x in ns.parts -%}{{ x.p or fallback(x.t) }}{%- endfor -%} +{%- else -%} +{%- if not ns.bad -%} + {%- set k = namespace(keys=[]) -%} + {%- for x in ns.parts -%} + {%- set pr = 1 if 'ˈ' in x.p else 0 -%} + {#- misaki: sorted((has_primary, weight, index)) -#} + {%- set k.keys = k.keys + [{'pr': pr, 'key': pr * 1000000 + (weight(x.p) | int) * 100 + loop.index0, 'i': loop.index0}] -%} + {%- endfor -%} + {%- set prim = k.keys | sum(attribute='pr') -%} + {%- if n == 2 and ns.parts[0].t | length == 1 -%}{%- set d.demote = [1] -%} + {%- elif n >= 2 and prim > (n + 1) // 2 -%}{%- set d.demote = (k.keys | sort(attribute='key'))[:n // 2] | map(attribute='i') | list -%} + {%- endif -%} +{%- endif -%} +{%- for x in ns.parts -%}{{ apply_stress(x.p, -0.5) if loop.index0 in d.demote else x.p }}{%- endfor -%} +{%- endif -%} +{%- endif -%} +{%- endmacro -%} + +{#- unknown word in mixed case, read part by part: "OneAPI" -> One + API, '' if the word has one part -#} +{%- macro camel(text, tag, fv) -%} +{%- set parts = tts_norm_scan(text, '(?:%u(?!%l))+|%u?%l+') | selectattr('rule') | map(attribute='text') | list -%} +{%- if parts | length > 1 and parts | join('') == text -%} + {%- for p in parts -%} + {{- get_nnp(p) if p | length == 1 else (lex_word(p, tag, fv) or fallback(p)) -}} + {%- endfor -%} +{%- endif -%} +{%- endmacro -%} + +{#- nxt: next token text ('' at end) -#} +{%- macro word_tok(text, tag, fv, fto, nxt) -%} +{%- set w = text | replace('‘', "'") | replace('’', "'") -%} +{%- set ps = lex_word(w, tag, fv, fto) -%} +{%- if ps -%}{{ ps }} +{#- "etc" + "." -> dictionary entry "etc." (the dot is kept as punctuation) -#} +{%- elif nxt[:1] == '.' and has(w ~ '.') -%}{{ lookup(w ~ '.', tag, none, fv) }} +{#- trailing closing quote, e.g. after a plural possessive -#} +{%- elif w.endswith("'") and lex_word(w[:-1], tag, fv) -%}{{ lex_word(w[:-1], tag, fv) }}” +{#- unknown name + 's -#} +{%- elif w.endswith("'s") -%}{{ (lex_word(w[:-2], tag, fv) or fallback(w[:-2])) ~ gold("'s") }} +{%- endif -%} +{%- endmacro -%} + +{#- ---------- main ---------- -#} +{#- only english has a text to phonemes step: the prompt of the other voices is already phonemes -#} +{%- set lc = lang[:2] if lang else ('en' if not (voice is defined and voice) or voice[:1] in 'ab' else 'other') -%} +{%- if lc != 'en' -%} + {{- messages[-1].content | trim -}} +{%- else -%} +{%- set text = messages[-1].content | trim -%} +{%- set t = namespace(s=tts_norm_sub(text, '%s+', ' ')) -%} +{%- for f in FOLD -%}{%- set t.s = tts_norm_sub(t.s, f[0], f[1]) -%}{%- endfor -%} +{%- set text = t.s -%} +{%- set segs = tts_norm_scan(text, [ + ['abbr', '(? 3 -%}{%- set tk.toks = tk.toks + [w[:-3], w[-3:]] -%} + {%- elif m | length == 1 and m[0].rule -%}{%- set tk.toks = tk.toks + [m[0].groups[0], m[0].groups[1]] -%} + {%- else -%}{%- set tk.toks = tk.toks + [w] -%}{%- endif -%} + {%- elif s.rule == 'wdot' -%} + {%- set tk.toks = tk.toks + ([s.text] if has(s.text) else [s.text[:-1], '.']) -%} + {%- elif s.rule == 'money' -%}{%- set tk.toks = tk.toks + [s.groups[0], s.groups[1]] -%} + {%- elif s.rule == 'qopen' -%}{%- set tk.toks = tk.toks + ['``'] -%} + {%- elif s.rule == 'quote' -%}{%- set tk.toks = tk.toks + ["''"] -%} + {%- elif s.rule == 'dash' -%}{%- set tk.toks = tk.toks + ['--'] -%} + {%- elif s.rule == 'punct' -%}{%- set tk.toks = tk.toks + [PTB_PUNCT.get(s.text, s.text)] -%} + {%- else -%}{%- set tk.toks = tk.toks + [s.text] -%} + {%- endif -%} + {%- set tk.first = tk.first + [n0 if (tk.toks | length) > n0 else -1] -%} +{%- endfor -%} +{%- set tags = pos_tag(tk.toks).split(' ') if tk.toks else [] -%} + +{#- 2. G2P right to left (misaki's future-vowel context) -#} +{%- set GLUE = ['abbr', 'money', 'num', 'dig', 'cmpd', 'wdot', 'word', 'sym'] -%} +{%- set ns = namespace(fv='none', fto=false, out=[], next='', next_text='') -%} +{%- for s in segs | reverse -%} + {%- set i = nseg - 1 - loop.index0 -%} + {%- set tag = tags[tk.first[i]] if tk.first[i] >= 0 else '' -%} + {%- if s.rule == 'wdot' and has(s.text) -%} + {%- set ps = lex_word(s.text, tag, ns.fv) -%} + {%- elif s.rule == 'wdot' -%} + {%- set w = lex_word(s.text[:-1], tag, 'none') or fallback(s.text[:-1]) -%} + {%- set ps = w ~ '.' -%} + {%- elif s.rule is none -%} + {%- set ps = tts_norm_sub(s.text, '%S', '') -%} + {%- elif s.rule == 'abbr' -%} + {%- set ps = get_nnp(s.text) -%} + {%- elif s.rule == 'money' -%} + {%- set ps = get_number('', s.groups[1], '', s.groups[0]) -%} + {%- elif s.rule == 'num' -%} + {%- set ps = get_number(s.groups[0], s.groups[1], s.groups[2], '') -%} + {%- elif s.rule == 'dig' -%} + {#- digits glued to letters, e.g. "221B" -#} + {%- set ps = get_number('', s.text, '', '') -%} + {%- elif s.rule == 'cmpd' -%} + {%- set ps = compound(s.text, tag, ns.fv) -%} + {%- elif s.rule == 'word' -%} + {%- set ps = word_tok(s.text, tag, ns.fv, ns.fto, ns.next_text) or camel(s.text, tag, ns.fv) or fallback(s.text) -%} + {%- elif s.rule == 'sym' -%} + {%- set ps = lex_word(s.text, tag, ns.fv) or unk -%} + {%- elif s.rule == 'dash' -%} + {%- set ps = '—' -%} + {%- elif s.rule == 'qopen' -%} + {%- set ps = '“' -%} + {%- elif s.rule == 'quote' -%} + {%- set ps = '”' -%} + {%- else -%} + {%- set ps = s.text -%} + {%- endif -%} + {%- if s.rule is not none -%} + {%- set ns.fv = next_fv(ps, ns.fv) -%} + {%- set ns.fto = s.text in ['to', 'To'] or (s.text == 'TO' and tag in ['TO', 'IN']) -%} + {%- endif -%} + {#- misaki puts a space between glued tokens of mixed kinds, e.g. "20%" -#} + {%- if s.rule in GLUE and ns.next in GLUE and ps -%}{%- set ps = ps ~ ' ' -%}{%- endif -%} + {%- set ns.next = s.rule if s.rule is not none else '' -%} + {%- if s.rule is not none -%}{%- set ns.next_text = s.text -%}{%- endif -%} + {%- set ns.out = [ps] + ns.out -%} +{%- endfor -%} + +{{- (ns.out | join('') | replace('ɾ', 'T') | replace('ʔ', 't')) | trim -}} +{%- endif -%} diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 1869c1203..530fc8e12 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -176,6 +176,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_QWEN3TTS, "qwen3tts" }, { LLM_ARCH_POCKETTTS, "pockettts" }, { LLM_ARCH_ZONOS3, "zonos3" }, + { LLM_ARCH_KOKORO, "kokoro" }, { LLM_ARCH_K2_HORIZON, "k2-horizon" }, { LLM_ARCH_BEAM, "beam" }, { LLM_ARCH_KOLIBRI1, "kolibri1" }, @@ -629,6 +630,7 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_POS_EMBD, "position_embd" }, { LLM_TENSOR_FFN_ACT, "blk.%d.ffn.act" }, { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" }, + { LLM_TENSOR_TOKEN_EMBD_PROJ, "token_embd_proj" }, { LLM_TENSOR_TOKEN_TYPES, "token_types" }, { LLM_TENSOR_CLS, "cls" }, { LLM_TENSOR_CLS_OUT, "cls.output" }, @@ -929,6 +931,7 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_TOKEN_TYPES, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, {LLM_TENSOR_HRM_Z_L_INIT, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, {LLM_TENSOR_TOKEN_EMBD_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, // do the norms on the first layer (not the input layer) + {LLM_TENSOR_TOKEN_EMBD_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_OUTPUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, {LLM_TENSOR_CLS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, {LLM_TENSOR_CLS_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, diff --git a/src/llama-arch.h b/src/llama-arch.h index d015b5d5a..05a00c3ac 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -177,6 +177,7 @@ enum llm_arch { LLM_ARCH_QWEN3TTS, LLM_ARCH_POCKETTTS, LLM_ARCH_ZONOS3, + LLM_ARCH_KOKORO, LLM_ARCH_MINIMAX_01, LLM_ARCH_HRM_TEXT, LLM_ARCH_K2_HORIZON, @@ -531,6 +532,7 @@ enum llm_kv { enum llm_tensor { LLM_TENSOR_TOKEN_EMBD, LLM_TENSOR_TOKEN_EMBD_NORM, + LLM_TENSOR_TOKEN_EMBD_PROJ, LLM_TENSOR_TOKEN_TYPES, LLM_TENSOR_POS_EMBD, LLM_TENSOR_DENSE_2_OUT, diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 15ec05408..9adb96364 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -128,6 +128,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_pockettts(params); case LLM_ARCH_ZONOS3: return new llama_model_zonos3(params); + case LLM_ARCH_KOKORO: + return new llama_model_kokoro(params); case LLM_ARCH_PHI2: return new llama_model_phi2(params); case LLM_ARCH_PHI3: @@ -2987,6 +2989,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, case LLM_ARCH_LLADA_MOE: case LLM_ARCH_RND1: case LLM_ARCH_CLEF: + case LLM_ARCH_KOKORO: { res = nullptr; } break; @@ -3719,6 +3722,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_GLM5_NEXT: case LLM_ARCH_STARCODER: case LLM_ARCH_INKLING2: + case LLM_ARCH_KOKORO: return LLAMA_ROPE_TYPE_NONE; // use what we call a normal RoPE, operating on pairs of consecutive head values diff --git a/src/models/kokoro.cpp b/src/models/kokoro.cpp new file mode 100644 index 000000000..c236134af --- /dev/null +++ b/src/models/kokoro.cpp @@ -0,0 +1,145 @@ +#include "models.h" + +// backbone of the Kokoro TTS pipeline: the PL-BERT phoneme encoder (an ALBERT model) +// - token and position embeddings have a small width, a projection brings them to n_embd +// - one layer is stored and applied n_layer times +// it has no lm_head, the rest of the model is inside the mmproj + +void llama_model_kokoro::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); + + type = LLM_TYPE_UNKNOWN; +} + +void llama_model_kokoro::load_arch_tensors(llama_model_loader & ml) { + LLAMA_LOAD_LOCALS; + + // the width of the token and position embeddings comes from the weights + const ggml_tensor * pos_meta = ml.get_tensor_meta(tn(LLM_TENSOR_POS_EMBD, "weight").str().c_str()); + const int64_t n_embd_tok = pos_meta ? pos_meta->ne[0] : n_embd; + + // the converter pads the rows with zeros from n_embd_tok to n_embd, token and embedding inputs must have the same width + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + // the token type embedding is folded into the positions by the converter + pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd_tok, n_ctx_train}, 0); + + tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd_tok}, 0); + tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd_tok}, 0); + + tok_embd_proj = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_PROJ, "weight", 0), {n_embd_tok, n_embd}, 0); + tok_embd_proj_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_PROJ, "bias", 0), {n_embd}, 0); + + // no output head, the logits are unused; reuse the embedding table so a sampler can still run + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + + // the layers share their weights, only the first one is stored + { + auto & layer = layers[0]; + const int i = 0; + + create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0); + + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0); + layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0); + + layer.attn_out_norm = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}, 0); + layer.attn_out_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}, 0); + + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, 0); + + layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, 0); + layer.layer_out_norm_b = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd}, 0); + } +} + +std::unique_ptr llama_model_kokoro::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(params); +} + +llama_model_kokoro::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { + const auto & model = static_cast(this->model); + + const int64_t n_embd_head = hparams.n_embd_head_v(); + const int64_t n_embd_tok = model.pos_embd->ne[0]; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + + ggml_tensor * cur; + ggml_tensor * inpL; + + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * embd_pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos); + ggml_build_forward_order(gf, embd_pos); + + // token + position (+ type) embeddings, in the small width + inpL = build_inp_embd(model.tok_embd); + inpL = ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd_tok, n_tokens, inpL->nb[1], 0)); + inpL = ggml_add(ctx0, inpL, embd_pos); + cb(inpL, "inp_embd", -1); + + inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, 0); + cb(inpL, "inp_norm", 0); + + inpL = ggml_add(ctx0, ggml_mul_mat(ctx0, model.tok_embd_proj, inpL), model.tok_embd_proj_b); + cb(inpL, "inp_proj", 0); + + auto * inp_attn = build_attn_inp_no_cache(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + const auto & layer = model.layers[0]; + + for (int il = 0; il < n_layer; ++il) { + cur = inpL; + + { + auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, + n_embd_head, n_head, n_head_kv, il); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + cur = build_attn(inp_attn, + layer.wo, layer.wo_b, layer.wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il); + cb(cur, "kqv_out", il); + } + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + cur = ggml_add(ctx0, cur, inpL); + cur = build_norm(cur, layer.attn_out_norm, layer.attn_out_norm_b, LLM_NORM, il); + + ggml_tensor * ffn_inp = cur; + cb(ffn_inp, "ffn_inp", il); + + cur = build_ffn(cur, + layer.ffn_up, layer.ffn_up_b, NULL, + NULL, NULL, NULL, + layer.ffn_down, layer.ffn_down_b, NULL, NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cur = build_norm(cur, layer.layer_out_norm, layer.layer_out_norm_b, LLM_NORM, il); + + inpL = cur; + } + + cur = inpL; + + cb(cur, "result_embd", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; +} diff --git a/src/models/models.h b/src/models/models.h index 932887c91..ccbd1be62 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -769,6 +769,22 @@ struct llama_model_pockettts : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; +struct llama_model_kokoro : public llama_model_base { + llama_model_kokoro(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + // token + position embeddings are narrower than n_embd, then projected up + ggml_tensor * tok_embd_proj = nullptr; + ggml_tensor * tok_embd_proj_b = nullptr; + + struct graph : public llm_graph_context { + graph(const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + struct llama_model_zonos3 : public llama_model_base { llama_model_zonos3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 36e48a0f2..0b7e8e4a9 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -399,7 +399,7 @@ endif() # libmtmd llama_build_and_test(test-mtmd-c-api.c LABEL behavior LINK mtmd) -llama_build_and_test(test-mtmd-impl.cpp LABEL behavior LINK mtmd) +llama_build_and_test(test-mtmd-impl.cpp LABEL behavior LINK mtmd llama-common) # GGUF model data fetcher library for tests that need real model metadata # Only compile when cpp-httplib has SSL support (CPPHTTPLIB_OPENSSL_SUPPORT) diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 8b3d4bb86..92123211a 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -1184,6 +1184,9 @@ static bool arch_supported(const llm_arch arch) { arch == LLM_ARCH_NEO_BERT || arch == LLM_ARCH_JINA_BERT_V2 || arch == LLM_ARCH_JINA_BERT_V3 || arch == LLM_ARCH_EUROBERT) { return false; // TODO vocab } + if (arch == LLM_ARCH_KOKORO) { + return false; // TODO encoder without memory, a sequence must fit in one ubatch + } if (arch == LLM_ARCH_PLM) { return false; // TODO tensor shapes } diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp index 5dd3d9384..e24365e59 100644 --- a/tests/test-mtmd-impl.cpp +++ b/tests/test-mtmd-impl.cpp @@ -7,12 +7,22 @@ #include "mtmd-helper.h" #include "mtmd-image.h" #include "mtmd-internal.h" +#include "models/models.h" + +#include "chat-tts-norm.h" +#include "jinja/lexer.h" +#include "jinja/parser.h" +#include "jinja/runtime.h" #include +#include #include #include #include +#include #include +#include +#include #include #include #include @@ -881,6 +891,177 @@ MAKE_TEST(test_helper_resampler) { t.assert_true("invalid rate", mtmd_helper_resampler_init(0, rate_out) == nullptr); } +// +// kokoro +// + +// a dictionary for tts_dict(): entries in any order, sorted here +static common_tts_dict kokoro_dict(std::vector> entries) { + std::sort(entries.begin(), entries.end()); + common_tts_dict dict; + for (auto & e : entries) { + dict.keys.push_back(e.first); + dict.values.push_back(e.second); + } + return dict; +} + +// the "tts" template with small dictionaries: a few lexicon words, fixed tags, and a model for unknown words over the letters z a b i n g +MAKE_TEST(test_kokoro_tts_template) { + std::string path = __FILE__; + const size_t pos = path.find_last_of("/\\"); + path = (pos == std::string::npos ? std::string(".") : path.substr(0, pos)) + "/../models/templates/hexgrad-Kokoro-82M-tts.jinja"; + std::ifstream fs(path, std::ios_base::binary); + if (!t.assert_true("template file is readable: " + path, fs.is_open())) { + return; + } + const std::string tmpl((std::istreambuf_iterator(fs)), std::istreambuf_iterator()); + + const auto dicts = std::make_shared(common_tts_dicts{ + {"gold", kokoro_dict({ + {"hello", "\"h\xc9\x99l\xcb\x88O\""}, + {"world", "\"w\xcb\x88\xc9\x9c\xc9\xb9ld\""}, + {"I", "\"\xcb\x88I\""}, + {"read", "{\"ADJ\":\"\xc9\xb9\xcb\x88\xc9\x9b" "d\",\"DEFAULT\":\"\xc9\xb9\xcb\x88id\",\"VBD\":\"\xc9\xb9\xcb\x88\xc9\x9b" "d\",\"VBN\":\"\xc9\xb9\xcb\x88\xc9\x9b" "d\",\"VBP\":\"\xc9\xb9\xcb\x88\xc9\x9b" "d\"}"}, + {"it", "\"\xc9\xaat\""}, + {"three", "\"\xce\xb8\xc9\xb9\xcb\x88i\""}, + {"cat", "\"k\xcb\x88\xc3\xa6t\""}, + {"walk", "\"w\xcb\x88\xc9\x94k\""}, + {"the", "\"\xc3\xb0i\""}, + {"Z", "\"z\xcb\x88i\""}, + {"'s", "\"s\""}, + {"a", "\"A\""}, + {"and", "\"\xc3\xa6nd\""}, + {"is", "\"\xc9\xaaz\""}, + })}, + {"silver", kokoro_dict({ + })}, + {"gold_gb", kokoro_dict({ + {"hello", "\"h\xc9\x99l\xcb\x88Q\""}, + {"world", "\"w\xcb\x88\xc9\x9c\xcb\x90ld\""}, + {"walk", "\"w\xcb\x88\xc9\x94\xcb\x90k\""}, + })}, + {"silver_gb", kokoro_dict({ + })}, + {"pos_classes", kokoro_dict({ + {"", "[\"NN\",\"NNP\",\"PRP\",\"VBD\"]"}, + })}, + {"pos_tagdict", kokoro_dict({ + {"I", "\"PRP\""}, + {"read", "\"VBD\""}, + {"it", "\"PRP\""}, + {".", "\".\""}, + {",", "\",\""}, + })}, + {"pos_weights", kokoro_dict({ + {"bias", "{\"c0\":1.0}"}, + })}, + {"lts", kokoro_dict({ + {"z", "{\"z\":0.1}"}, + {"b", "{\"b\":0.1}"}, + {"i", "{\"\xc9\xaa\":0.1}"}, + {"n", "{\"\xc5\x8b\":0.1}"}, + {"g", "{\"\":0.1}"}, + {"a", "{\"\xcb\x88\xc3\xa6\":0.1,\"\xc3\xa6\":0.06}"}, + {"z:z a", "{\"\xcb\x88\xc3\xa6\":0.2,\"\xc3\xa6\":0.01}"}, + {"$", "{\"\":0.2}"}, + })}, + {"lts_gb", kokoro_dict({ + {"z", "{\"z\":0.1}"}, + {"b", "{\"b\":0.1}"}, + {"i", "{\"\xc9\xaa\":0.1}"}, + {"n", "{\"\xc5\x8b\":0.1}"}, + {"g", "{\"\":0.1}"}, + {"a", "{\"\xcb\x88\xc9\x91\xcb\x90\":0.1,\"\xc9\x91\xcb\x90\":0.06}"}, + {"z:z a", "{\"\xcb\x88\xc9\x91\xcb\x90\":0.2,\"\xc9\x91\xcb\x90\":0.01}"}, + {"$", "{\"\":0.2}"}, + })}, + }); + + // voice, text, expected phonemes + const std::vector> cases = { + // words of the lexicon + { "af_heart", "Hello world.", "h\xc9\x99l\xcb\x88O w\xcb\x88\xc9\x9c\xc9\xb9ld." }, + // the pronunciation depends on the part of speech + { "af_heart", "I read it.", "\xcb\x8cI \xc9\xb9\xcb\x88\xc9\x9b" "d \xc9\xaat." }, + // a number, and the -s suffix rule + { "af_heart", "3 cats", "\xce\xb8\xc9\xb9\xcb\x88i k\xcb\x88\xc3\xa6ts" }, + // the -ed and -ing suffix rules + { "af_heart", "walked, walking", "w\xcb\x88\xc9\x94kt, w\xcb\x88\xc9\x94k\xc9\xaa\xc5\x8b" }, + // not in the lexicon: predicted from the spelling + { "af_heart", "zab", "z\xcb\x88\xc3\xa6" "b" }, + // the free search gives two primary stresses, the stress pass keeps one + { "af_heart", "zaba", "z\xcb\x88\xc3\xa6" "b\xc3\xa6" }, + // a doubled consonant says one sound + { "af_heart", "zabbing", "z\xcb\x88\xc3\xa6" "b\xc9\xaa\xc5\x8b" }, + // an unknown word with 's + { "af_heart", "The zab's cat", "\xc3\xb0\xc9\x99 z\xcb\x88\xc3\xa6" "bs k\xcb\x88\xc3\xa6t" }, + // no vowel: spelled out + { "af_heart", "zz", "z\xcb\x8ciz\xcb\x88i" }, + // an accented letter is read as its base letter + { "af_heart", "H\xc3\xa9ll\xc3\xb3 world.", "h\xc9\x99l\xcb\x88O w\xcb\x88\xc9\x9c\xc9\xb9ld." }, + // digits glued to letters are read as a number + { "af_heart", "3Z cats", "\xce\xb8\xc9\xb9\xcb\x88i z\xcb\x88i k\xcb\x88\xc3\xa6ts" }, + // an unknown word in mixed case is read part by part + { "af_heart", "catWalk", "k\xcb\x88\xc3\xa6tw\xcb\x88\xc9\x94k" }, + // a british voice uses the british lexicon + { "bf_emma", "Hello world.", "h\xc9\x99l\xcb\x88Q w\xcb\x88\xc9\x9c\xcb\x90ld." }, + // and the british model for unknown words + { "bf_emma", "zaba", "z\xcb\x88\xc9\x91\xcb\x90" "b\xc9\x91\xcb\x90" }, + // a voice of another language takes phonemes, they pass through + { "ef_dora", "\xcb\x88ola, m\xcb\x88undo.", "\xcb\x88ola, m\xcb\x88undo." }, + }; + + jinja::lexer lexer; + const jinja::program ast = jinja::parse_from_tokens(lexer.tokenize(tmpl)); + for (const auto & c : cases) { + std::string out; + try { + jinja::context ctx(tmpl); + common_chat_tts_norm_add_builtins(ctx, dicts); + auto msg = jinja::mk_val(); + msg->insert("role", jinja::mk_val("user")); + msg->insert("content", jinja::mk_val(c[1])); + auto msgs = jinja::mk_val(); + msgs->push_back(msg); + ctx.set_val("messages", msgs); + ctx.set_val("language", jinja::mk_val("")); + ctx.set_val("voice", jinja::mk_val(c[0])); + jinja::runtime runtime(ctx); + out = jinja::runtime::gather_string_parts(runtime.execute(ast))->as_string().str(); + } catch (const std::exception & e) { + out = std::string("exception: ") + e.what(); + } + if (!t.assert_true(c[0] + ": " + c[1], out == c[2])) { + t.log("Expected: " + c[2]); + t.log("Actual : " + out); + } + } +} + +// every frame is the constant 0.25 (only the DC bin, of magnitude n_fft * 0.25) +// the overlap-add of hann windows at a quarter hop sums to 2, their squares to 1.5: the output is 0.25 * 2 / 1.5 +MAKE_TEST(test_kokoro_istft) { + const int n_fft = 20, hop = 5, n_frames = 9; + std::vector spec((size_t) n_frames * (n_fft + 2), 0.0f); + for (int fr = 0; fr < n_frames; fr++) { + float * row = spec.data() + (size_t) fr * (n_fft + 2); + // the first half of a row is log magnitudes, the second half phases + for (int k = 0; k <= n_fft / 2; k++) { + row[k] = k == 0 ? logf(n_fft * 0.25f) : -100.0f; + } + } + std::vector audio; + mtmd_graph_kokoro::istft(spec, audio); + t.assert_equal("samples", (size_t) (n_frames - 1) * hop, audio.size()); + // the first and last hop are covered by fewer frames + float max_err = 0.0f; + for (size_t i = hop; i + hop < audio.size(); i++) { + max_err = std::max(max_err, std::fabs(audio[i] - 1.0f / 3.0f)); + } + t.assert_true("constant signal, max error " + std::to_string(max_err), max_err < 1e-4f); +} + // // main // diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 233ef1c62..de8c473dd 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -95,6 +95,7 @@ add_library(mtmd models/pockettts-spkenc.cpp models/pockettts-gen.cpp models/soprano.cpp + models/kokoro.cpp models/step3vl.cpp models/siglip.cpp models/whisper-enc.cpp diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md index 5a01e425d..55ad1c5b3 100644 --- a/tools/mtmd/README-dev.md +++ b/tools/mtmd/README-dev.md @@ -96,6 +96,28 @@ Some stages are a full LLM-sized transformer that attends to the keys/values of - Non-causal rows: `LLAMA_ATTENTION_TYPE_NON_CAUSAL`, all rows in one ubatch (`n_ubatch` >= rows), they see every cached position of the sequence. Remove them with `llama_memory_seq_rm()` after each pass - Size the context per call (`n_ctx`, `n_ubatch`) instead of keeping one for the whole session +### Text preprocessing (the `tts` chat template) + +Anything a model needs done to the input text before tokenization (prompt format, text normalization, text to phonemes) goes in `tokenizer.chat_template.tts`, as a jinja program. It receives the text as `messages[0].content`, plus `language` and `voice`. + +Examples: +- Prompt format: `conversion/lfm3.py` +- Text normalization: `models/templates/Zyphra-ZONOS3-Mini-tts.jinja` +- Text to phonemes (G2P): `models/templates/hexgrad-Kokoro-82M-tts.jinja` + +Functions available to the template (see `docs/tts-norm-patterns.md`): +- `tts_norm_scan()` / `tts_norm_sub()`: pattern matching for text normalization +- `tts_dict()`: lookup in a dictionary stored in the GGUF, such as a pronunciation lexicon + +Add test cases to `tests/test-jinja.cpp`, which also checks them against the python reference with `-py`. + +Rules for phoneme-based models: +- C++ only holds the pieces that every such model needs (the functions above). The rules of one model (lexicon handling, stress, suffixes, number spelling, ...) live in its jinja template +- The reason is binary size: phoneme logic compiled into C++ is shipped to every user, and most of them never run a TTS model. A template only costs the users of that model +- Do not add a model-specific or language-specific G2P step in C++, and do not add a dependency for it (e.g. espeak-ng) + +G2P in jinja has limits, mostly for languages other than English (no lexicon, or rules that need a real morphological analyzer). For now the escape hatch is on the application side: the user runs the text to phonemes step in their own code and sends the phonemes with `raw_prompt` (see the `/tts` endpoint of `llama-server`), which skips the template. Document it in `docs/multimodal/tts.md` when a model relies on it. + ### Checklist for porting new audio generation models to mtmd 1. Make sure to consult merged PRs about adding new TTS models, especially reviewer comments @@ -116,7 +138,7 @@ Some stages are a full LLM-sized transformer that attends to the keys/values of - In most cases, you can hard-code model configs in the model graph class, or in `mtmd_hparams` - If some values need to be exposed to the `mtmd_helper` layer, hard-code them in `mtmd_helper` and distinguish by pipeline and `mtmd_gen_audio_info::model_variant` if necessary - Do NOT add new GGUF metadata or new fields to `mtmd_gen_audio_info` unless you can prove that you absolutely need them - - If the model requires special preprocessing of the input text prompt (prompt format, text normalization), put it in `tokenizer.chat_template.tts` as a jinja program. It receives the text as `messages[0].content`, plus `language` and `voice`. See `conversion/lfm3.py` (prompt format) and `models/templates/Zyphra-ZONOS3-Mini-tts.jinja` (text normalization) as examples. For text normalization, use `tts_norm_scan()` / `tts_norm_sub()`, and `tts_dict()` for dictionaries stored in the GGUF such as a pronunciation lexicon (see `docs/tts-norm-patterns.md`), and add test cases to `tests/test-jinja.cpp`, which also checks them against the python reference with `-py` + - If the model requires special preprocessing of the input text prompt, see [Text preprocessing](#text-preprocessing-the-tts-chat-template) 4. Make sure most of the changes happen inside `mtmd-helper-gen.cpp`. A good PR looks like this: - 10-20% changes is to add new backbone (text) model and conversion - 60% changes inside `mtmd-helper-gen.cpp` diff --git a/tools/mtmd/models/kokoro.cpp b/tools/mtmd/models/kokoro.cpp new file mode 100644 index 000000000..1379701c3 --- /dev/null +++ b/tools/mtmd/models/kokoro.cpp @@ -0,0 +1,515 @@ +#include "models.h" + +#include +#include +#include +#include +#include + +// Kokoro: StyleTTS2 prosody predictor + iSTFTNet decoder. Tensors keep the names of the reference modules (a.gen..*) +// One utterance runs the 3 graphs of stage_type. The LSTMs, the alignment, the harmonic source and the ISTFT run on the host in between +// The graphs keep activations as [T, C], the layout of the conv ops + +static const int KOKORO_N_HIDDEN = 512; +static const int KOKORO_N_HARM = 9; // fundamental + 8 overtones +static const int KOKORO_UPS = 300; // samples per F0 step: upsample rates 10 * 6, times the ISTFT hop +static const int KOKORO_N_FFT = 20; +static const int KOKORO_HOP = 5; +static const float KOKORO_SR = 24000.0f; + +ggml_tensor * mtmd_graph_kokoro::tensor(const std::string & name) const { + const auto it = model.cbx_tensors.find(name); + if (it == model.cbx_tensors.end() || !it->second) { + throw std::runtime_error("kokoro: missing tensor " + name); + } + return it->second; +} + +bool mtmd_graph_kokoro::has_tensor(const std::string & name) const { + return model.cbx_tensors.find(name) != model.cbx_tensors.end(); +} + +// x [T, IC] -> [T_out, OC], with the bias if the module has one +ggml_tensor * mtmd_graph_kokoro::conv(const std::string & name, ggml_tensor * x, int stride, int pad, int dil) const { + ggml_tensor * y = ggml_conv_1d(ctx0, tensor(name + ".weight"), x, stride, pad, dil); + if (has_tensor(name + ".bias")) { + ggml_tensor * b = tensor(name + ".bias"); + y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0])); + } + return y; +} + +// AdaIN1d: instance norm over time, then scale and shift from the style +ggml_tensor * mtmd_graph_kokoro::adain(const std::string & name, ggml_tensor * x, ggml_tensor * style) const { + const int64_t n_ch = x->ne[1]; + ggml_tensor * h = ggml_add(ctx0, ggml_mul_mat(ctx0, tensor(name + ".fc.weight"), style), tensor(name + ".fc.bias")); + ggml_tensor * gamma = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, h, n_ch, 0), 1, n_ch); + ggml_tensor * beta = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, h, n_ch, n_ch * ggml_element_size(h)), 1, n_ch); + ggml_tensor * xn = ggml_norm(ctx0, x, 1e-5f); + return ggml_add(ctx0, ggml_add(ctx0, xn, ggml_mul(ctx0, xn, gamma)), beta); +} + +// x + sin^2(alpha x) / alpha, alpha per channel +ggml_tensor * mtmd_graph_kokoro::snake(const std::string & name, ggml_tensor * x) const { + ggml_tensor * alpha = tensor(name); + alpha = ggml_reshape_2d(ctx0, alpha, 1, alpha->ne[0]); + ggml_tensor * sx = ggml_sin(ctx0, ggml_mul(ctx0, x, alpha)); + sx = ggml_div(ctx0, ggml_mul(ctx0, sx, sx), alpha); + return ggml_add(ctx0, x, sx); +} + +// a [T, C] and b [T, C] -> [2T, C] with a on the even steps +ggml_tensor * mtmd_graph_kokoro::interleave(ggml_tensor * a, ggml_tensor * b) const { + const int64_t n_t = a->ne[0], n_ch = a->ne[1]; + ggml_tensor * ab = ggml_concat(ctx0, ggml_reshape_3d(ctx0, ggml_cont(ctx0, a), 1, n_t, n_ch), + ggml_reshape_3d(ctx0, ggml_cont(ctx0, b), 1, n_t, n_ch), 0); + return ggml_reshape_2d(ctx0, ggml_cont(ctx0, ab), 2 * n_t, n_ch); +} + +// AdainResBlk1d, upsample doubles the length +ggml_tensor * mtmd_graph_kokoro::adain_resblk(const std::string & name, ggml_tensor * x, ggml_tensor * style, bool upsample) const { + const int64_t n_t = x->ne[0], n_ch = x->ne[1]; + + ggml_tensor * r = adain(name + ".norm1", x, style); + r = ggml_leaky_relu(ctx0, r, 0.2f, false); + if (upsample) { + // depthwise ConvTranspose1d(kernel 3, stride 2, padding 1, output_padding 1), weight stored as [C, 3]: + // y[2i] = w1 x[i], y[2i+1] = w2 x[i] + w0 x[i+1] + ggml_tensor * w = tensor(name + ".pool.weight"); + ggml_tensor * b = tensor(name + ".pool.bias"); + auto tap = [&](int k) { + return ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, w, n_ch, (size_t) k * n_ch * ggml_element_size(w)), 1, n_ch); + }; + ggml_tensor * next = ggml_cont(ctx0, ggml_view_2d(ctx0, r, n_t - 1, n_ch, r->nb[1], ggml_element_size(r))); + next = ggml_pad(ctx0, next, 1, 0, 0, 0); + ggml_tensor * even = ggml_mul(ctx0, r, tap(1)); + ggml_tensor * odd = ggml_add(ctx0, ggml_mul(ctx0, r, tap(2)), ggml_mul(ctx0, next, tap(0))); + r = ggml_add(ctx0, interleave(even, odd), ggml_reshape_2d(ctx0, b, 1, n_ch)); + } + r = conv(name + ".conv1", r, 1, 1, 1); + r = adain(name + ".norm2", r, style); + r = ggml_leaky_relu(ctx0, r, 0.2f, false); + r = conv(name + ".conv2", r, 1, 1, 1); + + ggml_tensor * sc = upsample ? interleave(x, x) : x; + if (has_tensor(name + ".conv1x1.weight")) { + sc = conv(name + ".conv1x1", sc, 1, 0, 1); + } + return ggml_scale(ctx0, ggml_add(ctx0, r, sc), (float) M_SQRT1_2); +} + +// AdaINResBlock1 of the generator: three units with dilations 1/3/5 on convs1 +ggml_tensor * mtmd_graph_kokoro::gen_resblock(const std::string & name, ggml_tensor * x, ggml_tensor * style) const { + static const int dil[3] = {1, 3, 5}; + for (int j = 0; j < 3; j++) { + const std::string js = std::to_string(j); + const int k = (int) tensor(name + ".convs1." + js + ".weight")->ne[0]; + ggml_tensor * xt = adain(name + ".adain1." + js, x, style); + xt = snake(name + ".alpha1." + js, xt); + xt = conv(name + ".convs1." + js, xt, 1, (k * dil[j] - dil[j]) / 2, dil[j]); + xt = adain(name + ".adain2." + js, xt, style); + xt = snake(name + ".alpha2." + js, xt); + xt = conv(name + ".convs2." + js, xt, 1, (k - 1) / 2, 1); + x = ggml_add(ctx0, xt, x); + } + return x; +} + +ggml_tensor * mtmd_graph_kokoro::input(const char * name, ggml_type type, int64_t ne0, int64_t ne1) const { + ggml_tensor * t = ggml_new_tensor_2d(ctx0, type, ne0, ne1); + ggml_set_name(t, name); + ggml_set_input(t); + return t; +} + +ggml_cgraph * mtmd_graph_kokoro::output(const char * name, ggml_tensor * t) const { + ggml_set_name(t, name); + ggml_set_output(t); + ggml_build_forward_expand(gf, t); + return gf; +} + +// tokens -> text encoder CNN output [C, n_tok] +ggml_cgraph * mtmd_graph_kokoro::build_text() { + const std::string p = "a.gen.text."; + ggml_tensor * tokens = input("inp_tokens", GGML_TYPE_I32, n_tok, 1); + tokens = ggml_reshape_1d(ctx0, tokens, n_tok); + + ggml_tensor * x = ggml_get_rows(ctx0, tensor(p + "embedding.weight"), tokens); // [C, T] + for (int i = 0; has_tensor(p + "cnn." + std::to_string(i) + ".0.weight"); i++) { + const std::string q = p + "cnn." + std::to_string(i); + const int k = (int) tensor(q + ".0.weight")->ne[0]; + x = conv(q + ".0", ggml_cont(ctx0, ggml_transpose(ctx0, x)), 1, (k - 1) / 2, 1); + // LayerNorm over the channels + x = ggml_norm(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, x)), 1e-5f); + x = ggml_add(ctx0, ggml_mul(ctx0, x, tensor(q + ".1.gamma")), tensor(q + ".1.beta")); + x = ggml_leaky_relu(ctx0, x, 0.2f, false); + } + return output("out_text", x); +} + +// shared LSTM output [C, n_frames] -> F0 and energy curves, 2 steps per frame +ggml_cgraph * mtmd_graph_kokoro::build_f0n() { + const std::string p = "a.gen.pred."; + ggml_tensor * inp = input("inp_x", GGML_TYPE_F32, KOKORO_N_HIDDEN, n_frames); + ggml_tensor * style = input("inp_style", GGML_TYPE_F32, n_style, 1); + style = ggml_reshape_1d(ctx0, style, n_style); + ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); + + for (const char * curve : {"F0", "N"}) { + ggml_tensor * cur = x; + for (int i = 0; i < 3; i++) { + const std::string q = p + curve + "." + std::to_string(i); + cur = adain_resblk(q, cur, style, has_tensor(q + ".pool.weight")); + } + cur = conv(p + curve + "_proj", cur, 1, 0, 1); + ggml_set_name(cur, curve[0] == 'F' ? "out_f0" : "out_n"); + ggml_set_output(cur); + ggml_build_forward_expand(gf, cur); + } + return gf; +} + +// aligned text features, F0, energy and the source STFT -> conv_post output [n_fft + 2, n_har] +ggml_cgraph * mtmd_graph_kokoro::build_dec() { + const std::string p = "a.gen.dec."; + ggml_tensor * asr = input("inp_asr", GGML_TYPE_F32, KOKORO_N_HIDDEN, n_frames); + ggml_tensor * f0 = input("inp_f0", GGML_TYPE_F32, 2 * n_frames, 1); + ggml_tensor * en = input("inp_n", GGML_TYPE_F32, 2 * n_frames, 1); + ggml_tensor * har = input("inp_har", GGML_TYPE_F32, KOKORO_N_FFT + 2, n_har); + ggml_tensor * style = input("inp_style", GGML_TYPE_F32, n_style, 1); + style = ggml_reshape_1d(ctx0, style, n_style); + asr = ggml_cont(ctx0, ggml_transpose(ctx0, asr)); + har = ggml_cont(ctx0, ggml_transpose(ctx0, har)); + + f0 = conv(p + "F0_conv", f0, 2, 1, 1); + en = conv(p + "N_conv", en, 2, 1, 1); + + ggml_tensor * x = ggml_concat(ctx0, ggml_concat(ctx0, asr, f0, 1), en, 1); + x = adain_resblk(p + "encode", x, style, false); + + ggml_tensor * asr_res = conv(p + "asr_res.0", asr, 1, 0, 1); + for (int i = 0; has_tensor(p + "decode." + std::to_string(i) + ".conv1.weight"); i++) { + const std::string q = p + "decode." + std::to_string(i); + x = ggml_concat(ctx0, ggml_concat(ctx0, ggml_concat(ctx0, x, asr_res, 1), f0, 1), en, 1); + x = adain_resblk(q, x, style, has_tensor(q + ".pool.weight")); + } + + const std::string g = p + "generator."; + int n_ups = 0; + while (has_tensor(g + "ups." + std::to_string(n_ups) + ".weight")) { + n_ups++; + } + for (int i = 0; i < n_ups; i++) { + const std::string is = std::to_string(i); + ggml_tensor * up_w = tensor(g + "ups." + is + ".weight"); + ggml_tensor * up_b = tensor(g + "ups." + is + ".bias"); + // the transposed conv has kernel 2 * stride and torch padding stride / 2 + const int stride = (int) up_w->ne[0] / 2; + const int trim = stride / 2; + + x = ggml_leaky_relu(ctx0, x, 0.1f, false); + + // the source STFT is brought to the rate of this stage + ggml_tensor * src_w = tensor(g + "noise_convs." + is + ".weight"); + const int src_k = (int) src_w->ne[0]; + const int src_s = src_k > 1 ? src_k / 2 : 1; + ggml_tensor * src = conv(g + "noise_convs." + is, har, src_s, src_k > 1 ? (src_s + 1) / 2 : 0, 1); + src = gen_resblock(g + "noise_res." + is, src, style); + + x = ggml_conv_transpose_1d(ctx0, up_w, x, stride, 0, 1); + x = ggml_cont(ctx0, ggml_view_2d(ctx0, x, x->ne[0] - 2 * trim, x->ne[1], x->nb[1], (size_t) trim * ggml_element_size(x))); + x = ggml_add(ctx0, x, ggml_reshape_2d(ctx0, up_b, 1, up_b->ne[0])); + if (i == n_ups - 1) { + x = ggml_pad_reflect_1d(ctx0, x, 1, 0); + } + x = ggml_add(ctx0, x, src); + + ggml_tensor * acc = nullptr; + for (int j = 0; j < 3; j++) { + ggml_tensor * r = gen_resblock(g + "resblocks." + std::to_string(i * 3 + j), x, style); + acc = acc ? ggml_add(ctx0, acc, r) : r; + } + x = ggml_scale(ctx0, acc, 1.0f / 3.0f); + } + x = ggml_leaky_relu(ctx0, x, 0.01f, false); + x = conv(g + "conv_post", x, 1, 3, 1); + return output("out_spec", ggml_cont(ctx0, ggml_transpose(ctx0, x))); +} + +ggml_cgraph * mtmd_graph_kokoro::build() { + switch (stage) { + case STAGE_TEXT: return build_text(); + case STAGE_F0N: return build_f0n(); + case STAGE_DEC: return build_dec(); + } + GGML_ABORT("invalid kokoro stage"); +} + +// +// host side +// + +static std::vector kokoro_read(const mtmd_model & model, const std::string & name, size_t n_expect) { + std::vector v(n_expect); + if (model.cbx_read_tensor(name.c_str(), v.data(), v.size()) != n_expect) { + throw std::runtime_error("kokoro: unexpected tensor " + name); + } + return v; +} + +static inline float kokoro_sigmoid(float x) { + return 1.0f / (1.0f + expf(-x)); +} + +// y = W x, W row-major [n_out, n_in] +static void kokoro_matvec(const float * w, const float * x, int n_out, int n_in, float * y) { + for (int o = 0; o < n_out; o++) { + const float * row = w + (size_t) o * n_in; + float acc[8] = {0.0f}; + int i = 0; + for (; i + 8 <= n_in; i += 8) { + for (int k = 0; k < 8; k++) { + acc[k] += row[i + k] * x[i + k]; + } + } + float sum = 0.0f; + for (; i < n_in; i++) { + sum += row[i] * x[i]; + } + for (int k = 0; k < 8; k++) { + sum += acc[k]; + } + y[o] = sum; + } +} + +// bidirectional single-layer LSTM (torch layout, gates i f g o): x [n_t, n_in] -> y [n_t, 2 * n_h] +static void kokoro_lstm(const mtmd_model & model, const std::string & name, const std::vector & x, int n_t, int n_in, + std::vector & y) { + const int n_h = KOKORO_N_HIDDEN / 2; + y.assign((size_t) n_t * 2 * n_h, 0.0f); + std::vector gi(4 * n_h), gh(4 * n_h), h(n_h), c(n_h); + for (int dir = 0; dir < 2; dir++) { + const std::string sfx = dir == 0 ? "" : "_reverse"; + const auto w_ih = kokoro_read(model, name + ".weight_ih_l0" + sfx, (size_t) 4 * n_h * n_in); + const auto w_hh = kokoro_read(model, name + ".weight_hh_l0" + sfx, (size_t) 4 * n_h * n_h); + const auto bias = kokoro_read(model, name + ".bias_l0" + sfx, (size_t) 4 * n_h); + std::fill(h.begin(), h.end(), 0.0f); + std::fill(c.begin(), c.end(), 0.0f); + for (int s = 0; s < n_t; s++) { + const int t = dir == 0 ? s : n_t - 1 - s; + kokoro_matvec(w_ih.data(), x.data() + (size_t) t * n_in, 4 * n_h, n_in, gi.data()); + kokoro_matvec(w_hh.data(), h.data(), 4 * n_h, n_h, gh.data()); + float * out = y.data() + (size_t) t * 2 * n_h + (size_t) dir * n_h; + for (int j = 0; j < n_h; j++) { + const float ig = kokoro_sigmoid(gi[j] + gh[j] + bias[j]); + const float fg = kokoro_sigmoid(gi[n_h + j] + gh[n_h + j] + bias[n_h + j]); + const float gg = tanhf(gi[2 * n_h + j] + gh[2 * n_h + j] + bias[2 * n_h + j]); + const float og = kokoro_sigmoid(gi[3 * n_h + j] + gh[3 * n_h + j] + bias[3 * n_h + j]); + c[j] = fg * c[j] + ig * gg; + h[j] = og * tanhf(c[j]); + out[j] = h[j]; + } + } + } +} + +// rows of x repeated by their duration: [n_tok, n_ch] -> [n_frames, n_ch] +static std::vector kokoro_align(const std::vector & x, const std::vector & dur, int n_ch) { + std::vector y; + for (size_t t = 0; t < dur.size(); t++) { + for (int32_t r = 0; r < dur[t]; r++) { + y.insert(y.end(), x.begin() + t * n_ch, x.begin() + (t + 1) * n_ch); + } + } + return y; +} + +int mtmd_graph_kokoro::prosody(const mtmd_model & model, const float * bert, int n_embd, const float * text_cnn, int n_tok, + const float * style, float speed, std::vector & shared, std::vector & asr) { + const int n_h = KOKORO_N_HIDDEN; + const int n_cat = n_h + n_style; + + // bert_encoder, then each row is followed by the style + const auto enc_w = kokoro_read(model, "a.gen.bert_enc.weight", (size_t) n_h * n_embd); + const auto enc_b = kokoro_read(model, "a.gen.bert_enc.bias", n_h); + std::vector x((size_t) n_tok * n_cat); + for (int t = 0; t < n_tok; t++) { + float * row = x.data() + (size_t) t * n_cat; + kokoro_matvec(enc_w.data(), bert + (size_t) t * n_embd, n_h, n_embd, row); + for (int j = 0; j < n_h; j++) { + row[j] += enc_b[j]; + } + memcpy(row + n_h, style, n_style * sizeof(float)); + } + + // DurationEncoder: LSTM, AdaLayerNorm, style again + std::vector y; + std::vector gamma_beta(2 * n_h); + for (int il = 0; model.cbx_tensors.count(mtmd_format("a.gen.pred.text_encoder.lstms.%d.weight_ih_l0", 2 * il)); il++) { + kokoro_lstm(model, mtmd_format("a.gen.pred.text_encoder.lstms.%d", 2 * il), x, n_tok, n_cat, y); + const std::string norm = mtmd_format("a.gen.pred.text_encoder.lstms.%d.fc", 2 * il + 1); + const auto fc_w = kokoro_read(model, norm + ".weight", (size_t) 2 * n_h * n_style); + const auto fc_b = kokoro_read(model, norm + ".bias", (size_t) 2 * n_h); + kokoro_matvec(fc_w.data(), style, 2 * n_h, n_style, gamma_beta.data()); + for (int t = 0; t < n_tok; t++) { + const float * src = y.data() + (size_t) t * n_h; + float * dst = x.data() + (size_t) t * n_cat; + double mean = 0.0, var = 0.0; + for (int j = 0; j < n_h; j++) { + mean += src[j]; + } + mean /= n_h; + for (int j = 0; j < n_h; j++) { + var += (src[j] - mean) * (src[j] - mean); + } + const double scale = 1.0 / sqrt(var / n_h + 1e-5); + for (int j = 0; j < n_h; j++) { + const float v = (float) ((src[j] - mean) * scale); + dst[j] = (1.0f + gamma_beta[j] + fc_b[j]) * v + gamma_beta[n_h + j] + fc_b[n_h + j]; + } + } + } + + // durations: frames per token + kokoro_lstm(model, "a.gen.pred.lstm", x, n_tok, n_cat, y); + const ggml_tensor * proj = model.cbx_tensors.at("a.gen.pred.duration_proj.linear_layer.weight"); + const int n_bins = (int) proj->ne[1]; + const auto proj_w = kokoro_read(model, "a.gen.pred.duration_proj.linear_layer.weight", (size_t) n_bins * n_h); + const auto proj_b = kokoro_read(model, "a.gen.pred.duration_proj.linear_layer.bias", n_bins); + std::vector dur(n_tok); + std::vector logits(n_bins); + int n_frames = 0; + for (int t = 0; t < n_tok; t++) { + kokoro_matvec(proj_w.data(), y.data() + (size_t) t * n_h, n_bins, n_h, logits.data()); + float sum = 0.0f; + for (int j = 0; j < n_bins; j++) { + sum += kokoro_sigmoid(logits[j] + proj_b[j]); + } + // round half to even, like torch.round + dur[t] = std::max(1, (int) nearbyintf(sum / speed)); + n_frames += dur[t]; + } + + kokoro_lstm(model, "a.gen.pred.shared", kokoro_align(x, dur, n_cat), n_frames, n_cat, shared); + + // text encoder: its LSTM runs on the CNN output + kokoro_lstm(model, "a.gen.text.lstm", std::vector(text_cnn, text_cnn + (size_t) n_tok * n_h), n_tok, n_h, y); + asr = kokoro_align(y, dur, n_h); + return n_frames; +} + +// hn-nsf source: f0 [2 * n_frames] -> harmonic source, then its STFT as [n_har, 22] rows of magnitudes and phases +int mtmd_graph_kokoro::har_source(const mtmd_model & model, const std::vector & f0, std::mt19937 & rng, std::vector & har) { + const int n_f0 = (int) f0.size(); + const int64_t n_wav = (int64_t) n_f0 * KOKORO_UPS; + const auto lin_w = kokoro_read(model, "a.gen.dec.generator.m_source.l_linear.weight", KOKORO_N_HARM); + const auto lin_b = kokoro_read(model, "a.gen.dec.generator.m_source.l_linear.bias", 1); + + // SineGen: the phase adds up at the F0 rate, then it is interpolated to the sample rate + // the reference also adds a random start phase, but its own downsampling drops it + std::vector phase((size_t) n_f0 * KOKORO_N_HARM); + for (int h = 0; h < KOKORO_N_HARM; h++) { + float cum = 0.0f; + for (int i = 0; i < n_f0; i++) { + float rad = f0[i] * (float) (h + 1) / KOKORO_SR; + rad -= floorf(rad); + cum += rad; + phase[(size_t) i * KOKORO_N_HARM + h] = cum * 2.0f * (float) M_PI * KOKORO_UPS; + } + } + std::normal_distribution gauss(0.0f, 1.0f); + std::vector src((size_t) n_wav); + for (int64_t t = 0; t < n_wav; t++) { + const float pos = std::max(0.0f, ((float) t + 0.5f) / KOKORO_UPS - 0.5f); + const int i0 = std::min((int) pos, n_f0 - 1); + const int i1 = std::min(i0 + 1, n_f0 - 1); + const float frac = pos - (float) i0; + const float uv = f0[t / KOKORO_UPS] > 10.0f ? 1.0f : 0.0f; // voiced threshold + const float namp = uv * 0.003f + (1.0f - uv) * 0.1f / 3.0f; + float merged = lin_b[0]; + for (int h = 0; h < KOKORO_N_HARM; h++) { + const float ph = (1.0f - frac) * phase[(size_t) i0 * KOKORO_N_HARM + h] + frac * phase[(size_t) i1 * KOKORO_N_HARM + h]; + merged += lin_w[h] * (0.1f * sinf(ph) * uv + namp * gauss(rng)); + } + src[(size_t) t] = tanhf(merged); + } + + // STFT: periodic hann window, centered with reflect padding + const int n_bins = KOKORO_N_FFT / 2 + 1; + const int n_har = (int) (n_wav / KOKORO_HOP) + 1; + float win[KOKORO_N_FFT], tab_cos[KOKORO_N_FFT], tab_sin[KOKORO_N_FFT]; + for (int i = 0; i < KOKORO_N_FFT; i++) { + win[i] = 0.5f - 0.5f * cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT); + tab_cos[i] = cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT); + tab_sin[i] = sinf(2.0f * (float) M_PI * i / KOKORO_N_FFT); + } + har.resize((size_t) n_har * 2 * n_bins); + for (int fr = 0; fr < n_har; fr++) { + float frame[KOKORO_N_FFT]; + for (int i = 0; i < KOKORO_N_FFT; i++) { + int64_t idx = (int64_t) fr * KOKORO_HOP - KOKORO_N_FFT / 2 + i; + if (idx < 0) { + idx = -idx; + } + if (idx >= n_wav) { + idx = 2 * (n_wav - 1) - idx; + } + frame[i] = src[(size_t) idx] * win[i]; + } + float * out = har.data() + (size_t) fr * 2 * n_bins; + for (int k = 0; k < n_bins; k++) { + float re = 0.0f, im = 0.0f; + for (int i = 0; i < KOKORO_N_FFT; i++) { + const int a = (k * i) % KOKORO_N_FFT; + re += frame[i] * tab_cos[a]; + im -= frame[i] * tab_sin[a]; + } + out[k] = sqrtf(re * re + im * im); + out[n_bins + k] = atan2f(im, re); + } + } + return n_har; +} + +// conv_post rows [n_frames, 22] -> waveform: magnitude exp(x), phase sin(x), hann overlap-add +void mtmd_graph_kokoro::istft(const std::vector & spec, std::vector & audio) { + const int n_bins = KOKORO_N_FFT / 2 + 1; + const int n_frames = (int) (spec.size() / (2 * n_bins)); + const int64_t n_out = (int64_t) (n_frames - 1) * KOKORO_HOP; + float win[KOKORO_N_FFT], tab_cos[KOKORO_N_FFT], tab_sin[KOKORO_N_FFT]; + for (int i = 0; i < KOKORO_N_FFT; i++) { + win[i] = 0.5f - 0.5f * cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT); + tab_cos[i] = cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT); + tab_sin[i] = sinf(2.0f * (float) M_PI * i / KOKORO_N_FFT); + } + std::vector acc((size_t) n_out + KOKORO_N_FFT, 0.0f); + std::vector wsum((size_t) n_out + KOKORO_N_FFT, 0.0f); + for (int fr = 0; fr < n_frames; fr++) { + const float * row = spec.data() + (size_t) fr * 2 * n_bins; + float re[KOKORO_N_FFT / 2 + 1], im[KOKORO_N_FFT / 2 + 1]; + for (int k = 0; k < n_bins; k++) { + const float mag = expf(row[k]); + const float ph = sinf(row[n_bins + k]); + re[k] = mag * cosf(ph); + im[k] = mag * sinf(ph); + } + for (int i = 0; i < KOKORO_N_FFT; i++) { + float v = 0.0f; + for (int k = 0; k < n_bins; k++) { + const int a = (k * i) % KOKORO_N_FFT; + const float w = (k == 0 || k == KOKORO_N_FFT / 2) ? 1.0f : 2.0f; + v += w * (re[k] * tab_cos[a] - im[k] * tab_sin[a]); + } + const size_t o = (size_t) fr * KOKORO_HOP + i; + acc[o] += v / KOKORO_N_FFT * win[i]; + wsum[o] += win[i] * win[i]; + } + } + audio.resize((size_t) std::max(n_out, 0)); + for (size_t i = 0; i < audio.size(); i++) { + const size_t j = i + KOKORO_N_FFT / 2; // drop the centering pad + audio[i] = wsum[j] > 1e-11f ? acc[j] / wsum[j] : 0.0f; + } +} diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 5e55e29bc..802561765 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -258,6 +258,47 @@ struct mtmd_graph_chatterbox : mtmd_graph_chatterbox_base { ggml_cgraph * build_vocoder(int n_mel, int n_stft); }; +struct mtmd_graph_kokoro : mtmd_graph { + enum stage_type { + STAGE_TEXT, // tokens -> text encoder CNN + STAGE_F0N, // shared LSTM output -> F0 and energy curves + STAGE_DEC, // aligned features + source STFT -> STFT coefficients + }; + static constexpr int n_style = 128; // a voice row is the decoder style, then the predictor style + + stage_type stage; + int n_tok = 0; + int n_frames = 0; + int n_har = 0; + mtmd_graph_kokoro(mtmd_subcontext * ctx, const mtmd_image_f32 & img, stage_type stage, int n_tok, int n_frames, int n_har) + : mtmd_graph(ctx, img), stage(stage), n_tok(n_tok), n_frames(n_frames), n_har(n_har) {} + ggml_cgraph * build() override; + + // host side, between the graphs + // bert [n_tok, n_embd], text_cnn [n_tok, 512], style is the predictor style + // fills shared and asr as [n_frames, 512], returns n_frames + static int prosody(const mtmd_model & model, const float * bert, int n_embd, const float * text_cnn, int n_tok, + const float * style, float speed, std::vector & shared, std::vector & asr); + // fills har as [n_har, 22], returns n_har + static int har_source(const mtmd_model & model, const std::vector & f0, std::mt19937 & rng, std::vector & har); + static void istft(const std::vector & spec, std::vector & audio); + + private: + ggml_tensor * tensor(const std::string & name) const; + bool has_tensor(const std::string & name) const; + ggml_tensor * conv(const std::string & name, ggml_tensor * x, int stride, int pad, int dil) const; + ggml_tensor * adain(const std::string & name, ggml_tensor * x, ggml_tensor * style) const; + ggml_tensor * snake(const std::string & name, ggml_tensor * x) const; + ggml_tensor * interleave(ggml_tensor * a, ggml_tensor * b) const; + ggml_tensor * adain_resblk(const std::string & name, ggml_tensor * x, ggml_tensor * style, bool upsample) const; + ggml_tensor * gen_resblock(const std::string & name, ggml_tensor * x, ggml_tensor * style) const; + ggml_tensor * input(const char * name, ggml_type type, int64_t ne0, int64_t ne1) const; + ggml_cgraph * output(const char * name, ggml_tensor * t) const; + ggml_cgraph * build_text(); + ggml_cgraph * build_f0n(); + ggml_cgraph * build_dec(); +}; + struct mtmd_graph_lfm3a : mtmd_graph { mtmd_graph_lfm3a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp index 350aa4c9f..c57549d6e 100644 --- a/tools/mtmd/mtmd-graph.cpp +++ b/tools/mtmd/mtmd-graph.cpp @@ -1097,6 +1097,13 @@ std::unique_ptr mtmd_get_graph_builder(mtmd_subcontext * ctx, const const int n_frames = params && params->feats ? (int) (params->feats->size() / ctx->model.hparams.projection_dim) : 2; builder = std::make_unique(ctx, img, n_frames); } break; + case PROJECTOR_TYPE_KOKORO: + { + // without a stage (warmup), a short decoder graph: 2 frames, 600 samples each, one STFT row per 5 samples + const auto * io = params ? params->kokoro : nullptr; + builder = io ? std::make_unique(ctx, img, (mtmd_graph_kokoro::stage_type) io->stage, io->n_tok, io->n_frames, io->n_har) + : std::make_unique(ctx, img, mtmd_graph_kokoro::STAGE_DEC, 0, 2, 241); + } break; case PROJECTOR_TYPE_YOUTUVL: { builder = std::make_unique(ctx, img); diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index cac18b345..bd7ee1885 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -128,6 +128,8 @@ public: virtual bool supports_continue() const { return false; } // mtmd_helper_gen_audio_inp::speed and loudness other than the defaults virtual bool supports_delivery() const { return false; } + // speed alone, for models without loudness control + virtual bool supports_speed() const { return supports_delivery(); } // mtmd_helper_gen_audio_inp::cfg_scale other than the default virtual bool supports_cfg() const { return false; } // mtmd_helper_gen_audio_inp::flow_steps other than the default @@ -1121,6 +1123,244 @@ private: bool wav_header_sent = false; }; +// Kokoro is not autoregressive: the backbone encodes the phonemes once, the mmproj turns its states into the whole utterance +// the prompt is a phoneme string (the output of the "tts" chat template), one token per character +class kokoro_gen_audio_pipeline : public mtmd_gen_audio_pipeline { +public: + using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; + + // the predicted durations are divided by the speed + bool supports_speed() const override { return true; } + + void reset() override { + segments.clear(); + audio.clear(); + wav.clear(); + n_encoded = 0; + decoded = false; + sent = false; + } + + int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { + reset(); + if (!inp->prompt || inp->prompt_len == 0 || inp->seq_id < 0 || (uint32_t) inp->seq_id >= llama_n_seq_max(lctx)) { + LOG_ERR("mtmd_helper_gen_audio: kokoro requires a prompt\n"); + return 1; + } + if (inp->speaker_ref && !mtmd_bitmap_is_voice(inp->speaker_ref)) { + LOG_ERR("mtmd_helper_gen_audio: kokoro has no voice cloning, use one of its voices\n"); + return 1; + } + seq_id = inp->seq_id; + out_type = inp->out_type; + stream = inp->stream; + speed = inp->speed; + + // one token per phoneme character, unknown characters are dropped like in the reference + if (token_of.empty()) { + for (llama_token id = 0; id < llama_vocab_n_tokens(vocab); id++) { + if (llama_vocab_get_attr(vocab, id) & LLAMA_TOKEN_ATTR_NORMAL) { + token_of[llama_vocab_get_text(vocab, id)] = id; + } + } + } + std::vector phonemes; + const std::string prompt(inp->prompt, inp->prompt_len); + for (size_t i = 0; i < prompt.size();) { + const unsigned char c = (unsigned char) prompt[i]; + const size_t n = c < 0x80 ? 1 : c < 0xE0 ? 2 : c < 0xF0 ? 3 : 4; + const auto it = token_of.find(prompt.substr(i, n)); + if (it != token_of.end()) { + phonemes.push_back(it->second); + } + i += n; + } + if (phonemes.empty()) { + LOG_ERR("mtmd_helper_gen_audio: the prompt has no phoneme the model knows\n"); + return 1; + } + + // a voice is a table of style rows, the row is picked by the number of phonemes + const char * voice = inp->speaker_ref ? mtmd_bitmap_get_id(inp->speaker_ref) : mtmd_gen_audio_voice_name(mctx, 0); + if (!voice) { + LOG_ERR("mtmd_helper_gen_audio: the model has no voice\n"); + return 1; + } + const std::string name = std::string("voice.") + voice; + std::vector table(mtmd_gen_audio_read_tensor(mctx, name.c_str(), nullptr, 0)); + if (table.size() < n_voice_row || table.size() % n_voice_row != 0 || + mtmd_gen_audio_read_tensor(mctx, name.c_str(), table.data(), table.size()) != table.size()) { + LOG_ERR("mtmd_helper_gen_audio: unknown voice \"%s\"\n", voice); + return 1; + } + + // the backbone takes a whole segment in one batch, padded on both ends + const size_t n_max = std::min({(size_t) llama_n_ctx_seq(lctx), (size_t) llama_n_ubatch(lctx), (size_t) llama_model_n_ctx_train(model)}); + if (n_max < 3) { + LOG_ERR("mtmd_helper_gen_audio: the context is too small\n"); + return 1; + } + const llama_token pad = llama_vocab_bos(vocab); + for (size_t pos = 0; pos < phonemes.size();) { + const size_t len = split_len(phonemes, pos, n_max - 2); + segment seg; + seg.tokens.push_back(pad); + seg.tokens.insert(seg.tokens.end(), phonemes.begin() + pos, phonemes.begin() + pos + len); + seg.tokens.push_back(pad); + const size_t row = std::min(len - 1, table.size() / n_voice_row - 1); + seg.feats.assign(table.begin() + row * n_voice_row, table.begin() + (row + 1) * n_voice_row); + segments.push_back(std::move(seg)); + pos += len; + // a segment does not start with a space + while (pos < phonemes.size() && is_any(phonemes[pos], " ")) { + pos++; + } + } + + if (!init_gen_state(inp->seed)) { + return 1; + } + return 0; + } + + // the encoder is not causal, it takes one whole segment per call + int32_t step_prompt(int32_t) override { + if (segments.empty()) { + return -1; + } + if (n_encoded == segments.size()) { + return 0; + } + auto & seg = segments[n_encoded]; + const int n = (int) seg.tokens.size(); + llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); + if (!batch_set_tokens(batch.get(), seg.tokens.data(), n, 0, seq_id)) { + return -1; + } + for (int i = 0; i < n; ++i) { + llama_batch_ext_set_output_embd(batch.get(), i, true); + } + if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) { + LOG_ERR("mtmd_helper_gen_audio: kokoro backbone failed\n"); + return -1; + } + for (int i = 0; i < n; ++i) { + const float * h = llama_get_embeddings_ith(lctx, i); + if (!h) { + return -1; + } + seg.feats.insert(seg.feats.end(), h, h + n_embd); + } + n_encoded++; + return (int32_t) (segments.size() - n_encoded); + } + + // nothing is generated step by step + int32_t step_gen(llama_token, const float *, const float ** h_state_out, bool * out_stop) override { + *h_state_out = nullptr; + *out_stop = true; + return !segments.empty() && n_encoded == segments.size() ? 0 : 1; + } + + int32_t get_output(int32_t * sample_rate, const char ** data, size_t * data_len, int64_t * n_samples) override { + *sample_rate = info.sample_rate; + *data_len = 0; + if (n_samples) { *n_samples = 0; } + if (stream) { + // the audio comes out in one chunk after flush() + if (!decoded || sent) { + return 0; + } + sent = true; + } else if (!decode_audio()) { + return 1; + } + if (n_samples) { *n_samples = audio.size(); } + if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { + *data = (const char *) audio.data(); + *data_len = audio.size() * sizeof(float); + } else { + wav.clear(); + if (!write_wav16(wav, audio, info.sample_rate)) { + return 1; + } + *data = wav.data(); + *data_len = wav.size(); + } + return 0; + } + + int32_t flush() override { + return decode_audio() ? 0 : 1; + } + +private: + struct segment { + std::vector tokens; + std::vector feats; // the voice row, then the backbone state of each token + }; + + bool is_any(llama_token id, const char * chars) const { + const char * text = llama_vocab_get_text(vocab, id); + return text[0] != '\0' && strstr(chars, text) != nullptr; + } + + // length of the segment that starts at pos: a long text is cut at the last sentence end that fits, else at a weaker break + size_t split_len(const std::vector & phonemes, size_t pos, size_t n_max) const { + if (phonemes.size() - pos <= n_max) { + return phonemes.size() - pos; + } + for (const char * breaks : {"!.?\xe2\x80\xa6", ":;", ",\xe2\x80\x94", " "}) { + for (size_t len = n_max; len > 1; len--) { + if (is_any(phonemes[pos + len - 1], breaks)) { + return len; + } + } + } + return n_max; + } + + bool decode_audio() { + if (decoded) { + return true; + } + if (segments.empty() || n_encoded != segments.size()) { + return false; + } + for (const auto & seg : segments) { + std::vector codes(seg.tokens.begin(), seg.tokens.end()); + mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; + inp.codes = codes.data(); + inp.n_codes = codes.size(); + inp.feats = seg.feats.data(); + inp.n_feats = seg.feats.size(); + inp.speed = speed; + mtmd_gen_out out{}; + if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { + return false; + } + audio.insert(audio.end(), out.audio, out.audio + out.n_samples); + } + decoded = true; + return true; + } + + static constexpr size_t n_voice_row = 256; // decoder style, then predictor style + + llama_seq_id seq_id = 0; + std::unordered_map token_of; + std::vector segments; + size_t n_encoded = 0; + float speed = 1.0f; + std::vector audio; + std::vector wav; + mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; + bool stream = false; + bool decoded = false; + bool sent = false; +}; + // Soprano uses the hidden state that predicts each non-EOS token, including the prompt's last state. class soprano_gen_audio_pipeline : public mtmd_gen_audio_pipeline { public: @@ -4322,6 +4562,8 @@ static std::unique_ptr make_pipeline(llama_context * lc return std::unique_ptr(new pockettts_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_SOPRANO: return std::unique_ptr(new soprano_gen_audio_pipeline(lctx, mctx)); + case MTMD_GEN_AUDIO_TYPE_KOKORO: + return std::unique_ptr(new kokoro_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_LFM3A: return std::unique_ptr(new lfm3a_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_CHATTERBOX: @@ -4398,8 +4640,12 @@ int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_ LOG_ERR("mtmd_helper_gen_audio: speed must be > 0 and loudness < 0 LUFS (0 = model default)\n"); return 2; } - if ((inp->speed != 1.0f || inp->loudness != 0.0f) && !ctx->pipeline->supports_delivery()) { - LOG_ERR("mtmd_helper_gen_audio: the model does not support speed or loudness\n"); + if (inp->speed != 1.0f && !ctx->pipeline->supports_speed()) { + LOG_ERR("mtmd_helper_gen_audio: the model does not support speed\n"); + return 2; + } + if (inp->loudness != 0.0f && !ctx->pipeline->supports_delivery()) { + LOG_ERR("mtmd_helper_gen_audio: the model does not support loudness\n"); return 2; } if (!std::isfinite(inp->cfg_scale) || inp->cfg_scale < 0.0f) { diff --git a/tools/mtmd/mtmd-impl.h b/tools/mtmd/mtmd-impl.h index e1dc95cee..142b56e8d 100644 --- a/tools/mtmd/mtmd-impl.h +++ b/tools/mtmd/mtmd-impl.h @@ -615,6 +615,7 @@ enum projector_type { PROJECTOR_TYPE_POCKETTTS_SPKENC, PROJECTOR_TYPE_POCKETTTS_GEN, PROJECTOR_TYPE_SOPRANO, + PROJECTOR_TYPE_KOKORO, PROJECTOR_TYPE_LFM3A_GEN, PROJECTOR_TYPE_DRAMA3_GEN, PROJECTOR_TYPE_ZONOS3_GEN, @@ -711,6 +712,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_POCKETTTS_SPKENC, "pockettts_spkenc"}, { PROJECTOR_TYPE_POCKETTTS_GEN, "pockettts_gen"}, { PROJECTOR_TYPE_SOPRANO, "soprano"}, + { PROJECTOR_TYPE_KOKORO, "kokoro"}, { PROJECTOR_TYPE_LFM3A_GEN, "lfm3a_gen"}, { PROJECTOR_TYPE_DRAMA3_GEN, "drama3_gen"}, { PROJECTOR_TYPE_ZONOS3_GEN, "zonos3_gen"}, diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp index c352e28d9..38992233d 100644 --- a/tools/mtmd/mtmd-model.cpp +++ b/tools/mtmd/mtmd-model.cpp @@ -1172,6 +1172,10 @@ struct mtmd_model_loader { hparams.audio_sample_rate = 32000; hparams.audio_hop_len = 512; } break; + case PROJECTOR_TYPE_KOKORO: + { + hparams.audio_sample_rate = 24000; + } break; case PROJECTOR_TYPE_PADDLEOCR: { hparams.n_merge = 2; @@ -1578,6 +1582,7 @@ struct mtmd_model_loader { model.proj_type != PROJECTOR_TYPE_ZONOS3_SPKENC && model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN && model.proj_type != PROJECTOR_TYPE_SOPRANO && + model.proj_type != PROJECTOR_TYPE_KOKORO && model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO && model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR && model.proj_type != PROJECTOR_TYPE_CHATTERBOX && @@ -3459,6 +3464,21 @@ struct mtmd_model_loader { } } } break; + case PROJECTOR_TYPE_KOKORO: + { + // the graphs and the host code look the tensors up by name, see models/kokoro.cpp + // a voice is a table of style rows, the audio pipeline reads the row it needs + for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta.get()); t; t = ggml_get_next_tensor(ctx_meta.get(), t)) { + const std::string name = t->name; + if (name.rfind("a.gen.", 0) == 0) { + model.cbx_tensors[name] = get_tensor(name); + } else if (name.rfind("voice.", 0) == 0) { + model.cbx_tensors[name] = get_tensor(name); + model.gen_voices[name.substr(6)] = {}; + } + } + LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size()); + } break; case PROJECTOR_TYPE_SOPRANO: { auto & v = model.vocos; @@ -4955,6 +4975,7 @@ int mtmd_model::n_mmproj_embd() const { // talker conditioning rows, the speaker projection carries their width return cbx_tensors.at("a.cenc.spkr_enc.weight")->ne[1]; case PROJECTOR_TYPE_SOPRANO: + case PROJECTOR_TYPE_KOKORO: return hparams.projection_dim; case PROJECTOR_TYPE_PARAKEET: return mm_1_w->ne[1]; diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp index d6ec8e862..4e4d418e4 100644 --- a/tools/mtmd/mtmd-subcontext.cpp +++ b/tools/mtmd/mtmd-subcontext.cpp @@ -521,6 +521,87 @@ mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) { return st.release(); } +bool mtmd_subcontext::encode_kokoro(mtmd_subctx_encode_params * params) { + const int n_embd = model.n_mmproj_embd(); + const int n_style = mtmd_graph_kokoro::n_style; + if (params->gen_process != MTMD_SUBCTX_GEN_PROCESS_GEN_WAV || !params->codes || !params->feats || !params->gen_state || !params->out_audio) { + LOG_ERR("%s: kokoro takes the phoneme tokens and their backbone states\n", __func__); + return false; + } + const auto & tokens = *params->codes; + const auto & feats = *params->feats; + const int n_tok = (int) tokens.size(); + if (n_tok < 2 || feats.size() != (size_t) 2 * n_style + (size_t) n_tok * n_embd) { + LOG_ERR("%s: expected a voice row of %d values, then %d values for each of the %d tokens\n", __func__, 2 * n_style, n_embd, n_tok); + return false; + } + const int64_t n_vocab = model.cbx_tensors.at("a.gen.text.embedding.weight")->ne[1]; + for (int32_t t : tokens) { + if (t < 0 || t >= n_vocab) { + LOG_ERR("%s: phoneme token out of range (%d, vocabulary size %lld)\n", __func__, t, (long long) n_vocab); + return false; + } + } + const std::vector style_dec(feats.begin(), feats.begin() + n_style); + const std::vector style_pred(feats.begin() + n_style, feats.begin() + 2 * n_style); + const float * bert = feats.data() + 2 * n_style; + + auto run = [&](mtmd_subctx_kokoro_io & io) { + mtmd_subctx_encode_params p = *params; + p.kokoro = &io; + p.out_audio = nullptr; + return encode(&p); + }; + + std::vector text_cnn; + { + mtmd_subctx_kokoro_io io; + io.stage = mtmd_graph_kokoro::STAGE_TEXT; + io.n_tok = n_tok; + io.tokens = &tokens; + io.out0 = &text_cnn; + if (!run(io)) { + return false; + } + } + + std::vector shared, asr; + const int n_frames = mtmd_graph_kokoro::prosody(model, bert, n_embd, text_cnn.data(), n_tok, style_pred.data(), params->speed, shared, asr); + + std::vector f0, en; + { + mtmd_subctx_kokoro_io io; + io.stage = mtmd_graph_kokoro::STAGE_F0N; + io.n_frames = n_frames; + io.x = &shared; + io.style = &style_pred; + io.out0 = &f0; + io.out1 = &en; + if (!run(io)) { + return false; + } + } + + std::vector har, spec; + { + mtmd_subctx_kokoro_io io; + io.stage = mtmd_graph_kokoro::STAGE_DEC; + io.n_frames = n_frames; + io.n_har = mtmd_graph_kokoro::har_source(model, f0, params->gen_state->rng, har); + io.x = &asr; + io.style = &style_dec; + io.f0 = &f0; + io.en = &en; + io.har = &har; + io.out0 = &spec; + if (!run(io)) { + return false; + } + } + mtmd_graph_kokoro::istft(spec, *params->out_audio); + return true; +} + bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { const mtmd_image_f32_batch & imgs = *params->imgs; int n_batch_cur = imgs.entries.size(); @@ -536,6 +617,10 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { warmup(*params->imgs); } + if (model.proj_type == PROJECTOR_TYPE_KOKORO && !params->kokoro) { + return encode_kokoro(params); + } + const bool needs_gen_state = params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE || params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV || params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS; @@ -1514,6 +1599,29 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { { set_input_f32("inp_feats", *params->feats); } break; + case PROJECTOR_TYPE_KOKORO: + { + const auto * io = params->kokoro; + GGML_ASSERT(io); + switch (io->stage) { + case mtmd_graph_kokoro::STAGE_TEXT: + { + std::vector tokens = *io->tokens; + set_input_i32("inp_tokens", tokens); + } break; + case mtmd_graph_kokoro::STAGE_F0N: + set_input_f32("inp_x", *io->x); + set_input_f32("inp_style", *io->style); + break; + case mtmd_graph_kokoro::STAGE_DEC: + set_input_f32("inp_asr", *io->x); + set_input_f32("inp_style", *io->style); + set_input_f32("inp_f0", *io->f0); + set_input_f32("inp_n", *io->en); + set_input_f32("inp_har", *io->har); + break; + } + } break; case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN: { set_input_f32("inp_feats", *params->feats); @@ -2401,6 +2509,22 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { return false; } + if (model.proj_type == PROJECTOR_TYPE_KOKORO) { + const auto * io = params->kokoro; + auto read = [&](const char * name, std::vector * out) { + ggml_tensor * t = ggml_graph_get_tensor(gf, name); + GGML_ASSERT(t != nullptr && out != nullptr); + out->resize(ggml_nelements(t)); + ggml_backend_tensor_get(t, out->data(), 0, ggml_nbytes(t)); + }; + switch (io->stage) { + case mtmd_graph_kokoro::STAGE_TEXT: read("out_text", io->out0); break; + case mtmd_graph_kokoro::STAGE_F0N: read("out_f0", io->out0); read("out_n", io->out1); break; + case mtmd_graph_kokoro::STAGE_DEC: read("out_spec", io->out0); break; + } + return true; + } + if (embeddings != nullptr) { // sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one) const int n_tokens_out = embeddings->ne[1]; diff --git a/tools/mtmd/mtmd-subcontext.h b/tools/mtmd/mtmd-subcontext.h index d43acec04..e92ef01ee 100644 --- a/tools/mtmd/mtmd-subcontext.h +++ b/tools/mtmd/mtmd-subcontext.h @@ -114,6 +114,22 @@ struct mtmd_subctx_gen_state { } }; +// kokoro: inputs and outputs of one graph stage, see mtmd_graph_kokoro +struct mtmd_subctx_kokoro_io { + int stage = 0; + int n_tok = 0; + int n_frames = 0; + int n_har = 0; + const std::vector * tokens = nullptr; // STAGE_TEXT + const std::vector * x = nullptr; // STAGE_F0N: shared LSTM output, STAGE_DEC: aligned text features + const std::vector * style = nullptr; // predictor or decoder half of the voice row + const std::vector * f0 = nullptr; // STAGE_DEC + const std::vector * en = nullptr; + const std::vector * har = nullptr; + std::vector * out0 = nullptr; // text features, F0, or STFT coefficients + std::vector * out1 = nullptr; // STAGE_F0N: energy +}; + struct mtmd_subctx_encode_params { int n_threads = 1; const mtmd_image_f32_batch * imgs = nullptr; @@ -148,6 +164,10 @@ struct mtmd_subctx_encode_params { int vocode_n_stft = 0; std::vector * out_spec = nullptr; + // kokoro: set by encode_kokoro() for each of its graph stages + const mtmd_subctx_kokoro_io * kokoro = nullptr; + float speed = 1.0f; // kokoro: the predicted durations are divided by it + // audio gen: state of one sequence, encode() updates it in place struct mtmd_subctx_gen_state * gen_state = nullptr; @@ -205,6 +225,9 @@ private: mtmd_image_f32_batch get_dummy_batch() const; + // kokoro GEN_WAV: codes are the phoneme tokens, feats the voice row followed by the backbone state of each token + bool encode_kokoro(mtmd_subctx_encode_params * params); + // only initialize backend buffers, but do not allocate them yet support_info_graph reserve_compute_meta(const mtmd_image_f32_batch & batch); }; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 7c22a1041..a7bfe8f49 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -2694,6 +2694,10 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { info.type = MTMD_GEN_AUDIO_TYPE_FISH_S2; info.sample_rate = 44100; break; + case PROJECTOR_TYPE_KOKORO: + info.type = MTMD_GEN_AUDIO_TYPE_KOKORO; + info.sample_rate = 24000; + break; default: info.type = MTMD_GEN_AUDIO_TYPE_NONE; break; @@ -3091,7 +3095,8 @@ mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * na mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { mtmd_gen_inp inp{}; - inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; + inp.speed = 1.0f; if (!ctx->ctx_gen_a) { return inp; } @@ -3110,6 +3115,7 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { inp.temp = 0.3f; break; case PROJECTOR_TYPE_SOPRANO: + case PROJECTOR_TYPE_KOKORO: inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; break; case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN: @@ -3738,6 +3744,46 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * } } + if (sctx->model.proj_type == PROJECTOR_TYPE_KOKORO) { + if (inp->type != MTMD_GEN_PROCESS_TYPE_GEN_WAV || !inp->codes || inp->n_codes == 0 || !inp->feats || inp->n_feats == 0) { + LOG_ERR("%s: kokoro requires the phoneme tokens and their features\n", __func__); + return 1; + } + if (!std::isfinite(inp->speed) || inp->speed <= 0.0f) { + LOG_ERR("%s: speed must be > 0\n", __func__); + return 1; + } + const std::vector in_codes(inp->codes, inp->codes + inp->n_codes); + const std::vector in_feats(inp->feats, inp->feats + inp->n_feats); + + // the batch entry is unused, present to satisfy the encode interface + mtmd_image_f32 dummy; + dummy.set_size({1, 1}, false, true); + dummy.cpy_buf(std::vector(1, 0.0f)); + mtmd_image_f32_batch batch; + batch.is_audio = true; + batch.entries.push_back(std::move(dummy)); + + mtmd_subctx_encode_params params; + params.imgs = &batch; + params.n_threads = ctx->n_threads; + params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV; + params.codes = &in_codes; + params.feats = &in_feats; + params.speed = inp->speed; + params.out_audio = &state->out_audio; + params.gen_state = state->gen_state; + + if (!sctx->encode(¶ms)) { + LOG_ERR("%s: encode failed (gen_wav)\n", __func__); + return 1; + } + + out->audio = state->out_audio.data(); + out->n_samples = state->out_audio.size(); + return 0; + } + if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_WAV && sctx->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN) { return mtmd_gen_wav_lfm3a(ctx, state, inp, out); } diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index 8d5b42a9f..2ed43c38f 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -478,6 +478,7 @@ enum mtmd_gen_audio_type { MTMD_GEN_AUDIO_TYPE_YUE2, MTMD_GEN_AUDIO_TYPE_MINIMAX_MUSIC3, MTMD_GEN_AUDIO_TYPE_FISH_S2, + MTMD_GEN_AUDIO_TYPE_KOKORO, }; struct mtmd_gen_audio_info { @@ -526,10 +527,12 @@ struct mtmd_gen_inp { // pass either codes (discrete) or feats (continuous), depending on the pipeline // lfm3a GEN_CODE also takes codes: the codes revealed so far, -1 for a masked codebook // zonos3 GEN_WAV: -1 for a residual code not known yet, the frame is decoded without it + // kokoro GEN_WAV takes both: codes are the phoneme tokens, feats is the voice row (256 values) then the backbone state of each token int32_t * codes; size_t n_codes; const float * feats; size_t n_feats; + float speed; // kokoro: speaking speed, > 0 (default: 1.0) // yue2 GEN_CODE: n_codes is the frame count, codes is not read (the backbone state holds them) // backbone_state: llama_state_seq_get_data() of the backbone_n_past positions (prompt, codes, end token) diff --git a/tools/server/README.md b/tools/server/README.md index 6dddbc81b..4643eb7a5 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -862,6 +862,8 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For the list `lang`: Language code for the utterance (model-dependent, e.g. `en`, `zh`). Optional. +`raw_prompt`: If `true`, `input` is passed to the model as is, without the text preprocessing of the model's `tts` chat template (prompt format, text normalization, text to phonemes). For example, a Kokoro model then takes a phoneme string. Default: `false` + `voice`: Voice for the utterance, one of the `voices` listed by `GET /props` (model-dependent, e.g. `female` or `male` for LFM3-Audio). Optional: a model with predefined voices speaks with the first of them when neither `voice` nor `speaker_ref` is given. It rejects any other name, and a predefined voice cannot be combined with `speaker_ref`. `speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence. Rejected when the model has no voice encoder (e.g. Pocket TTS without voice cloning). @@ -1678,7 +1680,7 @@ Generates speech from text, for models that can generate audio. The request is c `stream_format`: `audio` streams the response as it becomes available, see `stream` in `/tts` for the WAV header it writes. `sse` is rejected. The response is not streamed when omitted. -`speed`: Speaking speed, see `speed` in `/tts`. Only models that support it accept a value other than `1.0`. `loudness`, `cfg_scale` and `flow_steps` of `/tts` can be used as well. +`speed`: Speaking speed, see `speed` in `/tts`. Only models that support it accept a value other than `1.0`. `loudness`, `cfg_scale`, `flow_steps`, `lang` and `raw_prompt` of `/tts` can be used as well. `instructions`: Ignored. diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 6a57aa8ca..145cfa047 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -6984,7 +6984,9 @@ std::unique_ptr server_routes::handle_tts_impl(const serve // speaker_ref in the request overrides the default voice const bool has_speaker_ref = req.files.count("speaker_ref") > 0 || !json_value(body, "speaker_ref_b64", std::string()).empty(); const std::string voice = json_value(body, "voice", has_speaker_ref ? std::string() : params.tts_voice); - const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice); + // raw_prompt: the input goes to the model as is (e.g. phonemes), the "tts" chat template is skipped + const bool raw_prompt = json_value(body, "raw_prompt", false); + const std::string tts_prompt = raw_prompt ? std::string() : common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice); // a model with predefined voices only speaks with one of them mtmd::bitmap_ptr voice_ref; if (!voice.empty() && info.n_voices > 0) { diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp index 47f17de89..68e4d09ae 100644 --- a/tools/tts/tts.cpp +++ b/tools/tts/tts.cpp @@ -135,7 +135,13 @@ int main(int argc, char ** argv) { // a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it auto tmpls = common_chat_templates_init(model, ""); - std::string prompt = common_chat_get_tts_prompt(tmpls.get(), params.prompt, params.tts_lang, params.tts_voice); + std::string prompt; + try { + prompt = common_chat_get_tts_prompt(tmpls.get(), params.prompt, params.tts_lang, params.tts_voice); + } catch (const std::exception & e) { + LOG_ERR("failed to prepare the text to speak: %s\n", e.what()); + return 1; + } if (prompt.empty()) { prompt = params.prompt; }