Files
llama.cpp/scripts/chat_tts_norm.py
T
Xuan-Son Nguyen 8d1599fca1 chat: add tts_dict for kokoro G2P template (#32397)
* chat: add tts_dict for kokoro G2P template

* more testing

* value_float_t range check

* no json-in-meta, add llama_model_meta_arr_str/_n
2026-11-17 12:52:50 +01:00

579 lines
19 KiB
Python

#!/usr/bin/env python3
"""Reference implementation of the template functions tts_norm_scan(), tts_norm_sub() (TTS normalization patterns) and tts_dict().
The specification is in docs/tts-norm-patterns.md. This file must give the same result as the C++ implementation
(common/chat-tts-norm.cpp) for every input; tests/test-jinja.cpp checks this in python mode (test-jinja -py).
Usage with jinja2:
env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan
env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub
env.globals["tts_dict"] = chat_tts_norm.make_tts_dict(dicts) # dicts: name -> {"keys", "values"} of tokenizer.tts_dict.<name>
"""
from __future__ import annotations
import json
from dataclasses import dataclass, field
from typing import Any, Callable
# character classes, explicit codepoint ranges (keep in sync with common/chat-tts-norm.cpp)
TTS_NORM_SPACE = [
(0x0009, 0x000D), (0x0020, 0x0020), (0x0085, 0x0085), (0x00A0, 0x00A0), (0x1680, 0x1680),
(0x2000, 0x200A), (0x2028, 0x2029), (0x202F, 0x202F), (0x205F, 0x205F), (0x3000, 0x3000),
]
TTS_NORM_PUNCT = [
(0x0021, 0x002F), (0x003A, 0x0040), (0x005B, 0x0060), (0x007B, 0x007E),
(0x00A1, 0x00A9), (0x00AB, 0x00B4), (0x00B6, 0x00B9), (0x00BB, 0x00BF), (0x00D7, 0x00D7), (0x00F7, 0x00F7),
(0x2010, 0x2027), (0x2030, 0x205E), (0x20A0, 0x20CF), (0x2100, 0x218F), (0x2190, 0x2BFF),
(0x3001, 0x3003), (0x3008, 0x3020), (0x3030, 0x3030), (0x303D, 0x303D), (0x30FB, 0x30FB),
(0xFE10, 0xFE19), (0xFE30, 0xFE4F), (0xFE50, 0xFE6B),
(0xFF01, 0xFF0F), (0xFF1A, 0xFF20), (0xFF3B, 0xFF40), (0xFF5B, 0xFF65),
(0x1F000, 0x1FAFF),
]
TTS_NORM_CONTROL = [
(0x0000, 0x001F), (0x007F, 0x009F), (0x200B, 0x200F), (0x2060, 0x206F), (0xFEFF, 0xFEFF),
]
TTS_NORM_HAN = [
(0x2E80, 0x2FDF), (0x3005, 0x3005), (0x3007, 0x3007), (0x3021, 0x3029), (0x3038, 0x303B),
(0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), (0x20000, 0x323AF),
]
CLASS_LETTERS = "dspawulhc"
INF = 1 << 31
MAX_REPEAT = 1000
MAX_PATTERN_LEN = 4096
def _in(ranges: list[tuple[int, int]], c: int) -> bool:
return any(lo <= c <= hi for lo, hi in ranges)
def _is_alpha(c: int) -> bool:
if ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z"):
return True
if c in (0xAA, 0xB5, 0xBA):
return True
return c >= 0xC0 and not _in(TTS_NORM_SPACE, c) and not _in(TTS_NORM_PUNCT, c) and not _in(TTS_NORM_CONTROL, c)
def _class_match(kind: str, c: int) -> bool:
if kind == "d":
return ord("0") <= c <= ord("9")
if kind == "s":
return _in(TTS_NORM_SPACE, c)
if kind == "p":
return _in(TTS_NORM_PUNCT, c)
if kind == "a":
return _is_alpha(c)
if kind == "w":
return _is_alpha(c) or ord("0") <= c <= ord("9")
if kind == "u":
return ord("A") <= c <= ord("Z")
if kind == "l":
return ord("a") <= c <= ord("z")
if kind == "h":
return _in(TTS_NORM_HAN, c)
if kind == "c":
return _in(TTS_NORM_CONTROL, c)
return False
def _ascii_lower(c: int) -> int:
return c + 32 if ord("A") <= c <= ord("Z") else c
def _ascii_swapcase(c: int) -> int:
if ord("A") <= c <= ord("Z"):
return c + 32
if ord("a") <= c <= ord("z"):
return c - 32
return c
def _is_ascii_alnum(c: int) -> bool:
return ord("0") <= c <= ord("9") or ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z")
# AST
LITERAL, SET, ANY, START, END, BOUNDARY, GROUP, LOOKAHEAD, LOOKBEHIND = range(9)
ZERO_WIDTH = (START, END, BOUNDARY, LOOKAHEAD, LOOKBEHIND)
@dataclass
class _Set:
neg: bool = False
ranges: list[tuple[int, int]] = field(default_factory=list)
classes: list[tuple[str, bool]] = field(default_factory=list)
@dataclass
class _Node:
op: int = GROUP
cp: int = 0
set: _Set = field(default_factory=_Set)
alts: list[list[_Item]] = field(default_factory=list)
capture: int = -1
neg: bool = False
@dataclass
class _Item:
atom: _Node
min: int = 1
max: int = 1
class PatternError(ValueError):
pass
class _Parser:
def __init__(self, s: list[int]):
self.s = s
self.p = 0
self.n_caps = 0
def fail(self, msg: str):
raise PatternError(f"invalid TTS norm pattern: {msg} at position {self.p}")
def eof(self) -> bool:
return self.p >= len(self.s)
def peek_is(self, c: str) -> bool:
return not self.eof() and self.s[self.p] == ord(c)
def expect(self, c: str, what: str):
if not self.peek_is(c):
self.fail(f"expected {what}")
self.p += 1
def parse_alts(self) -> list[list[_Item]]:
alts = [self.parse_seq()]
while self.peek_is("|"):
self.p += 1
alts.append(self.parse_seq())
return alts
def parse_seq(self) -> list[_Item]:
seq: list[_Item] = []
while not self.eof() and self.s[self.p] not in (ord("|"), ord(")")):
item = _Item(self.parse_atom())
self.parse_quantifier(item)
seq.append(item)
return seq
def parse_int(self) -> int:
if self.eof() or not (ord("0") <= self.s[self.p] <= ord("9")):
self.fail("expected number in quantifier")
v = 0
while not self.eof() and ord("0") <= self.s[self.p] <= ord("9"):
v = v * 10 + self.s[self.p] - ord("0")
if v > MAX_REPEAT:
self.fail(f"repetition count exceeds {MAX_REPEAT}")
self.p += 1
return v
def parse_quantifier(self, item: _Item):
if self.eof():
return
c = chr(self.s[self.p])
if c not in "?*+{":
return
if item.atom.op in ZERO_WIDTH:
self.fail("zero-width assertion cannot be repeated")
self.p += 1
if c == "?":
item.min, item.max = 0, 1
elif c == "*":
item.min, item.max = 0, INF
elif c == "+":
item.min, item.max = 1, INF
else:
item.min = self.parse_int()
item.max = item.min
if self.peek_is(","):
self.p += 1
item.max = INF if self.peek_is("}") else self.parse_int()
self.expect("}", "'}' to close quantifier")
if item.max < item.min:
self.fail("quantifier max is smaller than min")
if not self.eof() and chr(self.s[self.p]) in "?*+{":
self.fail("multiple quantifiers")
def parse_group(self) -> _Node:
node = _Node(op=GROUP)
if self.peek_is("?"):
self.p += 1
if self.eof():
self.fail("unterminated group")
c = chr(self.s[self.p])
if c == ":":
self.p += 1
elif c in "=!":
self.p += 1
node.op = LOOKAHEAD
node.neg = c == "!"
elif c == "<" and self.p + 1 < len(self.s) and self.s[self.p + 1] in (ord("="), ord("!")):
node.op = LOOKBEHIND
node.neg = self.s[self.p + 1] == ord("!")
self.p += 2
if self.eof() or self.s[self.p] == ord(")"):
self.fail("lookbehind must contain exactly one codepoint matcher")
atom = self.parse_atom()
if atom.op not in (LITERAL, SET, ANY):
self.fail("lookbehind must contain exactly one codepoint matcher")
if not self.peek_is(")"):
self.fail("lookbehind must contain exactly one codepoint matcher")
self.p += 1
node.alts = [[_Item(atom)]]
return node
elif c == "i":
self.fail("(?i) is only allowed at the start of the pattern")
else:
self.fail("unknown group type")
else:
node.capture = self.n_caps
self.n_caps += 1
node.alts = self.parse_alts()
self.expect(")", "')' to close group")
return node
# returns ("class", (kind, neg)) | ("boundary", None) | ("literal", cp)
def parse_escape(self, in_set: bool) -> tuple[str, Any]:
if self.eof():
self.fail("dangling '%'")
e = self.s[self.p]
self.p += 1
if e < 0x80 and chr(e).lower() in CLASS_LETTERS:
return "class", (chr(e).lower(), ord("A") <= e <= ord("Z"))
if e == ord("b") and not in_set:
return "boundary", None
if _is_ascii_alnum(e):
self.p -= 1
self.fail(f"unknown escape '%{chr(e)}'")
return "literal", e
def parse_set(self) -> _Node:
node = _Node(op=SET)
if self.peek_is("^"):
node.set.neg = True
self.p += 1
empty = True
while True:
if self.eof():
self.fail("unterminated set")
if self.s[self.p] == ord("]"):
self.p += 1
break
empty = False
if self.s[self.p] == ord("%"):
self.p += 1
kind, val = self.parse_escape(True)
if kind == "class":
node.set.classes.append(val)
continue
lo = val
else:
lo = self.s[self.p]
self.p += 1
hi = lo
if self.peek_is("-") and self.p + 1 < len(self.s) and self.s[self.p + 1] != ord("]"):
self.p += 1
if self.s[self.p] == ord("%"):
self.p += 1
kind, val = self.parse_escape(True)
if kind == "class":
self.fail("character class cannot be used in a range")
hi = val
else:
hi = self.s[self.p]
self.p += 1
if hi < lo:
self.fail("invalid range in set")
node.set.ranges.append((lo, hi))
if empty:
self.fail("empty set")
return node
def parse_atom(self) -> _Node:
c = chr(self.s[self.p])
self.p += 1
if c == "(":
return self.parse_group()
if c == "[":
return self.parse_set()
if c == ".":
return _Node(op=ANY)
if c == "^":
return _Node(op=START)
if c == "$":
return _Node(op=END)
if c == "%":
kind, val = self.parse_escape(False)
if kind == "class":
node = _Node(op=SET)
node.set.classes.append(val)
return node
if kind == "boundary":
return _Node(op=BOUNDARY)
return _Node(op=LITERAL, cp=val)
if c in "?*+{":
self.p -= 1
self.fail("nothing to repeat")
if c in "]}":
self.p -= 1
self.fail(f"unescaped '{c}', use '%{c}'")
return _Node(op=LITERAL, cp=ord(c))
class _Matcher:
def __init__(self, t: list[int], icase: bool, n_groups: int):
self.t = t
self.icase = icase
self.caps: list[tuple[int, int]] = [(-1, -1)] * n_groups
def is_w(self, i: int) -> bool:
return i < len(self.t) and _class_match("w", self.t[i])
def set_match(self, st: _Set, c: int) -> bool:
def test(x: int) -> bool:
if any(lo <= x <= hi for lo, hi in st.ranges):
return True
return any(_class_match(kind, x) != neg for kind, neg in st.classes)
m = test(c)
if not m and self.icase:
o = _ascii_swapcase(c)
if o != c:
m = test(o)
return m != st.neg
def match_alts(self, alts: list[list[_Item]], i: int) -> int:
"""returns the end position, or -1 if no alternative matches"""
for seq in alts:
saved = list(self.caps)
end = self.match_seq(seq, i)
if end >= 0:
return end
self.caps = saved
return -1
def match_seq(self, seq: list[_Item], i: int) -> int:
pos = i
for it in seq:
count = 0
while count < it.max:
is_group = it.atom.op == GROUP
saved = list(self.caps) if is_group else None
e = self.match_once(it.atom, pos)
if e < 0:
if saved is not None:
self.caps = saved
break
if e == pos:
# an empty iteration would repeat forever, it satisfies any remaining minimum
count = max(count + 1, it.min)
break
pos = e
count += 1
if count < it.min:
return -1
return pos
def match_once(self, node: _Node, i: int) -> int:
"""returns the end position, or -1 on failure"""
n = len(self.t)
op = node.op
if op == LITERAL:
if i < n and (self.t[i] == node.cp or (self.icase and _ascii_lower(self.t[i]) == _ascii_lower(node.cp))):
return i + 1
return -1
if op == SET:
return i + 1 if i < n and self.set_match(node.set, self.t[i]) else -1
if op == ANY:
return i + 1 if i < n else -1
if op == START:
return i if i == 0 else -1
if op == END:
return i if i == n else -1
if op == BOUNDARY:
return i if (i > 0 and self.is_w(i - 1)) != self.is_w(i) else -1
if op == GROUP:
e = self.match_alts(node.alts, i)
if e < 0:
return -1
if node.capture >= 0:
self.caps[node.capture] = (i, e)
return e
if op == LOOKAHEAD:
# captures made inside a lookahead are discarded
saved = list(self.caps)
ok = self.match_alts(node.alts, i) >= 0
self.caps = saved
return i if ok != node.neg else -1
if op == LOOKBEHIND:
ok = i > 0 and self.match_once(node.alts[0][0].atom, i - 1) == i
return i if ok != node.neg else -1
return -1
class Pattern:
def __init__(self, src: str):
if len(src.encode("utf-8")) > MAX_PATTERN_LEN:
raise PatternError(f"invalid TTS norm pattern: pattern is longer than {MAX_PATTERN_LEN} bytes")
self.icase = False
parser = _Parser([ord(c) for c in src])
if src.startswith("(?i)"):
self.icase = True
parser.p = 4
self.root = _Node(op=GROUP, alts=parser.parse_alts())
if not parser.eof():
parser.fail("unbalanced ')'")
self.n_caps = parser.n_caps
def match_at(self, t: list[int], pos: int) -> tuple[int, list[tuple[int, int]]] | None:
m = _Matcher(t, self.icase, self.n_caps)
end = m.match_alts(self.root.alts, pos)
if end < 0:
return None
return end, m.caps
# (rule index or -1, start, end, groups) with codepoint offsets
Segment = tuple[int, int, int, list[tuple[int, int]]]
def scan_segments(text: str, rules: list[Pattern]) -> list[Segment]:
t = [ord(c) for c in text]
n = len(t)
out: list[Segment] = []
i = 0
pending = 0
while i < n:
matched = False
for r, rule in enumerate(rules):
res = rule.match_at(t, i)
if res is not None and res[0] > i:
end, groups = res
if pending < i:
out.append((-1, pending, i, []))
out.append((r, i, end, groups))
i = end
pending = i
matched = True
break
if not matched:
i += 1
if pending < n:
out.append((-1, pending, n, []))
return out
def _rules_from_value(rules: Any) -> list[tuple[str, str]]:
if isinstance(rules, str):
return [("match", rules)]
if isinstance(rules, dict):
out = []
for k, v in rules.items():
if not isinstance(v, str):
raise TypeError("tts_norm_scan: rule pattern must be a string")
out.append((str(k), v))
return out
if isinstance(rules, (list, tuple)):
out = []
for item in rules:
if not isinstance(item, (list, tuple)) or len(item) != 2 or not isinstance(item[0], str) or not isinstance(item[1], str):
raise TypeError("tts_norm_scan: each rule must be a [name, pattern] pair of strings")
out.append((item[0], item[1]))
return out
raise TypeError("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs")
def tts_norm_scan(text: str, rules: Any) -> list[dict[str, Any]]:
rule_srcs = _rules_from_value(rules)
patterns = [Pattern(src) for _, src in rule_srcs]
out: list[dict[str, Any]] = []
for r, start, end, groups in scan_segments(text, patterns):
# groups that did not participate are empty strings, so they render as nothing and are falsy
vals = ["" if gs < 0 else text[gs:ge] for gs, ge in groups]
out.append({"rule": rule_srcs[r][0] if r >= 0 else None, "text": text[start:end], "groups": vals})
return out
def _parse_repl(repl: str, pattern: Pattern) -> list[str | int]:
pieces: list[str | int] = []
lit = ""
i = 0
while i < len(repl):
c = repl[i]
if c != "%":
lit += c
i += 1
continue
if i + 1 >= len(repl):
raise PatternError("invalid TTS norm replacement: dangling '%'")
i += 1
c = repl[i]
if c == "%":
lit += "%"
elif "0" <= c <= "9":
g = ord(c) - ord("0")
if g > pattern.n_caps:
raise PatternError(f"invalid TTS norm replacement: group %{c} does not exist")
if lit:
pieces.append(lit)
lit = ""
pieces.append(g)
else:
raise PatternError(f"invalid TTS norm replacement: unknown escape '%{c}'")
i += 1
if lit:
pieces.append(lit)
return pieces
def tts_norm_sub(text: str, pattern: str, repl: str) -> str:
pat = Pattern(pattern)
pieces = _parse_repl(repl, pat)
out = []
for r, start, end, groups in scan_segments(text, [pat]):
if r < 0:
out.append(text[start:end])
continue
for piece in pieces:
if isinstance(piece, str):
out.append(piece)
elif piece == 0:
out.append(text[start:end])
else:
gs, ge = groups[piece - 1]
if gs >= 0:
out.append(text[gs:ge])
return "".join(out)
def make_tts_dict(dicts: dict[str, dict[str, list[str]]]) -> Callable[[str, str], Any]:
# a dictionary is {"keys": [...], "values": [...]}, the values are JSON documents
parsed: dict[str, dict[str, str]] = {}
def tts_dict(name: str, key: str) -> Any:
if not isinstance(name, str) or not isinstance(key, str):
raise TypeError("tts_dict: name and key must be strings")
if name not in dicts:
raise KeyError(f"tts_dict: unknown dictionary '{name}'")
if name not in parsed:
keys, values = dicts[name]["keys"], dicts[name]["values"]
if len(keys) != len(values):
raise ValueError("tts_dict: malformed dictionary, a key has no value")
parsed[name] = dict(zip(keys, values))
doc = parsed[name].get(key)
return None if doc is None else json.loads(doc)
return tts_dict