* chat: add tts_dict for kokoro G2P template * more testing * value_float_t range check * no json-in-meta, add llama_model_meta_arr_str/_n
579 lines
19 KiB
Python
579 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Reference implementation of the template functions tts_norm_scan(), tts_norm_sub() (TTS normalization patterns) and tts_dict().
|
|
|
|
The specification is in docs/tts-norm-patterns.md. This file must give the same result as the C++ implementation
|
|
(common/chat-tts-norm.cpp) for every input; tests/test-jinja.cpp checks this in python mode (test-jinja -py).
|
|
|
|
Usage with jinja2:
|
|
|
|
env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan
|
|
env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub
|
|
env.globals["tts_dict"] = chat_tts_norm.make_tts_dict(dicts) # dicts: name -> {"keys", "values"} of tokenizer.tts_dict.<name>
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Callable
|
|
|
|
# character classes, explicit codepoint ranges (keep in sync with common/chat-tts-norm.cpp)
|
|
|
|
TTS_NORM_SPACE = [
|
|
(0x0009, 0x000D), (0x0020, 0x0020), (0x0085, 0x0085), (0x00A0, 0x00A0), (0x1680, 0x1680),
|
|
(0x2000, 0x200A), (0x2028, 0x2029), (0x202F, 0x202F), (0x205F, 0x205F), (0x3000, 0x3000),
|
|
]
|
|
|
|
TTS_NORM_PUNCT = [
|
|
(0x0021, 0x002F), (0x003A, 0x0040), (0x005B, 0x0060), (0x007B, 0x007E),
|
|
(0x00A1, 0x00A9), (0x00AB, 0x00B4), (0x00B6, 0x00B9), (0x00BB, 0x00BF), (0x00D7, 0x00D7), (0x00F7, 0x00F7),
|
|
(0x2010, 0x2027), (0x2030, 0x205E), (0x20A0, 0x20CF), (0x2100, 0x218F), (0x2190, 0x2BFF),
|
|
(0x3001, 0x3003), (0x3008, 0x3020), (0x3030, 0x3030), (0x303D, 0x303D), (0x30FB, 0x30FB),
|
|
(0xFE10, 0xFE19), (0xFE30, 0xFE4F), (0xFE50, 0xFE6B),
|
|
(0xFF01, 0xFF0F), (0xFF1A, 0xFF20), (0xFF3B, 0xFF40), (0xFF5B, 0xFF65),
|
|
(0x1F000, 0x1FAFF),
|
|
]
|
|
|
|
TTS_NORM_CONTROL = [
|
|
(0x0000, 0x001F), (0x007F, 0x009F), (0x200B, 0x200F), (0x2060, 0x206F), (0xFEFF, 0xFEFF),
|
|
]
|
|
|
|
TTS_NORM_HAN = [
|
|
(0x2E80, 0x2FDF), (0x3005, 0x3005), (0x3007, 0x3007), (0x3021, 0x3029), (0x3038, 0x303B),
|
|
(0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), (0x20000, 0x323AF),
|
|
]
|
|
|
|
CLASS_LETTERS = "dspawulhc"
|
|
INF = 1 << 31
|
|
MAX_REPEAT = 1000
|
|
MAX_PATTERN_LEN = 4096
|
|
|
|
|
|
def _in(ranges: list[tuple[int, int]], c: int) -> bool:
|
|
return any(lo <= c <= hi for lo, hi in ranges)
|
|
|
|
|
|
def _is_alpha(c: int) -> bool:
|
|
if ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z"):
|
|
return True
|
|
if c in (0xAA, 0xB5, 0xBA):
|
|
return True
|
|
return c >= 0xC0 and not _in(TTS_NORM_SPACE, c) and not _in(TTS_NORM_PUNCT, c) and not _in(TTS_NORM_CONTROL, c)
|
|
|
|
|
|
def _class_match(kind: str, c: int) -> bool:
|
|
if kind == "d":
|
|
return ord("0") <= c <= ord("9")
|
|
if kind == "s":
|
|
return _in(TTS_NORM_SPACE, c)
|
|
if kind == "p":
|
|
return _in(TTS_NORM_PUNCT, c)
|
|
if kind == "a":
|
|
return _is_alpha(c)
|
|
if kind == "w":
|
|
return _is_alpha(c) or ord("0") <= c <= ord("9")
|
|
if kind == "u":
|
|
return ord("A") <= c <= ord("Z")
|
|
if kind == "l":
|
|
return ord("a") <= c <= ord("z")
|
|
if kind == "h":
|
|
return _in(TTS_NORM_HAN, c)
|
|
if kind == "c":
|
|
return _in(TTS_NORM_CONTROL, c)
|
|
return False
|
|
|
|
|
|
def _ascii_lower(c: int) -> int:
|
|
return c + 32 if ord("A") <= c <= ord("Z") else c
|
|
|
|
|
|
def _ascii_swapcase(c: int) -> int:
|
|
if ord("A") <= c <= ord("Z"):
|
|
return c + 32
|
|
if ord("a") <= c <= ord("z"):
|
|
return c - 32
|
|
return c
|
|
|
|
|
|
def _is_ascii_alnum(c: int) -> bool:
|
|
return ord("0") <= c <= ord("9") or ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z")
|
|
|
|
|
|
# AST
|
|
|
|
LITERAL, SET, ANY, START, END, BOUNDARY, GROUP, LOOKAHEAD, LOOKBEHIND = range(9)
|
|
ZERO_WIDTH = (START, END, BOUNDARY, LOOKAHEAD, LOOKBEHIND)
|
|
|
|
|
|
@dataclass
|
|
class _Set:
|
|
neg: bool = False
|
|
ranges: list[tuple[int, int]] = field(default_factory=list)
|
|
classes: list[tuple[str, bool]] = field(default_factory=list)
|
|
|
|
|
|
@dataclass
|
|
class _Node:
|
|
op: int = GROUP
|
|
cp: int = 0
|
|
set: _Set = field(default_factory=_Set)
|
|
alts: list[list[_Item]] = field(default_factory=list)
|
|
capture: int = -1
|
|
neg: bool = False
|
|
|
|
|
|
@dataclass
|
|
class _Item:
|
|
atom: _Node
|
|
min: int = 1
|
|
max: int = 1
|
|
|
|
|
|
class PatternError(ValueError):
|
|
pass
|
|
|
|
|
|
class _Parser:
|
|
def __init__(self, s: list[int]):
|
|
self.s = s
|
|
self.p = 0
|
|
self.n_caps = 0
|
|
|
|
def fail(self, msg: str):
|
|
raise PatternError(f"invalid TTS norm pattern: {msg} at position {self.p}")
|
|
|
|
def eof(self) -> bool:
|
|
return self.p >= len(self.s)
|
|
|
|
def peek_is(self, c: str) -> bool:
|
|
return not self.eof() and self.s[self.p] == ord(c)
|
|
|
|
def expect(self, c: str, what: str):
|
|
if not self.peek_is(c):
|
|
self.fail(f"expected {what}")
|
|
self.p += 1
|
|
|
|
def parse_alts(self) -> list[list[_Item]]:
|
|
alts = [self.parse_seq()]
|
|
while self.peek_is("|"):
|
|
self.p += 1
|
|
alts.append(self.parse_seq())
|
|
return alts
|
|
|
|
def parse_seq(self) -> list[_Item]:
|
|
seq: list[_Item] = []
|
|
while not self.eof() and self.s[self.p] not in (ord("|"), ord(")")):
|
|
item = _Item(self.parse_atom())
|
|
self.parse_quantifier(item)
|
|
seq.append(item)
|
|
return seq
|
|
|
|
def parse_int(self) -> int:
|
|
if self.eof() or not (ord("0") <= self.s[self.p] <= ord("9")):
|
|
self.fail("expected number in quantifier")
|
|
v = 0
|
|
while not self.eof() and ord("0") <= self.s[self.p] <= ord("9"):
|
|
v = v * 10 + self.s[self.p] - ord("0")
|
|
if v > MAX_REPEAT:
|
|
self.fail(f"repetition count exceeds {MAX_REPEAT}")
|
|
self.p += 1
|
|
return v
|
|
|
|
def parse_quantifier(self, item: _Item):
|
|
if self.eof():
|
|
return
|
|
c = chr(self.s[self.p])
|
|
if c not in "?*+{":
|
|
return
|
|
if item.atom.op in ZERO_WIDTH:
|
|
self.fail("zero-width assertion cannot be repeated")
|
|
self.p += 1
|
|
if c == "?":
|
|
item.min, item.max = 0, 1
|
|
elif c == "*":
|
|
item.min, item.max = 0, INF
|
|
elif c == "+":
|
|
item.min, item.max = 1, INF
|
|
else:
|
|
item.min = self.parse_int()
|
|
item.max = item.min
|
|
if self.peek_is(","):
|
|
self.p += 1
|
|
item.max = INF if self.peek_is("}") else self.parse_int()
|
|
self.expect("}", "'}' to close quantifier")
|
|
if item.max < item.min:
|
|
self.fail("quantifier max is smaller than min")
|
|
if not self.eof() and chr(self.s[self.p]) in "?*+{":
|
|
self.fail("multiple quantifiers")
|
|
|
|
def parse_group(self) -> _Node:
|
|
node = _Node(op=GROUP)
|
|
if self.peek_is("?"):
|
|
self.p += 1
|
|
if self.eof():
|
|
self.fail("unterminated group")
|
|
c = chr(self.s[self.p])
|
|
if c == ":":
|
|
self.p += 1
|
|
elif c in "=!":
|
|
self.p += 1
|
|
node.op = LOOKAHEAD
|
|
node.neg = c == "!"
|
|
elif c == "<" and self.p + 1 < len(self.s) and self.s[self.p + 1] in (ord("="), ord("!")):
|
|
node.op = LOOKBEHIND
|
|
node.neg = self.s[self.p + 1] == ord("!")
|
|
self.p += 2
|
|
if self.eof() or self.s[self.p] == ord(")"):
|
|
self.fail("lookbehind must contain exactly one codepoint matcher")
|
|
atom = self.parse_atom()
|
|
if atom.op not in (LITERAL, SET, ANY):
|
|
self.fail("lookbehind must contain exactly one codepoint matcher")
|
|
if not self.peek_is(")"):
|
|
self.fail("lookbehind must contain exactly one codepoint matcher")
|
|
self.p += 1
|
|
node.alts = [[_Item(atom)]]
|
|
return node
|
|
elif c == "i":
|
|
self.fail("(?i) is only allowed at the start of the pattern")
|
|
else:
|
|
self.fail("unknown group type")
|
|
else:
|
|
node.capture = self.n_caps
|
|
self.n_caps += 1
|
|
node.alts = self.parse_alts()
|
|
self.expect(")", "')' to close group")
|
|
return node
|
|
|
|
# returns ("class", (kind, neg)) | ("boundary", None) | ("literal", cp)
|
|
def parse_escape(self, in_set: bool) -> tuple[str, Any]:
|
|
if self.eof():
|
|
self.fail("dangling '%'")
|
|
e = self.s[self.p]
|
|
self.p += 1
|
|
if e < 0x80 and chr(e).lower() in CLASS_LETTERS:
|
|
return "class", (chr(e).lower(), ord("A") <= e <= ord("Z"))
|
|
if e == ord("b") and not in_set:
|
|
return "boundary", None
|
|
if _is_ascii_alnum(e):
|
|
self.p -= 1
|
|
self.fail(f"unknown escape '%{chr(e)}'")
|
|
return "literal", e
|
|
|
|
def parse_set(self) -> _Node:
|
|
node = _Node(op=SET)
|
|
if self.peek_is("^"):
|
|
node.set.neg = True
|
|
self.p += 1
|
|
empty = True
|
|
while True:
|
|
if self.eof():
|
|
self.fail("unterminated set")
|
|
if self.s[self.p] == ord("]"):
|
|
self.p += 1
|
|
break
|
|
empty = False
|
|
if self.s[self.p] == ord("%"):
|
|
self.p += 1
|
|
kind, val = self.parse_escape(True)
|
|
if kind == "class":
|
|
node.set.classes.append(val)
|
|
continue
|
|
lo = val
|
|
else:
|
|
lo = self.s[self.p]
|
|
self.p += 1
|
|
hi = lo
|
|
if self.peek_is("-") and self.p + 1 < len(self.s) and self.s[self.p + 1] != ord("]"):
|
|
self.p += 1
|
|
if self.s[self.p] == ord("%"):
|
|
self.p += 1
|
|
kind, val = self.parse_escape(True)
|
|
if kind == "class":
|
|
self.fail("character class cannot be used in a range")
|
|
hi = val
|
|
else:
|
|
hi = self.s[self.p]
|
|
self.p += 1
|
|
if hi < lo:
|
|
self.fail("invalid range in set")
|
|
node.set.ranges.append((lo, hi))
|
|
if empty:
|
|
self.fail("empty set")
|
|
return node
|
|
|
|
def parse_atom(self) -> _Node:
|
|
c = chr(self.s[self.p])
|
|
self.p += 1
|
|
if c == "(":
|
|
return self.parse_group()
|
|
if c == "[":
|
|
return self.parse_set()
|
|
if c == ".":
|
|
return _Node(op=ANY)
|
|
if c == "^":
|
|
return _Node(op=START)
|
|
if c == "$":
|
|
return _Node(op=END)
|
|
if c == "%":
|
|
kind, val = self.parse_escape(False)
|
|
if kind == "class":
|
|
node = _Node(op=SET)
|
|
node.set.classes.append(val)
|
|
return node
|
|
if kind == "boundary":
|
|
return _Node(op=BOUNDARY)
|
|
return _Node(op=LITERAL, cp=val)
|
|
if c in "?*+{":
|
|
self.p -= 1
|
|
self.fail("nothing to repeat")
|
|
if c in "]}":
|
|
self.p -= 1
|
|
self.fail(f"unescaped '{c}', use '%{c}'")
|
|
return _Node(op=LITERAL, cp=ord(c))
|
|
|
|
|
|
class _Matcher:
|
|
def __init__(self, t: list[int], icase: bool, n_groups: int):
|
|
self.t = t
|
|
self.icase = icase
|
|
self.caps: list[tuple[int, int]] = [(-1, -1)] * n_groups
|
|
|
|
def is_w(self, i: int) -> bool:
|
|
return i < len(self.t) and _class_match("w", self.t[i])
|
|
|
|
def set_match(self, st: _Set, c: int) -> bool:
|
|
def test(x: int) -> bool:
|
|
if any(lo <= x <= hi for lo, hi in st.ranges):
|
|
return True
|
|
return any(_class_match(kind, x) != neg for kind, neg in st.classes)
|
|
|
|
m = test(c)
|
|
if not m and self.icase:
|
|
o = _ascii_swapcase(c)
|
|
if o != c:
|
|
m = test(o)
|
|
return m != st.neg
|
|
|
|
def match_alts(self, alts: list[list[_Item]], i: int) -> int:
|
|
"""returns the end position, or -1 if no alternative matches"""
|
|
for seq in alts:
|
|
saved = list(self.caps)
|
|
end = self.match_seq(seq, i)
|
|
if end >= 0:
|
|
return end
|
|
self.caps = saved
|
|
return -1
|
|
|
|
def match_seq(self, seq: list[_Item], i: int) -> int:
|
|
pos = i
|
|
for it in seq:
|
|
count = 0
|
|
while count < it.max:
|
|
is_group = it.atom.op == GROUP
|
|
saved = list(self.caps) if is_group else None
|
|
e = self.match_once(it.atom, pos)
|
|
if e < 0:
|
|
if saved is not None:
|
|
self.caps = saved
|
|
break
|
|
if e == pos:
|
|
# an empty iteration would repeat forever, it satisfies any remaining minimum
|
|
count = max(count + 1, it.min)
|
|
break
|
|
pos = e
|
|
count += 1
|
|
if count < it.min:
|
|
return -1
|
|
return pos
|
|
|
|
def match_once(self, node: _Node, i: int) -> int:
|
|
"""returns the end position, or -1 on failure"""
|
|
n = len(self.t)
|
|
op = node.op
|
|
if op == LITERAL:
|
|
if i < n and (self.t[i] == node.cp or (self.icase and _ascii_lower(self.t[i]) == _ascii_lower(node.cp))):
|
|
return i + 1
|
|
return -1
|
|
if op == SET:
|
|
return i + 1 if i < n and self.set_match(node.set, self.t[i]) else -1
|
|
if op == ANY:
|
|
return i + 1 if i < n else -1
|
|
if op == START:
|
|
return i if i == 0 else -1
|
|
if op == END:
|
|
return i if i == n else -1
|
|
if op == BOUNDARY:
|
|
return i if (i > 0 and self.is_w(i - 1)) != self.is_w(i) else -1
|
|
if op == GROUP:
|
|
e = self.match_alts(node.alts, i)
|
|
if e < 0:
|
|
return -1
|
|
if node.capture >= 0:
|
|
self.caps[node.capture] = (i, e)
|
|
return e
|
|
if op == LOOKAHEAD:
|
|
# captures made inside a lookahead are discarded
|
|
saved = list(self.caps)
|
|
ok = self.match_alts(node.alts, i) >= 0
|
|
self.caps = saved
|
|
return i if ok != node.neg else -1
|
|
if op == LOOKBEHIND:
|
|
ok = i > 0 and self.match_once(node.alts[0][0].atom, i - 1) == i
|
|
return i if ok != node.neg else -1
|
|
return -1
|
|
|
|
|
|
class Pattern:
|
|
def __init__(self, src: str):
|
|
if len(src.encode("utf-8")) > MAX_PATTERN_LEN:
|
|
raise PatternError(f"invalid TTS norm pattern: pattern is longer than {MAX_PATTERN_LEN} bytes")
|
|
self.icase = False
|
|
parser = _Parser([ord(c) for c in src])
|
|
if src.startswith("(?i)"):
|
|
self.icase = True
|
|
parser.p = 4
|
|
self.root = _Node(op=GROUP, alts=parser.parse_alts())
|
|
if not parser.eof():
|
|
parser.fail("unbalanced ')'")
|
|
self.n_caps = parser.n_caps
|
|
|
|
def match_at(self, t: list[int], pos: int) -> tuple[int, list[tuple[int, int]]] | None:
|
|
m = _Matcher(t, self.icase, self.n_caps)
|
|
end = m.match_alts(self.root.alts, pos)
|
|
if end < 0:
|
|
return None
|
|
return end, m.caps
|
|
|
|
|
|
# (rule index or -1, start, end, groups) with codepoint offsets
|
|
Segment = tuple[int, int, int, list[tuple[int, int]]]
|
|
|
|
|
|
def scan_segments(text: str, rules: list[Pattern]) -> list[Segment]:
|
|
t = [ord(c) for c in text]
|
|
n = len(t)
|
|
out: list[Segment] = []
|
|
i = 0
|
|
pending = 0
|
|
while i < n:
|
|
matched = False
|
|
for r, rule in enumerate(rules):
|
|
res = rule.match_at(t, i)
|
|
if res is not None and res[0] > i:
|
|
end, groups = res
|
|
if pending < i:
|
|
out.append((-1, pending, i, []))
|
|
out.append((r, i, end, groups))
|
|
i = end
|
|
pending = i
|
|
matched = True
|
|
break
|
|
if not matched:
|
|
i += 1
|
|
if pending < n:
|
|
out.append((-1, pending, n, []))
|
|
return out
|
|
|
|
|
|
def _rules_from_value(rules: Any) -> list[tuple[str, str]]:
|
|
if isinstance(rules, str):
|
|
return [("match", rules)]
|
|
if isinstance(rules, dict):
|
|
out = []
|
|
for k, v in rules.items():
|
|
if not isinstance(v, str):
|
|
raise TypeError("tts_norm_scan: rule pattern must be a string")
|
|
out.append((str(k), v))
|
|
return out
|
|
if isinstance(rules, (list, tuple)):
|
|
out = []
|
|
for item in rules:
|
|
if not isinstance(item, (list, tuple)) or len(item) != 2 or not isinstance(item[0], str) or not isinstance(item[1], str):
|
|
raise TypeError("tts_norm_scan: each rule must be a [name, pattern] pair of strings")
|
|
out.append((item[0], item[1]))
|
|
return out
|
|
raise TypeError("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs")
|
|
|
|
|
|
def tts_norm_scan(text: str, rules: Any) -> list[dict[str, Any]]:
|
|
rule_srcs = _rules_from_value(rules)
|
|
patterns = [Pattern(src) for _, src in rule_srcs]
|
|
out: list[dict[str, Any]] = []
|
|
for r, start, end, groups in scan_segments(text, patterns):
|
|
# groups that did not participate are empty strings, so they render as nothing and are falsy
|
|
vals = ["" if gs < 0 else text[gs:ge] for gs, ge in groups]
|
|
out.append({"rule": rule_srcs[r][0] if r >= 0 else None, "text": text[start:end], "groups": vals})
|
|
return out
|
|
|
|
|
|
def _parse_repl(repl: str, pattern: Pattern) -> list[str | int]:
|
|
pieces: list[str | int] = []
|
|
lit = ""
|
|
i = 0
|
|
while i < len(repl):
|
|
c = repl[i]
|
|
if c != "%":
|
|
lit += c
|
|
i += 1
|
|
continue
|
|
if i + 1 >= len(repl):
|
|
raise PatternError("invalid TTS norm replacement: dangling '%'")
|
|
i += 1
|
|
c = repl[i]
|
|
if c == "%":
|
|
lit += "%"
|
|
elif "0" <= c <= "9":
|
|
g = ord(c) - ord("0")
|
|
if g > pattern.n_caps:
|
|
raise PatternError(f"invalid TTS norm replacement: group %{c} does not exist")
|
|
if lit:
|
|
pieces.append(lit)
|
|
lit = ""
|
|
pieces.append(g)
|
|
else:
|
|
raise PatternError(f"invalid TTS norm replacement: unknown escape '%{c}'")
|
|
i += 1
|
|
if lit:
|
|
pieces.append(lit)
|
|
return pieces
|
|
|
|
|
|
def tts_norm_sub(text: str, pattern: str, repl: str) -> str:
|
|
pat = Pattern(pattern)
|
|
pieces = _parse_repl(repl, pat)
|
|
out = []
|
|
for r, start, end, groups in scan_segments(text, [pat]):
|
|
if r < 0:
|
|
out.append(text[start:end])
|
|
continue
|
|
for piece in pieces:
|
|
if isinstance(piece, str):
|
|
out.append(piece)
|
|
elif piece == 0:
|
|
out.append(text[start:end])
|
|
else:
|
|
gs, ge = groups[piece - 1]
|
|
if gs >= 0:
|
|
out.append(text[gs:ge])
|
|
return "".join(out)
|
|
|
|
|
|
def make_tts_dict(dicts: dict[str, dict[str, list[str]]]) -> Callable[[str, str], Any]:
|
|
# a dictionary is {"keys": [...], "values": [...]}, the values are JSON documents
|
|
parsed: dict[str, dict[str, str]] = {}
|
|
|
|
def tts_dict(name: str, key: str) -> Any:
|
|
if not isinstance(name, str) or not isinstance(key, str):
|
|
raise TypeError("tts_dict: name and key must be strings")
|
|
if name not in dicts:
|
|
raise KeyError(f"tts_dict: unknown dictionary '{name}'")
|
|
if name not in parsed:
|
|
keys, values = dicts[name]["keys"], dicts[name]["values"]
|
|
if len(keys) != len(values):
|
|
raise ValueError("tts_dict: malformed dictionary, a key has no value")
|
|
parsed[name] = dict(zip(keys, values))
|
|
doc = parsed[name].get(key)
|
|
return None if doc is None else json.loads(doc)
|
|
|
|
return tts_dict
|