chat: allow TTS model to provide custom text normalizer via jinja (#32053)

* chat: allow TTS model to provide custom text normalizer via jinja

* rm tts_norm_is_reserved_name
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-11-11 17:34:55 +01:00
1 parent 956591f457
commit f7b886fe6f
7 files changed
+1769 -11

No files matched your search

+2
View File
@@ -66,6 +66,8 @@ add_library(${TARGET}
chat-diff-analyzer.cpp
chat-peg-parser.cpp
chat-peg-parser.h
chat-tts-norm.cpp
chat-tts-norm.h
chat.cpp
chat.h
common.cpp
+864
View File
@@ -0,0 +1,864 @@
#include "chat-tts-norm.h"
#include "jinja/runtime.h"
#include "jinja/value.h"
#include <algorithm>
#include <climits>
#include <stdexcept>
#include <string>
#include <vector>
// TTS normalization patterns, see docs/tts-norm-patterns.md for the specification
// keep this file in sync with the python reference: scripts/chat_tts_norm.py
using namespace jinja;
//
// character classes
// they are defined with explicit codepoint ranges, independent of any unicode database version,
// so that the C++ and python implementations agree on every codepoint
//
struct tts_norm_range {
uint32_t lo;
uint32_t hi;
};
static const tts_norm_range TTS_NORM_SPACE[] = {
{0x0009, 0x000D}, {0x0020, 0x0020}, {0x0085, 0x0085}, {0x00A0, 0x00A0}, {0x1680, 0x1680},
{0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F}, {0x205F, 0x205F}, {0x3000, 0x3000},
};
static const tts_norm_range TTS_NORM_PUNCT[] = {
{0x0021, 0x002F}, {0x003A, 0x0040}, {0x005B, 0x0060}, {0x007B, 0x007E},
{0x00A1, 0x00A9}, {0x00AB, 0x00B4}, {0x00B6, 0x00B9}, {0x00BB, 0x00BF}, {0x00D7, 0x00D7}, {0x00F7, 0x00F7},
{0x2010, 0x2027}, {0x2030, 0x205E}, {0x20A0, 0x20CF}, {0x2100, 0x218F}, {0x2190, 0x2BFF},
{0x3001, 0x3003}, {0x3008, 0x3020}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30FB, 0x30FB},
{0xFE10, 0xFE19}, {0xFE30, 0xFE4F}, {0xFE50, 0xFE6B},
{0xFF01, 0xFF0F}, {0xFF1A, 0xFF20}, {0xFF3B, 0xFF40}, {0xFF5B, 0xFF65},
{0x1F000, 0x1FAFF},
};
static const tts_norm_range TTS_NORM_CONTROL[] = {
{0x0000, 0x001F}, {0x007F, 0x009F}, {0x200B, 0x200F}, {0x2060, 0x206F}, {0xFEFF, 0xFEFF},
};
static const tts_norm_range TTS_NORM_HAN[] = {
{0x2E80, 0x2FDF}, {0x3005, 0x3005}, {0x3007, 0x3007}, {0x3021, 0x3029}, {0x3038, 0x303B},
{0x3400, 0x4DBF}, {0x4E00, 0x9FFF}, {0xF900, 0xFAFF}, {0x20000, 0x323AF},
};
template<size_t N>
static bool tts_norm_in(const tts_norm_range (&ranges)[N], uint32_t c) {
for (const auto & r : ranges) {
if (c >= r.lo && c <= r.hi) {
return true;
}
}
return false;
}
static bool tts_norm_is_alpha(uint32_t c) {
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
return true;
}
if (c == 0xAA || c == 0xB5 || c == 0xBA) {
return true;
}
return c >= 0xC0 && !tts_norm_in(TTS_NORM_SPACE, c) && !tts_norm_in(TTS_NORM_PUNCT, c) && !tts_norm_in(TTS_NORM_CONTROL, c);
}
// kind is one of "dspawulhc"
static bool tts_norm_class_match(char kind, uint32_t c) {
switch (kind) {
case 'd': return c >= '0' && c <= '9';
case 's': return tts_norm_in(TTS_NORM_SPACE, c);
case 'p': return tts_norm_in(TTS_NORM_PUNCT, c);
case 'a': return tts_norm_is_alpha(c);
case 'w': return tts_norm_is_alpha(c) || (c >= '0' && c <= '9');
case 'u': return c >= 'A' && c <= 'Z';
case 'l': return c >= 'a' && c <= 'z';
case 'h': return tts_norm_in(TTS_NORM_HAN, c);
case 'c': return tts_norm_in(TTS_NORM_CONTROL, c);
}
return false;
}
static bool tts_norm_is_class_letter(uint32_t c) {
if (c >= 'A' && c <= 'Z') {
c += 'a' - 'A';
}
return c < 0x80 && std::string("dspawulhc").find(static_cast<char>(c)) != std::string::npos;
}
static uint32_t tts_norm_ascii_lower(uint32_t c) {
return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c;
}
static uint32_t tts_norm_ascii_swapcase(uint32_t c) {
if (c >= 'A' && c <= 'Z') return c + ('a' - 'A');
if (c >= 'a' && c <= 'z') return c - ('a' - 'A');
return c;
}
static bool tts_norm_is_ascii_alnum(uint32_t c) {
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
}
// invalid bytes are decoded as U+FFFD, one per byte; offs[i] is the byte offset of codepoint i, offs[n] = text size
static void tts_norm_decode_utf8(const std::string & s, std::vector<uint32_t> & cpts, std::vector<size_t> & offs) {
cpts.clear();
offs.clear();
size_t i = 0;
const size_t n = s.size();
while (i < n) {
offs.push_back(i);
const unsigned char c = static_cast<unsigned char>(s[i]);
uint32_t cp = 0;
size_t len = 0;
if (c < 0x80) {
cp = c; len = 1;
} else if ((c & 0xE0) == 0xC0) {
cp = c & 0x1F; len = 2;
} else if ((c & 0xF0) == 0xE0) {
cp = c & 0x0F; len = 3;
} else if ((c & 0xF8) == 0xF0) {
cp = c & 0x07; len = 4;
}
bool ok = len > 0 && i + len <= n;
for (size_t k = 1; ok && k < len; k++) {
const unsigned char cc = static_cast<unsigned char>(s[i + k]);
if ((cc & 0xC0) != 0x80) {
ok = false;
} else {
cp = (cp << 6) | (cc & 0x3F);
}
}
if (!ok) {
cpts.push_back(0xFFFD);
i += 1;
} else {
cpts.push_back(cp);
i += len;
}
}
offs.push_back(n);
}
//
// AST
//
enum class tts_norm_op {
LITERAL, // one codepoint
SET, // one codepoint from a set; class escapes like %d are sets with a single class
ANY, // any codepoint
START, // ^
END, // $
BOUNDARY, // %b
GROUP, // (...), (?:...), (?<name>...)
LOOKAHEAD, // (?=...), (?!...)
LOOKBEHIND, // (?<=c), (?<!c) - exactly one codepoint
};
struct tts_norm_class {
char kind;
bool neg;
};
struct tts_norm_set {
bool neg = false;
std::vector<std::pair<uint32_t, uint32_t>> ranges;
std::vector<tts_norm_class> classes;
};
struct tts_norm_item;
using tts_norm_seq = std::vector<tts_norm_item>;
struct tts_norm_pattern_node {
tts_norm_op op = tts_norm_op::GROUP;
uint32_t cp = 0; // LITERAL
tts_norm_set set; // SET
std::vector<tts_norm_seq> alts; // GROUP, LOOKAHEAD; LOOKBEHIND holds a single item
int capture = -1; // GROUP: capture index, -1 for non-capturing
bool neg = false; // LOOKAHEAD, LOOKBEHIND
};
static constexpr int TTS_NORM_INF = INT_MAX;
static constexpr int TTS_NORM_MAX_REPEAT = 1000;
static constexpr size_t TTS_NORM_MAX_PATTERN_LEN = 4096;
struct tts_norm_item {
tts_norm_pattern_node atom;
int min = 1;
int max = 1;
};
static bool tts_norm_is_zero_width(tts_norm_op op) {
return op == tts_norm_op::START || op == tts_norm_op::END || op == tts_norm_op::BOUNDARY
|| op == tts_norm_op::LOOKAHEAD || op == tts_norm_op::LOOKBEHIND;
}
//
// parser
//
struct tts_norm_parser {
std::vector<uint32_t> s;
size_t p = 0;
size_t & n_caps;
tts_norm_parser(std::vector<uint32_t> src, size_t & n_caps) : s(std::move(src)), n_caps(n_caps) {}
[[noreturn]] void fail(const std::string & msg) const {
throw std::runtime_error("invalid TTS norm pattern: " + msg + " at position " + std::to_string(p));
}
bool eof() const { return p >= s.size(); }
uint32_t peek() const { return s[p]; }
bool peek_is(uint32_t c) const { return !eof() && s[p] == c; }
void expect(uint32_t c, const char * what) {
if (!peek_is(c)) {
fail(std::string("expected ") + what);
}
p++;
}
std::vector<tts_norm_seq> parse_alts() {
std::vector<tts_norm_seq> alts;
alts.push_back(parse_seq());
while (peek_is('|')) {
p++;
alts.push_back(parse_seq());
}
return alts;
}
tts_norm_seq parse_seq() {
tts_norm_seq seq;
while (!eof() && peek() != '|' && peek() != ')') {
tts_norm_item item;
item.atom = parse_atom();
parse_quantifier(item);
seq.push_back(std::move(item));
}
return seq;
}
int parse_int() {
if (eof() || peek() < '0' || peek() > '9') {
fail("expected number in quantifier");
}
int v = 0;
while (!eof() && peek() >= '0' && peek() <= '9') {
v = v * 10 + static_cast<int>(peek() - '0');
if (v > TTS_NORM_MAX_REPEAT) {
fail("repetition count exceeds " + std::to_string(TTS_NORM_MAX_REPEAT));
}
p++;
}
return v;
}
void parse_quantifier(tts_norm_item & item) {
if (eof()) {
return;
}
const uint32_t c = peek();
if (c != '?' && c != '*' && c != '+' && c != '{') {
return;
}
if (tts_norm_is_zero_width(item.atom.op)) {
fail("zero-width assertion cannot be repeated");
}
p++;
if (c == '?') {
item.min = 0; item.max = 1;
} else if (c == '*') {
item.min = 0; item.max = TTS_NORM_INF;
} else if (c == '+') {
item.min = 1; item.max = TTS_NORM_INF;
} else {
item.min = parse_int();
item.max = item.min;
if (peek_is(',')) {
p++;
item.max = peek_is('}') ? TTS_NORM_INF : parse_int();
}
expect('}', "'}' to close quantifier");
if (item.max < item.min) {
fail("quantifier max is smaller than min");
}
}
if (!eof() && (peek() == '?' || peek() == '*' || peek() == '+' || peek() == '{')) {
fail("multiple quantifiers");
}
}
tts_norm_pattern_node parse_group() {
// '(' already consumed
tts_norm_pattern_node node;
node.op = tts_norm_op::GROUP;
if (peek_is('?')) {
p++;
if (eof()) {
fail("unterminated group");
}
const uint32_t c = peek();
if (c == ':') {
p++;
} else if (c == '=' || c == '!') {
p++;
node.op = tts_norm_op::LOOKAHEAD;
node.neg = c == '!';
} else if (c == '<' && p + 1 < s.size() && (s[p + 1] == '=' || s[p + 1] == '!')) {
node.op = tts_norm_op::LOOKBEHIND;
node.neg = s[p + 1] == '!';
p += 2;
tts_norm_item item;
if (eof() || peek() == ')') {
fail("lookbehind must contain exactly one codepoint matcher");
}
item.atom = parse_atom();
if (item.atom.op != tts_norm_op::LITERAL && item.atom.op != tts_norm_op::SET && item.atom.op != tts_norm_op::ANY) {
fail("lookbehind must contain exactly one codepoint matcher");
}
if (!peek_is(')')) {
fail("lookbehind must contain exactly one codepoint matcher");
}
p++;
node.alts.push_back(tts_norm_seq{});
node.alts[0].push_back(std::move(item));
return node;
} else if (c == 'i') {
fail("(?i) is only allowed at the start of the pattern");
} else {
fail("unknown group type");
}
} else {
node.capture = static_cast<int>(n_caps++);
}
node.alts = parse_alts();
expect(')', "')' to close group");
return node;
}
// after '%' inside or outside of a set
// returns true if it is a class (stored in cls), false if it is a literal (stored in lit)
bool parse_escape(bool in_set, tts_norm_class & cls, uint32_t & lit, bool & boundary) {
boundary = false;
if (eof()) {
fail("dangling '%'");
}
const uint32_t e = s[p++];
if (tts_norm_is_class_letter(e)) {
cls.kind = static_cast<char>(tts_norm_ascii_lower(e));
cls.neg = e >= 'A' && e <= 'Z';
return true;
}
if (e == 'b' && !in_set) {
boundary = true;
return false;
}
if (tts_norm_is_ascii_alnum(e)) {
p--;
fail(std::string("unknown escape '%") + static_cast<char>(e) + "'");
}
lit = e;
return false;
}
tts_norm_pattern_node parse_set() {
// '[' already consumed
tts_norm_pattern_node node;
node.op = tts_norm_op::SET;
if (peek_is('^')) {
node.set.neg = true;
p++;
}
bool empty = true;
while (true) {
if (eof()) {
fail("unterminated set");
}
if (peek() == ']') {
p++;
break;
}
empty = false;
uint32_t lo = 0;
if (peek() == '%') {
p++;
tts_norm_class cls{};
bool boundary = false;
if (parse_escape(true, cls, lo, boundary)) {
node.set.classes.push_back(cls);
continue;
}
} else {
lo = s[p++];
}
uint32_t hi = lo;
if (peek_is('-') && p + 1 < s.size() && s[p + 1] != ']') {
p++;
if (peek() == '%') {
p++;
tts_norm_class cls{};
bool boundary = false;
if (parse_escape(true, cls, hi, boundary)) {
fail("character class cannot be used in a range");
}
} else {
hi = s[p++];
}
if (hi < lo) {
fail("invalid range in set");
}
}
node.set.ranges.emplace_back(lo, hi);
}
if (empty) {
fail("empty set");
}
return node;
}
tts_norm_pattern_node parse_atom() {
const uint32_t c = s[p++];
tts_norm_pattern_node node;
switch (c) {
case '(':
return parse_group();
case '[':
return parse_set();
case '.':
node.op = tts_norm_op::ANY;
return node;
case '^':
node.op = tts_norm_op::START;
return node;
case '$':
node.op = tts_norm_op::END;
return node;
case '%': {
tts_norm_class cls{};
uint32_t lit = 0;
bool boundary = false;
if (parse_escape(false, cls, lit, boundary)) {
node.op = tts_norm_op::SET;
node.set.classes.push_back(cls);
} else if (boundary) {
node.op = tts_norm_op::BOUNDARY;
} else {
node.op = tts_norm_op::LITERAL;
node.cp = lit;
}
return node;
}
case '?':
case '*':
case '+':
case '{':
p--;
fail("nothing to repeat");
case ']':
case '}':
p--;
fail(std::string("unescaped '") + static_cast<char>(c) + "', use '%" + static_cast<char>(c) + "'");
default:
node.op = tts_norm_op::LITERAL;
node.cp = c;
return node;
}
}
};
//
// matcher
// PEG-style: alternatives are ordered (the first one that matches wins) and quantifiers are greedy and possessive
// (they never give back what they consumed), so there is no backtracking
//
struct tts_norm_matcher {
const std::vector<uint32_t> & t;
bool icase;
std::vector<std::pair<int64_t, int64_t>> caps;
bool is_w(size_t i) const {
return i < t.size() && tts_norm_class_match('w', t[i]);
}
bool set_match(const tts_norm_set & st, uint32_t c) const {
auto test = [&](uint32_t x) {
for (const auto & r : st.ranges) {
if (x >= r.first && x <= r.second) {
return true;
}
}
for (const auto & k : st.classes) {
if (tts_norm_class_match(k.kind, x) != k.neg) {
return true;
}
}
return false;
};
bool m = test(c);
if (!m && icase) {
const uint32_t o = tts_norm_ascii_swapcase(c);
if (o != c) {
m = test(o);
}
}
return m != st.neg;
}
bool match_alts(const std::vector<tts_norm_seq> & alts, size_t i, size_t & end) {
for (const auto & seq : alts) {
auto saved = caps;
if (match_seq(seq, i, end)) {
return true;
}
caps = std::move(saved);
}
return false;
}
bool match_seq(const tts_norm_seq & seq, size_t i, size_t & end) {
size_t pos = i;
for (const auto & it : seq) {
int count = 0;
while (count < it.max) {
const bool is_group = it.atom.op == tts_norm_op::GROUP;
std::vector<std::pair<int64_t, int64_t>> saved;
if (is_group) {
saved = caps;
}
size_t e = pos;
if (!match_once(it.atom, pos, e)) {
if (is_group) {
caps = std::move(saved);
}
break;
}
if (e == pos) {
// an empty iteration would repeat forever, it satisfies any remaining minimum
count = std::max(count + 1, it.min);
break;
}
pos = e;
count++;
}
if (count < it.min) {
return false;
}
}
end = pos;
return true;
}
bool match_once(const tts_norm_pattern_node & node, size_t i, size_t & end) {
const size_t n = t.size();
switch (node.op) {
case tts_norm_op::LITERAL:
if (i < n && (t[i] == node.cp || (icase && tts_norm_ascii_lower(t[i]) == tts_norm_ascii_lower(node.cp)))) {
end = i + 1;
return true;
}
return false;
case tts_norm_op::SET:
if (i < n && set_match(node.set, t[i])) {
end = i + 1;
return true;
}
return false;
case tts_norm_op::ANY:
if (i < n) {
end = i + 1;
return true;
}
return false;
case tts_norm_op::START:
end = i;
return i == 0;
case tts_norm_op::END:
end = i;
return i == n;
case tts_norm_op::BOUNDARY:
end = i;
return (i > 0 && is_w(i - 1)) != is_w(i);
case tts_norm_op::GROUP: {
size_t e = i;
if (!match_alts(node.alts, i, e)) {
return false;
}
if (node.capture >= 0) {
caps[node.capture] = {static_cast<int64_t>(i), static_cast<int64_t>(e)};
}
end = e;
return true;
}
case tts_norm_op::LOOKAHEAD: {
// captures made inside a lookahead are discarded
auto saved = caps;
size_t e = i;
const bool ok = match_alts(node.alts, i, e);
caps = std::move(saved);
end = i;
return ok != node.neg;
}
case tts_norm_op::LOOKBEHIND: {
size_t e = 0;
const bool ok = i > 0 && match_once(node.alts[0][0].atom, i - 1, e) && e == i;
end = i;
return ok != node.neg;
}
}
return false;
}
};
//
// common_tts_norm_pattern
//
common_tts_norm_pattern::common_tts_norm_pattern(const std::string & src) {
if (src.size() > TTS_NORM_MAX_PATTERN_LEN) {
throw std::runtime_error("invalid TTS norm pattern: pattern is longer than " + std::to_string(TTS_NORM_MAX_PATTERN_LEN) + " bytes");
}
std::vector<uint32_t> cpts;
std::vector<size_t> offs;
tts_norm_decode_utf8(src, cpts, offs);
tts_norm_parser parser(std::move(cpts), n_caps);
static const std::vector<uint32_t> icase_prefix = {'(', '?', 'i', ')'};
if (parser.s.size() >= 4 && std::equal(icase_prefix.begin(), icase_prefix.end(), parser.s.begin())) {
icase = true;
parser.p = 4;
}
root = std::make_unique<tts_norm_pattern_node>();
root->op = tts_norm_op::GROUP;
root->alts = parser.parse_alts();
if (!parser.eof()) {
parser.fail("unbalanced ')'");
}
}
common_tts_norm_pattern::~common_tts_norm_pattern() = default;
common_tts_norm_pattern::common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept = default;
common_tts_norm_pattern & common_tts_norm_pattern::operator=(common_tts_norm_pattern &&) noexcept = default;
bool common_tts_norm_pattern::match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const {
tts_norm_matcher m{cpts, icase, std::vector<std::pair<int64_t, int64_t>>(n_caps, {-1, -1})};
size_t e = pos;
if (!m.match_alts(root->alts, pos, e)) {
return false;
}
end = e;
groups = std::move(m.caps);
return true;
}
size_t common_tts_norm_pattern::n_groups() const {
return n_caps;
}
//
// scan / sub
//
std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules) {
std::vector<uint32_t> cpts;
std::vector<size_t> offs;
tts_norm_decode_utf8(text, cpts, offs);
std::vector<common_tts_norm_segment> out;
const size_t n = cpts.size();
size_t i = 0;
size_t pending = 0; // start of the current unmatched run
std::vector<std::pair<int64_t, int64_t>> groups;
while (i < n) {
bool matched = false;
for (size_t r = 0; r < rules.size(); r++) {
size_t end = i;
if (rules[r]->match_at(cpts, i, end, groups) && end > i) {
if (pending < i) {
out.push_back({-1, offs[pending], offs[i], {}});
}
common_tts_norm_segment seg;
seg.rule = static_cast<int>(r);
seg.start = offs[i];
seg.end = offs[end];
for (const auto & g : groups) {
if (g.first < 0) {
seg.groups.emplace_back(std::string::npos, std::string::npos);
} else {
seg.groups.emplace_back(offs[g.first], offs[g.second]);
}
}
out.push_back(std::move(seg));
i = end;
pending = i;
matched = true;
break;
}
}
if (!matched) {
i++;
}
}
if (pending < n) {
out.push_back({-1, offs[pending], offs[n], {}});
}
return out;
}
struct tts_norm_repl_piece {
std::string literal;
int group = -1; // 0 = whole match, 1.. = groups
};
static std::vector<tts_norm_repl_piece> tts_norm_parse_repl(const std::string & repl, const common_tts_norm_pattern & pattern) {
std::vector<tts_norm_repl_piece> pieces;
std::string lit;
auto flush = [&]() {
if (!lit.empty()) {
pieces.push_back({lit, -1});
lit.clear();
}
};
for (size_t i = 0; i < repl.size(); i++) {
if (repl[i] != '%') {
lit += repl[i];
continue;
}
if (i + 1 >= repl.size()) {
throw std::runtime_error("invalid TTS norm replacement: dangling '%'");
}
const char c = repl[++i];
if (c == '%') {
lit += '%';
} else if (c >= '0' && c <= '9') {
const int g = c - '0';
if (static_cast<size_t>(g) > pattern.n_groups()) {
throw std::runtime_error("invalid TTS norm replacement: group %" + std::string(1, c) + " does not exist");
}
flush();
pieces.push_back({"", g});
} else {
throw std::runtime_error("invalid TTS norm replacement: unknown escape '%" + std::string(1, c) + "'");
}
}
flush();
return pieces;
}
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl) {
const auto pieces = tts_norm_parse_repl(repl, pattern);
std::string out;
for (const auto & seg : common_tts_norm_scan(text, {&pattern})) {
if (seg.rule < 0) {
out.append(text, seg.start, seg.end - seg.start);
continue;
}
for (const auto & piece : pieces) {
if (piece.group < 0) {
out += piece.literal;
} else if (piece.group == 0) {
out.append(text, seg.start, seg.end - seg.start);
} else {
const auto & g = seg.groups[piece.group - 1];
if (g.first != std::string::npos) {
out.append(text, g.first, g.second - g.first);
}
}
}
}
return out;
}
//
// template builtins
//
// rules can be: a pattern string (rule name "match"), a dict {name: pattern}, or a list of [name, pattern] pairs
static std::vector<std::pair<std::string, std::string>> tts_norm_rules_from_value(const value & v) {
std::vector<std::pair<std::string, std::string>> rules;
if (is_val<value_string>(v)) {
rules.emplace_back("match", v->as_string().str());
} else if (is_val<value_object>(v)) {
for (const auto & kv : v->as_ordered_object()) {
if (!is_val<value_string>(kv.second)) {
throw std::runtime_error("tts_norm_scan: rule pattern must be a string");
}
rules.emplace_back(kv.first->as_string().str(), kv.second->as_string().str());
}
} else if (is_val<value_array>(v)) {
for (const auto & item : v->as_array()) {
if (!is_val<value_array>(item) || item->as_array().size() != 2
|| !is_val<value_string>(item->as_array()[0]) || !is_val<value_string>(item->as_array()[1])) {
throw std::runtime_error("tts_norm_scan: each rule must be a [name, pattern] pair of strings");
}
rules.emplace_back(item->as_array()[0]->as_string().str(), item->as_array()[1]->as_string().str());
}
} else {
throw std::runtime_error("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs, got " + v->type());
}
return rules;
}
static value tts_norm_builtin_scan(const func_args & args) {
args.ensure_count(2, 2);
args.ensure_val<value_string>(args.get_pos(0));
const string text_val = args.get_pos(0)->as_string();
const std::string text = text_val.str();
const bool is_input = text_val.all_parts_are_input();
const auto rule_srcs = tts_norm_rules_from_value(args.get_pos(1));
std::vector<common_tts_norm_pattern> patterns;
patterns.reserve(rule_srcs.size());
std::vector<const common_tts_norm_pattern *> ptrs;
for (const auto & r : rule_srcs) {
patterns.emplace_back(r.second);
}
for (const auto & p : patterns) {
ptrs.push_back(&p);
}
auto mk_str = [&](size_t start, size_t end) -> value {
return mk_val<value_string>(string(text.substr(start, end - start), is_input));
};
auto out = mk_val<value_array>();
for (const auto & seg : common_tts_norm_scan(text, ptrs)) {
auto groups = mk_val<value_array>();
for (const auto & g : seg.groups) {
// groups that did not participate are empty strings, so they render as nothing and are falsy
groups->push_back(g.first == std::string::npos ? mk_str(0, 0) : mk_str(g.first, g.second));
}
auto obj = mk_val<value_object>();
obj->insert("rule", seg.rule < 0 ? value(mk_val<value_none>()) : value(mk_val<value_string>(rule_srcs[seg.rule].first)));
obj->insert("text", mk_str(seg.start, seg.end));
obj->insert("groups", groups);
out->push_back(obj);
}
return out;
}
static value tts_norm_builtin_sub(const func_args & args) {
args.ensure_count(3, 3);
args.ensure_vals<value_string, value_string, value_string>();
const string text_val = args.get_pos(0)->as_string();
const common_tts_norm_pattern pattern(args.get_pos(1)->as_string().str());
const std::string res = common_tts_norm_sub(text_val.str(), pattern, args.get_pos(2)->as_string().str());
return mk_val<value_string>(string(res, text_val.all_parts_are_input()));
}
void common_chat_tts_norm_add_builtins(context & ctx) {
ctx.set_val("tts_norm_scan", mk_val<value_func>("tts_norm_scan", tts_norm_builtin_scan));
ctx.set_val("tts_norm_sub", mk_val<value_func>("tts_norm_sub", tts_norm_builtin_sub));
}
+62
View File
@@ -0,0 +1,62 @@
#pragma once
#include <cstdint>
#include <memory>
#include <string>
#include <utility>
#include <vector>
// TTS normalization patterns: a small, fully specified matching language for TTS text normalization templates
//
// it is NOT a regex engine: the semantics are defined by llama.cpp (see docs/tts-norm-patterns.md), so that this
// implementation and the python reference (scripts/chat_tts_norm.py) give the same result for every input
//
// two functions are exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():
// tts_norm_scan(text, rules) -> list of segments {rule, text, groups}
// tts_norm_sub(text, pattern, repl) -> string
namespace jinja {
struct context;
}
struct tts_norm_pattern_node;
class common_tts_norm_pattern {
public:
// throws std::runtime_error on invalid pattern
explicit common_tts_norm_pattern(const std::string & src);
~common_tts_norm_pattern();
common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept;
common_tts_norm_pattern & operator=(common_tts_norm_pattern &&) noexcept;
// match anchored at codepoint index pos; on success, end is the codepoint index after the match
// groups[i] holds [start, end) codepoint indices of capture i+1, or {-1, -1} if the group did not participate
bool match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const;
size_t n_groups() const;
private:
std::unique_ptr<tts_norm_pattern_node> root;
size_t n_caps = 0;
bool icase = false;
};
// segment produced by common_tts_norm_scan(); byte offsets into the input text
struct common_tts_norm_segment {
int rule = -1; // index of the matched rule, -1 for unmatched text
size_t start = 0;
size_t end = 0;
std::vector<std::pair<size_t, size_t>> groups; // byte offsets; {npos, npos} for groups that did not participate
};
// split text into matched and unmatched segments
// at each position, rules are tried in order and the first non-empty match wins; otherwise one codepoint is skipped
std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules);
// replace every non-overlapping match of pattern with repl
// repl references: %0 (whole match), %1..%9 (groups), %% (literal percent)
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl);
// add tts_norm_scan() and tts_norm_sub() to the context of a task template
// they are not jinja global builtins on purpose: chat templates are not affected
void common_chat_tts_norm_add_builtins(jinja::context & ctx);
+3
View File
@@ -3,6 +3,7 @@
#include "chat-auto-parser-helpers.h"
#include "chat-auto-parser.h"
#include "chat-peg-parser.h"
#include "chat-tts-norm.h"
#include "common.h"
#include "ggml.h"
#include "json-schema-to-grammar.h"
@@ -807,6 +808,8 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
// renders a task template (e.g. "tts") with the given variables
static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) {
jinja::context ctx(tmpl.source());
// text normalization helpers: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md
common_chat_tts_norm_add_builtins(ctx);
jinja::global_from_json(ctx, vars, false);
ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
+125
View File
@@ -0,0 +1,125 @@
# TTS normalization patterns
Text-to-speech models often need their input normalized before tokenization: numbers, currencies, dates and symbols spelled out, punctuation mapped, spacing fixed. The exact rules are model-specific (they must match what the model was trained on), so they live in the model's `tts` task template rather than in C++.
Jinja has no regex support, and exposing `std::regex` / Python `re` would not help: the two differ in many details (`\d`, `\w`, `\b`, lookbehind, case folding, backtracking limits), so a template could give different results in llama.cpp and in Python. Instead, task templates get two functions with a small matching language whose semantics are defined here:
- `tts_norm_scan(text, rules)`: split text into matched and unmatched segments
- `tts_norm_sub(text, pattern, repl)`: replace every match
Implementations:
- C++: [common/chat-tts-norm.cpp](../common/chat-tts-norm.cpp), injected into task templates by `common_chat_template_render()` in [common/chat.cpp](../common/chat.cpp). They are not available to chat templates.
- Python reference: [scripts/chat_tts_norm.py](../scripts/chat_tts_norm.py), for use with `jinja2` (`env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan`)
- Both are checked against each other by `tests/test-jinja.cpp` (`test-jinja -py "tts norm patterns.*"`)
## Example
```jinja
{%- for s in tts_norm_scan(text, [
['money', '%$(%d+)(?:%.(%d%d))?'],
['percent', '(%d+)%%'],
['time', '(%d%d?):(%d%d)(?!%d)'],
['neg', '(?<!%w)-(%d+)'],
['number', '%d+'],
]) -%}
{%- if s.rule == 'money' -%}{%- set whole, cents = s.groups -%}{{ num(whole) }} dollars{{ ' and ' ~ num(cents) ~ ' cents' if cents else '' }}
{%- elif s.rule == 'percent' -%}{{ num(s.groups[0]) }} percent
{%- elif s.rule == 'time' -%}{%- set h, m = s.groups -%}{{ num(h) }} {{ num(m) }}
{%- elif s.rule == 'neg' -%}minus {{ num(s.groups[0]) }}
{%- elif s.rule == 'number' -%}{{ num(s.text) }}
{%- else -%}{{ s.text }}
{%- endif -%}
{%- endfor -%}
```
`It costs $3.50, 20% off, at 10:45, -5 degrees, 7 cats.` becomes `It costs three dollars and fifty cents, twenty percent off, at ten forty-five, minus five degrees, seven cats.` (with a `num` macro that spells numbers, see the "mini text normalizer" test in `tests/test-jinja.cpp`).
## Functions
### `tts_norm_scan(text, rules)`
`rules` is one of:
- a list of `[name, pattern]` pairs (recommended, the order is explicit)
- a dict `{name: pattern}`, in insertion order
- a single pattern string, the rule is named `match`
Scanning starts at the first codepoint. At each position, the rules are tried in order and the first one that produces a **non-empty** match wins; the scan continues after the match. If no rule matches, one codepoint is skipped. Matches never overlap.
The result is a list of segments covering the whole text, in order. Consecutive unmatched codepoints are merged into one segment. Each segment is a dict:
| key | matched segment | unmatched segment |
|---|---|---|
| `rule` | rule name | `none` |
| `text` | matched text | unmatched text |
| `groups` | list of capture groups, in order of their `(` | `[]` |
There are no named groups, unpack `groups` instead: `{% set whole, cents = s.groups %}`.
A group that did not participate in the match (e.g. inside a `?` that matched nothing) is an empty string, so it renders as nothing and is falsy.
### `tts_norm_sub(text, pattern, repl)`
Replaces every match of `pattern`, with the same scanning rules as `tts_norm_scan`. `repl` is a literal string with references:
| reference | value |
|---|---|
| `%0` | whole match |
| `%1` .. `%9` | group |
| `%%` | a literal `%` |
Any other `%` sequence in `repl`, or a reference to a group that does not exist, is an error.
## Pattern syntax
Patterns operate on Unicode codepoints. Invalid UTF-8 bytes (C++ only) are seen as U+FFFD, one per byte, and preserved in the output.
| syntax | meaning |
|---|---|
| `x` | the literal codepoint `x` |
| `%x` | escape: `%` followed by any non-alphanumeric codepoint is that literal codepoint (`%.`, `%$`, `%%`, `%(`, ...) |
| `.` | any codepoint, including newlines |
| `%d` `%s` `%p` `%a` `%w` `%u` `%l` `%h` `%c` | character classes, see below; the uppercase form (`%D`, ...) is the negation |
| `[...]` | set: literal codepoints, ranges `a-z`, classes `%d`; `[^...]` is the negation |
| `(...)` | capture group |
| `(?:...)` | group without capture |
| `a\|b` | ordered alternation |
| `?` `*` `+` `{m}` `{m,}` `{m,n}` | quantifiers, greedy and possessive; `m`, `n` up to 1000 |
| `^` `$` | start / end of the whole text |
| `%b` | boundary: between a `%w` codepoint and a non-`%w` codepoint (the edges of the text count as non-`%w`) |
| `(?=...)` `(?!...)` | lookahead, positive / negative |
| `(?<=c)` `(?<!c)` | lookbehind of exactly one codepoint: `c` is a literal, a set, a class or `.` |
| `(?i)` | at the very start of the pattern only: ignore ASCII case |
These characters must be escaped with `%` to be matched literally: `( ) [ ] { } . | ? * + ^ $ %`. Inside a set, only `]`, `%`, `-` (between two codepoints) and `^` (first position) are special.
### Matching semantics
The matcher is a PEG, not a backtracking regex:
- **Ordered alternation**: alternatives are tried left to right and the first one that matches is committed to. If what follows fails, the other alternatives are not retried. `(?:a|ab)c` never matches `abc`; write `(?:ab|a)c`.
- **Possessive quantifiers**: quantifiers match as many repetitions as possible and never give any back. `%d+0` never matches; `.*x` never matches. Write patterns that describe exactly what to consume.
- A repetition that matches the empty string ends the loop.
- Captures inside lookaheads are discarded. A capture inside a repeated group holds its last repetition.
This gives a simple and linear-time implementation with identical results in C++ and Python.
### Character classes
Classes are defined by explicit codepoint ranges, not by a Unicode database, so the result does not depend on the Unicode version of the platform.
| class | definition |
|---|---|
| `%d` | ASCII digits `0-9` only |
| `%s` | U+0009..U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000..U+200A, U+2028, U+2029, U+202F, U+205F, U+3000 |
| `%p` | punctuation and symbols: ASCII punctuation, U+00A1..U+00BF (except U+00AA, U+00B5, U+00BA), U+00D7, U+00F7, U+2010..U+2027, U+2030..U+205E, U+20A0..U+20CF (currency), U+2100..U+2BFF (letterlike, number forms, arrows, math, technical, box drawing, shapes, misc symbols, dingbats), CJK punctuation (U+3001..U+3003, U+3008..U+3020, U+3030, U+303D, U+30FB), U+FE10..U+FE19, U+FE30..U+FE6B, fullwidth punctuation (U+FF01..U+FF0F, U+FF1A..U+FF20, U+FF3B..U+FF40, U+FF5B..U+FF65), U+1F000..U+1FAFF (emoji) |
| `%c` | control and invisible format characters: U+0000..U+001F, U+007F..U+009F, U+200B..U+200F, U+2060..U+206F, U+FEFF |
| `%a` | ASCII letters, U+00AA, U+00B5, U+00BA, and every codepoint from U+00C0 that is not in `%s`, `%p` or `%c` |
| `%w` | `%a` or `%d` |
| `%u` / `%l` | ASCII uppercase / lowercase letters only |
| `%h` | Han: U+2E80..U+2FDF, U+3005, U+3007, U+3021..U+3029, U+3038..U+303B, U+3400..U+4DBF, U+4E00..U+9FFF, U+F900..U+FAFF, U+20000..U+323AF |
Classes may overlap (e.g. U+0085 is both `%s` and `%c`). `%a` intentionally includes all scripts, combining marks and CJK, so `(?<!%w)` and `%b` work as "not inside a word" for any language.
## Errors
Invalid patterns are rejected when the function is called, with a message and a position, for example: unbalanced parentheses, a quantifier with nothing to repeat, multiple quantifiers (`a+?`), repeating a zero-width assertion (`%b+`), unknown escapes (`%q`), unescaped `]` or `}`, an empty set, a lookbehind wider than one codepoint and `(?i)` not at the start.
+556
View File
@@ -0,0 +1,556 @@
#!/usr/bin/env python3
"""Reference implementation of TTS normalization patterns, the tts_norm_scan() and tts_norm_sub() template functions.
The specification is in docs/tts-norm-patterns.md. This file must give the same result as the C++ implementation
(common/chat-tts-norm.cpp) for every input; tests/test-jinja.cpp checks this in python mode (test-jinja -py).
Usage with jinja2:
env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan
env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
# character classes, explicit codepoint ranges (keep in sync with common/chat-tts-norm.cpp)
TTS_NORM_SPACE = [
(0x0009, 0x000D), (0x0020, 0x0020), (0x0085, 0x0085), (0x00A0, 0x00A0), (0x1680, 0x1680),
(0x2000, 0x200A), (0x2028, 0x2029), (0x202F, 0x202F), (0x205F, 0x205F), (0x3000, 0x3000),
]
TTS_NORM_PUNCT = [
(0x0021, 0x002F), (0x003A, 0x0040), (0x005B, 0x0060), (0x007B, 0x007E),
(0x00A1, 0x00A9), (0x00AB, 0x00B4), (0x00B6, 0x00B9), (0x00BB, 0x00BF), (0x00D7, 0x00D7), (0x00F7, 0x00F7),
(0x2010, 0x2027), (0x2030, 0x205E), (0x20A0, 0x20CF), (0x2100, 0x218F), (0x2190, 0x2BFF),
(0x3001, 0x3003), (0x3008, 0x3020), (0x3030, 0x3030), (0x303D, 0x303D), (0x30FB, 0x30FB),
(0xFE10, 0xFE19), (0xFE30, 0xFE4F), (0xFE50, 0xFE6B),
(0xFF01, 0xFF0F), (0xFF1A, 0xFF20), (0xFF3B, 0xFF40), (0xFF5B, 0xFF65),
(0x1F000, 0x1FAFF),
]
TTS_NORM_CONTROL = [
(0x0000, 0x001F), (0x007F, 0x009F), (0x200B, 0x200F), (0x2060, 0x206F), (0xFEFF, 0xFEFF),
]
TTS_NORM_HAN = [
(0x2E80, 0x2FDF), (0x3005, 0x3005), (0x3007, 0x3007), (0x3021, 0x3029), (0x3038, 0x303B),
(0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), (0x20000, 0x323AF),
]
CLASS_LETTERS = "dspawulhc"
INF = 1 << 31
MAX_REPEAT = 1000
MAX_PATTERN_LEN = 4096
def _in(ranges: list[tuple[int, int]], c: int) -> bool:
return any(lo <= c <= hi for lo, hi in ranges)
def _is_alpha(c: int) -> bool:
if ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z"):
return True
if c in (0xAA, 0xB5, 0xBA):
return True
return c >= 0xC0 and not _in(TTS_NORM_SPACE, c) and not _in(TTS_NORM_PUNCT, c) and not _in(TTS_NORM_CONTROL, c)
def _class_match(kind: str, c: int) -> bool:
if kind == "d":
return ord("0") <= c <= ord("9")
if kind == "s":
return _in(TTS_NORM_SPACE, c)
if kind == "p":
return _in(TTS_NORM_PUNCT, c)
if kind == "a":
return _is_alpha(c)
if kind == "w":
return _is_alpha(c) or ord("0") <= c <= ord("9")
if kind == "u":
return ord("A") <= c <= ord("Z")
if kind == "l":
return ord("a") <= c <= ord("z")
if kind == "h":
return _in(TTS_NORM_HAN, c)
if kind == "c":
return _in(TTS_NORM_CONTROL, c)
return False
def _ascii_lower(c: int) -> int:
return c + 32 if ord("A") <= c <= ord("Z") else c
def _ascii_swapcase(c: int) -> int:
if ord("A") <= c <= ord("Z"):
return c + 32
if ord("a") <= c <= ord("z"):
return c - 32
return c
def _is_ascii_alnum(c: int) -> bool:
return ord("0") <= c <= ord("9") or ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z")
# AST
LITERAL, SET, ANY, START, END, BOUNDARY, GROUP, LOOKAHEAD, LOOKBEHIND = range(9)
ZERO_WIDTH = (START, END, BOUNDARY, LOOKAHEAD, LOOKBEHIND)
@dataclass
class _Set:
neg: bool = False
ranges: list[tuple[int, int]] = field(default_factory=list)
classes: list[tuple[str, bool]] = field(default_factory=list)
@dataclass
class _Node:
op: int = GROUP
cp: int = 0
set: _Set = field(default_factory=_Set)
alts: list[list[_Item]] = field(default_factory=list)
capture: int = -1
neg: bool = False
@dataclass
class _Item:
atom: _Node
min: int = 1
max: int = 1
class PatternError(ValueError):
pass
class _Parser:
def __init__(self, s: list[int]):
self.s = s
self.p = 0
self.n_caps = 0
def fail(self, msg: str):
raise PatternError(f"invalid TTS norm pattern: {msg} at position {self.p}")
def eof(self) -> bool:
return self.p >= len(self.s)
def peek_is(self, c: str) -> bool:
return not self.eof() and self.s[self.p] == ord(c)
def expect(self, c: str, what: str):
if not self.peek_is(c):
self.fail(f"expected {what}")
self.p += 1
def parse_alts(self) -> list[list[_Item]]:
alts = [self.parse_seq()]
while self.peek_is("|"):
self.p += 1
alts.append(self.parse_seq())
return alts
def parse_seq(self) -> list[_Item]:
seq: list[_Item] = []
while not self.eof() and self.s[self.p] not in (ord("|"), ord(")")):
item = _Item(self.parse_atom())
self.parse_quantifier(item)
seq.append(item)
return seq
def parse_int(self) -> int:
if self.eof() or not (ord("0") <= self.s[self.p] <= ord("9")):
self.fail("expected number in quantifier")
v = 0
while not self.eof() and ord("0") <= self.s[self.p] <= ord("9"):
v = v * 10 + self.s[self.p] - ord("0")
if v > MAX_REPEAT:
self.fail(f"repetition count exceeds {MAX_REPEAT}")
self.p += 1
return v
def parse_quantifier(self, item: _Item):
if self.eof():
return
c = chr(self.s[self.p])
if c not in "?*+{":
return
if item.atom.op in ZERO_WIDTH:
self.fail("zero-width assertion cannot be repeated")
self.p += 1
if c == "?":
item.min, item.max = 0, 1
elif c == "*":
item.min, item.max = 0, INF
elif c == "+":
item.min, item.max = 1, INF
else:
item.min = self.parse_int()
item.max = item.min
if self.peek_is(","):
self.p += 1
item.max = INF if self.peek_is("}") else self.parse_int()
self.expect("}", "'}' to close quantifier")
if item.max < item.min:
self.fail("quantifier max is smaller than min")
if not self.eof() and chr(self.s[self.p]) in "?*+{":
self.fail("multiple quantifiers")
def parse_group(self) -> _Node:
node = _Node(op=GROUP)
if self.peek_is("?"):
self.p += 1
if self.eof():
self.fail("unterminated group")
c = chr(self.s[self.p])
if c == ":":
self.p += 1
elif c in "=!":
self.p += 1
node.op = LOOKAHEAD
node.neg = c == "!"
elif c == "<" and self.p + 1 < len(self.s) and self.s[self.p + 1] in (ord("="), ord("!")):
node.op = LOOKBEHIND
node.neg = self.s[self.p + 1] == ord("!")
self.p += 2
if self.eof() or self.s[self.p] == ord(")"):
self.fail("lookbehind must contain exactly one codepoint matcher")
atom = self.parse_atom()
if atom.op not in (LITERAL, SET, ANY):
self.fail("lookbehind must contain exactly one codepoint matcher")
if not self.peek_is(")"):
self.fail("lookbehind must contain exactly one codepoint matcher")
self.p += 1
node.alts = [[_Item(atom)]]
return node
elif c == "i":
self.fail("(?i) is only allowed at the start of the pattern")
else:
self.fail("unknown group type")
else:
node.capture = self.n_caps
self.n_caps += 1
node.alts = self.parse_alts()
self.expect(")", "')' to close group")
return node
# returns ("class", (kind, neg)) | ("boundary", None) | ("literal", cp)
def parse_escape(self, in_set: bool) -> tuple[str, Any]:
if self.eof():
self.fail("dangling '%'")
e = self.s[self.p]
self.p += 1
if e < 0x80 and chr(e).lower() in CLASS_LETTERS:
return "class", (chr(e).lower(), ord("A") <= e <= ord("Z"))
if e == ord("b") and not in_set:
return "boundary", None
if _is_ascii_alnum(e):
self.p -= 1
self.fail(f"unknown escape '%{chr(e)}'")
return "literal", e
def parse_set(self) -> _Node:
node = _Node(op=SET)
if self.peek_is("^"):
node.set.neg = True
self.p += 1
empty = True
while True:
if self.eof():
self.fail("unterminated set")
if self.s[self.p] == ord("]"):
self.p += 1
break
empty = False
if self.s[self.p] == ord("%"):
self.p += 1
kind, val = self.parse_escape(True)
if kind == "class":
node.set.classes.append(val)
continue
lo = val
else:
lo = self.s[self.p]
self.p += 1
hi = lo
if self.peek_is("-") and self.p + 1 < len(self.s) and self.s[self.p + 1] != ord("]"):
self.p += 1
if self.s[self.p] == ord("%"):
self.p += 1
kind, val = self.parse_escape(True)
if kind == "class":
self.fail("character class cannot be used in a range")
hi = val
else:
hi = self.s[self.p]
self.p += 1
if hi < lo:
self.fail("invalid range in set")
node.set.ranges.append((lo, hi))
if empty:
self.fail("empty set")
return node
def parse_atom(self) -> _Node:
c = chr(self.s[self.p])
self.p += 1
if c == "(":
return self.parse_group()
if c == "[":
return self.parse_set()
if c == ".":
return _Node(op=ANY)
if c == "^":
return _Node(op=START)
if c == "$":
return _Node(op=END)
if c == "%":
kind, val = self.parse_escape(False)
if kind == "class":
node = _Node(op=SET)
node.set.classes.append(val)
return node
if kind == "boundary":
return _Node(op=BOUNDARY)
return _Node(op=LITERAL, cp=val)
if c in "?*+{":
self.p -= 1
self.fail("nothing to repeat")
if c in "]}":
self.p -= 1
self.fail(f"unescaped '{c}', use '%{c}'")
return _Node(op=LITERAL, cp=ord(c))
class _Matcher:
def __init__(self, t: list[int], icase: bool, n_groups: int):
self.t = t
self.icase = icase
self.caps: list[tuple[int, int]] = [(-1, -1)] * n_groups
def is_w(self, i: int) -> bool:
return i < len(self.t) and _class_match("w", self.t[i])
def set_match(self, st: _Set, c: int) -> bool:
def test(x: int) -> bool:
if any(lo <= x <= hi for lo, hi in st.ranges):
return True
return any(_class_match(kind, x) != neg for kind, neg in st.classes)
m = test(c)
if not m and self.icase:
o = _ascii_swapcase(c)
if o != c:
m = test(o)
return m != st.neg
def match_alts(self, alts: list[list[_Item]], i: int) -> int:
"""returns the end position, or -1 if no alternative matches"""
for seq in alts:
saved = list(self.caps)
end = self.match_seq(seq, i)
if end >= 0:
return end
self.caps = saved
return -1
def match_seq(self, seq: list[_Item], i: int) -> int:
pos = i
for it in seq:
count = 0
while count < it.max:
is_group = it.atom.op == GROUP
saved = list(self.caps) if is_group else None
e = self.match_once(it.atom, pos)
if e < 0:
if saved is not None:
self.caps = saved
break
if e == pos:
# an empty iteration would repeat forever, it satisfies any remaining minimum
count = max(count + 1, it.min)
break
pos = e
count += 1
if count < it.min:
return -1
return pos
def match_once(self, node: _Node, i: int) -> int:
"""returns the end position, or -1 on failure"""
n = len(self.t)
op = node.op
if op == LITERAL:
if i < n and (self.t[i] == node.cp or (self.icase and _ascii_lower(self.t[i]) == _ascii_lower(node.cp))):
return i + 1
return -1
if op == SET:
return i + 1 if i < n and self.set_match(node.set, self.t[i]) else -1
if op == ANY:
return i + 1 if i < n else -1
if op == START:
return i if i == 0 else -1
if op == END:
return i if i == n else -1
if op == BOUNDARY:
return i if (i > 0 and self.is_w(i - 1)) != self.is_w(i) else -1
if op == GROUP:
e = self.match_alts(node.alts, i)
if e < 0:
return -1
if node.capture >= 0:
self.caps[node.capture] = (i, e)
return e
if op == LOOKAHEAD:
# captures made inside a lookahead are discarded
saved = list(self.caps)
ok = self.match_alts(node.alts, i) >= 0
self.caps = saved
return i if ok != node.neg else -1
if op == LOOKBEHIND:
ok = i > 0 and self.match_once(node.alts[0][0].atom, i - 1) == i
return i if ok != node.neg else -1
return -1
class Pattern:
def __init__(self, src: str):
if len(src.encode("utf-8")) > MAX_PATTERN_LEN:
raise PatternError(f"invalid TTS norm pattern: pattern is longer than {MAX_PATTERN_LEN} bytes")
self.icase = False
parser = _Parser([ord(c) for c in src])
if src.startswith("(?i)"):
self.icase = True
parser.p = 4
self.root = _Node(op=GROUP, alts=parser.parse_alts())
if not parser.eof():
parser.fail("unbalanced ')'")
self.n_caps = parser.n_caps
def match_at(self, t: list[int], pos: int) -> tuple[int, list[tuple[int, int]]] | None:
m = _Matcher(t, self.icase, self.n_caps)
end = m.match_alts(self.root.alts, pos)
if end < 0:
return None
return end, m.caps
# (rule index or -1, start, end, groups) with codepoint offsets
Segment = tuple[int, int, int, list[tuple[int, int]]]
def scan_segments(text: str, rules: list[Pattern]) -> list[Segment]:
t = [ord(c) for c in text]
n = len(t)
out: list[Segment] = []
i = 0
pending = 0
while i < n:
matched = False
for r, rule in enumerate(rules):
res = rule.match_at(t, i)
if res is not None and res[0] > i:
end, groups = res
if pending < i:
out.append((-1, pending, i, []))
out.append((r, i, end, groups))
i = end
pending = i
matched = True
break
if not matched:
i += 1
if pending < n:
out.append((-1, pending, n, []))
return out
def _rules_from_value(rules: Any) -> list[tuple[str, str]]:
if isinstance(rules, str):
return [("match", rules)]
if isinstance(rules, dict):
out = []
for k, v in rules.items():
if not isinstance(v, str):
raise TypeError("tts_norm_scan: rule pattern must be a string")
out.append((str(k), v))
return out
if isinstance(rules, (list, tuple)):
out = []
for item in rules:
if not isinstance(item, (list, tuple)) or len(item) != 2 or not isinstance(item[0], str) or not isinstance(item[1], str):
raise TypeError("tts_norm_scan: each rule must be a [name, pattern] pair of strings")
out.append((item[0], item[1]))
return out
raise TypeError("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs")
def tts_norm_scan(text: str, rules: Any) -> list[dict[str, Any]]:
rule_srcs = _rules_from_value(rules)
patterns = [Pattern(src) for _, src in rule_srcs]
out: list[dict[str, Any]] = []
for r, start, end, groups in scan_segments(text, patterns):
# groups that did not participate are empty strings, so they render as nothing and are falsy
vals = ["" if gs < 0 else text[gs:ge] for gs, ge in groups]
out.append({"rule": rule_srcs[r][0] if r >= 0 else None, "text": text[start:end], "groups": vals})
return out
def _parse_repl(repl: str, pattern: Pattern) -> list[str | int]:
pieces: list[str | int] = []
lit = ""
i = 0
while i < len(repl):
c = repl[i]
if c != "%":
lit += c
i += 1
continue
if i + 1 >= len(repl):
raise PatternError("invalid TTS norm replacement: dangling '%'")
i += 1
c = repl[i]
if c == "%":
lit += "%"
elif "0" <= c <= "9":
g = ord(c) - ord("0")
if g > pattern.n_caps:
raise PatternError(f"invalid TTS norm replacement: group %{c} does not exist")
if lit:
pieces.append(lit)
lit = ""
pieces.append(g)
else:
raise PatternError(f"invalid TTS norm replacement: unknown escape '%{c}'")
i += 1
if lit:
pieces.append(lit)
return pieces
def tts_norm_sub(text: str, pattern: str, repl: str) -> str:
pat = Pattern(pattern)
pieces = _parse_repl(repl, pat)
out = []
for r, start, end, groups in scan_segments(text, [pat]):
if r < 0:
out.append(text[start:end])
continue
for piece in pieces:
if isinstance(piece, str):
out.append(piece)
elif piece == 0:
out.append(text[start:end])
else:
gs, ge = groups[piece - 1]
if gs >= 0:
out.append(text[gs:ge])
return "".join(out)
+157 -11
View File
@@ -3,6 +3,7 @@
#include <random>
#include <cstdlib>
#include "chat-tts-norm.h"
#include "json.h"
#include "subproc.h"
@@ -16,8 +17,8 @@
using json = common_json;
static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect);
static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect);
static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm = false);
static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm = false);
static void test_whitespace_control(testing & t);
static void test_conditionals(testing & t);
@@ -39,6 +40,7 @@ static void test_stats(testing & t);
static void test_caps(testing & t);
static void test_string_parts(testing & t);
static void test_fuzzing(testing & t);
static void test_tts_norm_patterns(testing & t);
static bool g_python_mode = false;
@@ -75,6 +77,7 @@ int main(int argc, char *argv[]) {
t.test("string methods", test_string_methods);
t.test("array methods", test_array_methods);
t.test("object methods", test_object_methods);
t.test("tts norm patterns", test_tts_norm_patterns);
if (!g_python_mode) {
t.test("hasher", test_hasher);
t.test("stats", test_stats);
@@ -2713,13 +2716,134 @@ static void test_string_parts(testing & t) {
}
static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect) {
t.test(name, [&tmpl, &expect](testing & t) {
// TTS normalization patterns: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md
// these functions are only available to task templates (e.g. "tts"), so every test here passes tts_norm = true
static void test_tts_norm_patterns(testing & t) {
test_template(t, "not available without tts norm builtins",
"{{ tts_norm_scan is defined }}",
json::object(),
"False");
test_template(t, "tts_norm_scan splits text into matched and unmatched segments",
"{%- for s in tts_norm_scan(text, [['num', '%d+'], ['word', '%a+']]) -%}[{{ s.rule or '-' }}:{{ s.text }}]{%- endfor -%}",
{{"text", "abc 123, x9"}},
"[word:abc][-: ][num:123][-:, ][word:x][num:9]", true);
test_template(t, "tts_norm_scan groups, optional group that did not match is empty",
"{%- for s in tts_norm_scan(text, [['money', '%$(%d+)(?:%.(%d%d))?']]) -%}"
"{%- if s.rule -%}{%- set whole, cents = s.groups -%}{{ whole }}|{{ cents }}|{{ s.groups | length }};{%- endif -%}"
"{%- endfor -%}",
{{"text", "$3.50 and $7"}},
"3|50|2;7||2;", true);
test_template(t, "tts_norm_scan rules as dict, the first rule that matches wins",
"{%- for s in tts_norm_scan(text, {'ordinal': '%d+(?:st|nd|rd|th)%b', 'number': '%d+'}) -%}"
"{%- if s.rule -%}<{{ s.rule }}:{{ s.text }}>{%- else -%}{{ s.text }}{%- endif -%}"
"{%- endfor -%}",
{{"text", "the 3rd of 10 or 4three"}},
"the <ordinal:3rd> of <number:10> or <number:4>three", true);
// unlike regex, a failed sequence does not retry the other branches: (?:a|ab)c never matches "abc"
test_template(t, "ordered alternation commits to the first branch that matches",
"{%- for s in tts_norm_scan(text, '(?:a|ab)c') -%}[{{ s.rule or '-' }}:{{ s.text }}]{%- endfor -%}",
{{"text", "abc ac"}},
"[-:abc ][match:ac]", true);
// unlike regex, quantifiers never give back what they consumed: %d+0 never matches
test_template(t, "quantifiers are possessive",
"{{ tts_norm_sub(text, '%d+0', 'X') }}|{{ tts_norm_sub(text, '%d+', 'X') }}",
{{"text", "100 20"}},
"100 20|X X", true);
test_template(t, "quantifier ranges",
"{{ tts_norm_sub(text, '%d{3}-%d{2,}', '#') }}",
{{"text", "123-4 123-45 123-456789"}},
"123-4 # #", true);
test_template(t, "lookbehind: a minus sign is not part of a word",
"{{ tts_norm_sub(text, '(?<!%w)-(%d+)', 'minus %1') }}",
{{"text", "GPT-5 is -5, (-3)"}},
"GPT-5 is minus 5, (minus 3)", true);
test_template(t, "lookahead: clock time not followed by a digit",
"{{ tts_norm_sub(text, '(%d%d?):(%d%d)(?!%d)', '%1h%2') }}",
{{"text", "at 10:30 or 1:234"}},
"at 10h30 or 1:234", true);
test_template(t, "tts_norm_sub references: group, whole match and literal percent",
"{{ tts_norm_sub(text, '(%d+)%%', '%1 percent (%0) 100%%') }}",
{{"text", "up 50%"}},
"up 50 percent (50%) 100%", true);
test_template(t, "split letters and digits, non-ASCII letters included",
"{{ tts_norm_sub(tts_norm_sub(text, '(%a)(%d)', '%1 %2'), '(%d)(%a)', '%1 %2') }}",
{{"text", "R2D2 café2 4K"}},
"R 2 D 2 café 2 4 K", true);
test_template(t, "case-insensitive pattern and word boundary",
"{{ tts_norm_sub(text, '(?i)%bdr%.', 'doctor') }}",
{{"text", "Dr. Who and dr. No, not Mdr."}},
"doctor Who and doctor No, not Mdr.", true);
test_template(t, "sets with multi-byte codepoints, negated sets with classes",
"{{ tts_norm_sub(text, '[$€£]%s*(%d+)', '<%1>') }}|{{ tts_norm_sub(text, '[^%w%s]', '') }}",
{{"text", "€ 5 & $6!"}},
"<5> & <6>!| 5 6", true);
test_template(t, "remove spaces between Han characters",
"{{ tts_norm_sub(text, '(%h) +(%h)', '%1%2') }}",
{{"text", "你 好 world 世界"}},
"你好 world 世界", true);
test_template(t, "mini text normalizer",
"{%- set ones = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten',"
" 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'] -%}\n"
"{%- set tens = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'] -%}\n"
"{%- macro num(n) -%}{{ ones[n] if n < 20 else tens[(n / 10) | int] ~ ('-' ~ ones[n % 10] if n % 10 else '') }}{%- endmacro -%}\n"
"{%- for s in tts_norm_scan(text, [\n"
" ['money', '%$(%d+)(?:%.(%d%d))?'],\n"
" ['percent', '(%d+)%%'],\n"
" ['time', '(%d%d?):(%d%d)(?!%d)'],\n"
" ['neg', '(?<!%w)-(%d+)'],\n"
" ['number', '%d+'],\n"
"]) -%}\n"
"{%- if s.rule == 'money' -%}{%- set whole, cents = s.groups -%}{{ num(whole | int) }} dollars{{ ' and ' ~ num(cents | int) ~ ' cents' if cents else '' }}\n"
"{%- elif s.rule == 'percent' -%}{{ num(s.groups[0] | int) }} percent\n"
"{%- elif s.rule == 'time' -%}{%- set h, m = s.groups -%}{{ num(h | int) }} {{ num(m | int) }}\n"
"{%- elif s.rule == 'neg' -%}minus {{ num(s.groups[0] | int) }}\n"
"{%- elif s.rule == 'number' -%}{{ num(s.text | int) }}\n"
"{%- else -%}{{ s.text }}\n"
"{%- endif -%}\n"
"{%- endfor -%}",
{{"text", "It costs $3.50, 20% off, at 10:45, -5 degrees, 7 cats."}},
"It costs three dollars and fifty cents, twenty percent off, at ten forty-five, minus five degrees, seven cats.", true);
if (!g_python_mode) {
test_template_error(t, "error: unclosed group", "{{ tts_norm_scan('x', '(a') }}", "expected ')' to close group", true);
test_template_error(t, "error: unbalanced parenthesis", "{{ tts_norm_scan('x', 'a)') }}", "unbalanced ')'", true);
test_template_error(t, "error: nothing to repeat", "{{ tts_norm_scan('x', '*a') }}", "nothing to repeat", true);
test_template_error(t, "error: multiple quantifiers", "{{ tts_norm_scan('x', 'a+?') }}", "multiple quantifiers", true);
test_template_error(t, "error: bad quantifier range", "{{ tts_norm_scan('x', 'a{2,1}') }}", "quantifier max is smaller than min", true);
test_template_error(t, "error: repeated assertion", "{{ tts_norm_scan('x', '%b+') }}", "zero-width assertion cannot be repeated", true);
test_template_error(t, "error: unknown class", "{{ tts_norm_scan('x', '%q') }}", "unknown escape '%q'", true);
test_template_error(t, "error: unescaped bracket", "{{ tts_norm_scan('x', 'a]') }}", "unescaped ']'", true);
test_template_error(t, "error: no named groups", "{{ tts_norm_scan('x', '(?<n>a)') }}", "unknown group type", true);
test_template_error(t, "error: wide lookbehind", "{{ tts_norm_scan('x', '(?<=ab)c') }}", "lookbehind must contain exactly one codepoint matcher", true);
test_template_error(t, "error: missing group in replacement", "{{ tts_norm_sub('x', '(a)', '%2') }}", "group %2 does not exist", true);
test_template_error(t, "error: bad rules", "{{ tts_norm_scan('x', 1) }}", "rules must be a string, a dict or a list", true);
}
}
static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm) {
t.test(name, [&tmpl, &expect, tts_norm](testing & t) {
std::string error;
try {
jinja::lexer lexer;
jinja::program ast = jinja::parse_from_tokens(lexer.tokenize(tmpl));
jinja::context ctx(tmpl);
if (tts_norm) {
common_chat_tts_norm_add_builtins(ctx);
}
jinja::runtime runtime(ctx);
runtime.gather_string_parts(runtime.execute(ast));
} catch (const std::exception & e) {
@@ -2729,14 +2853,17 @@ static void test_template_error(testing & t, const std::string & name, const std
});
}
static void test_template_cpp(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) {
t.test(name, [&tmpl, &vars, &expect](testing & t) {
static void test_template_cpp(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) {
t.test(name, [&tmpl, &vars, &expect, tts_norm](testing & t) {
jinja::lexer lexer;
auto lexer_res = lexer.tokenize(tmpl);
jinja::program ast = jinja::parse_from_tokens(lexer_res);
jinja::context ctx(tmpl);
if (tts_norm) {
common_chat_tts_norm_add_builtins(ctx);
}
jinja::global_from_json(ctx, vars, true);
jinja::runtime runtime(ctx);
@@ -2788,17 +2915,36 @@ env.filters["tojson"] = lambda x, ensure_ascii=False, indent=None, separators=No
env.globals["strftime_now"] = lambda format: datetime.now().strftime(format)
env.globals["raise_exception"] = raise_exception
if merged_input.get("tts_norm_ref"):
import importlib.util
spec = importlib.util.spec_from_file_location("chat_tts_norm", merged_input["tts_norm_ref"])
chat_tts_norm = importlib.util.module_from_spec(spec)
sys.modules["chat_tts_norm"] = chat_tts_norm # required by dataclasses
spec.loader.exec_module(chat_tts_norm)
env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan
env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub
template = env.from_string(tmpl)
result = template.render(**vars_json)
sys.stdout.buffer.write(result.encode())
)";
static void test_template_py(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) {
t.test(name, [&tmpl, &vars, &expect](testing & t) {
// python reference of the TTS normalization patterns, relative to this file
static std::string tts_norm_ref_path() {
std::string path = __FILE__;
const size_t pos = path.find_last_of("/\\");
return (pos == std::string::npos ? std::string(".") : path.substr(0, pos)) + "/../scripts/chat_tts_norm.py";
}
static void test_template_py(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) {
t.test(name, [&tmpl, &vars, &expect, tts_norm](testing & t) {
// Prepare arguments
json merged;
merged["tmpl"] = json(tmpl);
merged["vars"] = vars;
if (tts_norm) {
merged["tts_norm_ref"] = tts_norm_ref_path();
}
#ifdef _WIN32
const char * python_executable = "python.exe";
@@ -2859,11 +3005,11 @@ static void test_template_py(testing & t, const std::string & name, const std::s
});
}
static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) {
static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) {
if (g_python_mode) {
test_template_py(t, name, tmpl, vars, expect);
test_template_py(t, name, tmpl, vars, expect, tts_norm);
} else {
test_template_cpp(t, name, tmpl, vars, expect);
test_template_cpp(t, name, tmpl, vars, expect, tts_norm);
}
}