diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 8e3c88dac..e38485ca0 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -66,6 +66,8 @@ add_library(${TARGET} chat-diff-analyzer.cpp chat-peg-parser.cpp chat-peg-parser.h + chat-tts-norm.cpp + chat-tts-norm.h chat.cpp chat.h common.cpp diff --git a/common/chat-tts-norm.cpp b/common/chat-tts-norm.cpp new file mode 100644 index 000000000..f84540ff9 --- /dev/null +++ b/common/chat-tts-norm.cpp @@ -0,0 +1,864 @@ +#include "chat-tts-norm.h" + +#include "jinja/runtime.h" +#include "jinja/value.h" + +#include +#include +#include +#include +#include + +// TTS normalization patterns, see docs/tts-norm-patterns.md for the specification +// keep this file in sync with the python reference: scripts/chat_tts_norm.py + +using namespace jinja; + +// +// character classes +// they are defined with explicit codepoint ranges, independent of any unicode database version, +// so that the C++ and python implementations agree on every codepoint +// + +struct tts_norm_range { + uint32_t lo; + uint32_t hi; +}; + +static const tts_norm_range TTS_NORM_SPACE[] = { + {0x0009, 0x000D}, {0x0020, 0x0020}, {0x0085, 0x0085}, {0x00A0, 0x00A0}, {0x1680, 0x1680}, + {0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F}, {0x205F, 0x205F}, {0x3000, 0x3000}, +}; + +static const tts_norm_range TTS_NORM_PUNCT[] = { + {0x0021, 0x002F}, {0x003A, 0x0040}, {0x005B, 0x0060}, {0x007B, 0x007E}, + {0x00A1, 0x00A9}, {0x00AB, 0x00B4}, {0x00B6, 0x00B9}, {0x00BB, 0x00BF}, {0x00D7, 0x00D7}, {0x00F7, 0x00F7}, + {0x2010, 0x2027}, {0x2030, 0x205E}, {0x20A0, 0x20CF}, {0x2100, 0x218F}, {0x2190, 0x2BFF}, + {0x3001, 0x3003}, {0x3008, 0x3020}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30FB, 0x30FB}, + {0xFE10, 0xFE19}, {0xFE30, 0xFE4F}, {0xFE50, 0xFE6B}, + {0xFF01, 0xFF0F}, {0xFF1A, 0xFF20}, {0xFF3B, 0xFF40}, {0xFF5B, 0xFF65}, + {0x1F000, 0x1FAFF}, +}; + +static const tts_norm_range TTS_NORM_CONTROL[] = { + {0x0000, 0x001F}, {0x007F, 0x009F}, {0x200B, 0x200F}, {0x2060, 0x206F}, {0xFEFF, 0xFEFF}, +}; + +static const tts_norm_range TTS_NORM_HAN[] = { + {0x2E80, 0x2FDF}, {0x3005, 0x3005}, {0x3007, 0x3007}, {0x3021, 0x3029}, {0x3038, 0x303B}, + {0x3400, 0x4DBF}, {0x4E00, 0x9FFF}, {0xF900, 0xFAFF}, {0x20000, 0x323AF}, +}; + +template +static bool tts_norm_in(const tts_norm_range (&ranges)[N], uint32_t c) { + for (const auto & r : ranges) { + if (c >= r.lo && c <= r.hi) { + return true; + } + } + return false; +} + +static bool tts_norm_is_alpha(uint32_t c) { + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) { + return true; + } + if (c == 0xAA || c == 0xB5 || c == 0xBA) { + return true; + } + return c >= 0xC0 && !tts_norm_in(TTS_NORM_SPACE, c) && !tts_norm_in(TTS_NORM_PUNCT, c) && !tts_norm_in(TTS_NORM_CONTROL, c); +} + +// kind is one of "dspawulhc" +static bool tts_norm_class_match(char kind, uint32_t c) { + switch (kind) { + case 'd': return c >= '0' && c <= '9'; + case 's': return tts_norm_in(TTS_NORM_SPACE, c); + case 'p': return tts_norm_in(TTS_NORM_PUNCT, c); + case 'a': return tts_norm_is_alpha(c); + case 'w': return tts_norm_is_alpha(c) || (c >= '0' && c <= '9'); + case 'u': return c >= 'A' && c <= 'Z'; + case 'l': return c >= 'a' && c <= 'z'; + case 'h': return tts_norm_in(TTS_NORM_HAN, c); + case 'c': return tts_norm_in(TTS_NORM_CONTROL, c); + } + return false; +} + +static bool tts_norm_is_class_letter(uint32_t c) { + if (c >= 'A' && c <= 'Z') { + c += 'a' - 'A'; + } + return c < 0x80 && std::string("dspawulhc").find(static_cast(c)) != std::string::npos; +} + +static uint32_t tts_norm_ascii_lower(uint32_t c) { + return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c; +} + +static uint32_t tts_norm_ascii_swapcase(uint32_t c) { + if (c >= 'A' && c <= 'Z') return c + ('a' - 'A'); + if (c >= 'a' && c <= 'z') return c - ('a' - 'A'); + return c; +} + +static bool tts_norm_is_ascii_alnum(uint32_t c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +// invalid bytes are decoded as U+FFFD, one per byte; offs[i] is the byte offset of codepoint i, offs[n] = text size +static void tts_norm_decode_utf8(const std::string & s, std::vector & cpts, std::vector & offs) { + cpts.clear(); + offs.clear(); + size_t i = 0; + const size_t n = s.size(); + while (i < n) { + offs.push_back(i); + const unsigned char c = static_cast(s[i]); + uint32_t cp = 0; + size_t len = 0; + if (c < 0x80) { + cp = c; len = 1; + } else if ((c & 0xE0) == 0xC0) { + cp = c & 0x1F; len = 2; + } else if ((c & 0xF0) == 0xE0) { + cp = c & 0x0F; len = 3; + } else if ((c & 0xF8) == 0xF0) { + cp = c & 0x07; len = 4; + } + bool ok = len > 0 && i + len <= n; + for (size_t k = 1; ok && k < len; k++) { + const unsigned char cc = static_cast(s[i + k]); + if ((cc & 0xC0) != 0x80) { + ok = false; + } else { + cp = (cp << 6) | (cc & 0x3F); + } + } + if (!ok) { + cpts.push_back(0xFFFD); + i += 1; + } else { + cpts.push_back(cp); + i += len; + } + } + offs.push_back(n); +} + +// +// AST +// + +enum class tts_norm_op { + LITERAL, // one codepoint + SET, // one codepoint from a set; class escapes like %d are sets with a single class + ANY, // any codepoint + START, // ^ + END, // $ + BOUNDARY, // %b + GROUP, // (...), (?:...), (?...) + LOOKAHEAD, // (?=...), (?!...) + LOOKBEHIND, // (?<=c), (?> ranges; + std::vector classes; +}; + +struct tts_norm_item; +using tts_norm_seq = std::vector; + +struct tts_norm_pattern_node { + tts_norm_op op = tts_norm_op::GROUP; + uint32_t cp = 0; // LITERAL + tts_norm_set set; // SET + std::vector alts; // GROUP, LOOKAHEAD; LOOKBEHIND holds a single item + int capture = -1; // GROUP: capture index, -1 for non-capturing + bool neg = false; // LOOKAHEAD, LOOKBEHIND +}; + +static constexpr int TTS_NORM_INF = INT_MAX; +static constexpr int TTS_NORM_MAX_REPEAT = 1000; +static constexpr size_t TTS_NORM_MAX_PATTERN_LEN = 4096; + +struct tts_norm_item { + tts_norm_pattern_node atom; + int min = 1; + int max = 1; +}; + +static bool tts_norm_is_zero_width(tts_norm_op op) { + return op == tts_norm_op::START || op == tts_norm_op::END || op == tts_norm_op::BOUNDARY + || op == tts_norm_op::LOOKAHEAD || op == tts_norm_op::LOOKBEHIND; +} + +// +// parser +// + +struct tts_norm_parser { + std::vector s; + size_t p = 0; + size_t & n_caps; + + tts_norm_parser(std::vector src, size_t & n_caps) : s(std::move(src)), n_caps(n_caps) {} + + [[noreturn]] void fail(const std::string & msg) const { + throw std::runtime_error("invalid TTS norm pattern: " + msg + " at position " + std::to_string(p)); + } + + bool eof() const { return p >= s.size(); } + uint32_t peek() const { return s[p]; } + bool peek_is(uint32_t c) const { return !eof() && s[p] == c; } + + void expect(uint32_t c, const char * what) { + if (!peek_is(c)) { + fail(std::string("expected ") + what); + } + p++; + } + + std::vector parse_alts() { + std::vector alts; + alts.push_back(parse_seq()); + while (peek_is('|')) { + p++; + alts.push_back(parse_seq()); + } + return alts; + } + + tts_norm_seq parse_seq() { + tts_norm_seq seq; + while (!eof() && peek() != '|' && peek() != ')') { + tts_norm_item item; + item.atom = parse_atom(); + parse_quantifier(item); + seq.push_back(std::move(item)); + } + return seq; + } + + int parse_int() { + if (eof() || peek() < '0' || peek() > '9') { + fail("expected number in quantifier"); + } + int v = 0; + while (!eof() && peek() >= '0' && peek() <= '9') { + v = v * 10 + static_cast(peek() - '0'); + if (v > TTS_NORM_MAX_REPEAT) { + fail("repetition count exceeds " + std::to_string(TTS_NORM_MAX_REPEAT)); + } + p++; + } + return v; + } + + void parse_quantifier(tts_norm_item & item) { + if (eof()) { + return; + } + const uint32_t c = peek(); + if (c != '?' && c != '*' && c != '+' && c != '{') { + return; + } + if (tts_norm_is_zero_width(item.atom.op)) { + fail("zero-width assertion cannot be repeated"); + } + p++; + if (c == '?') { + item.min = 0; item.max = 1; + } else if (c == '*') { + item.min = 0; item.max = TTS_NORM_INF; + } else if (c == '+') { + item.min = 1; item.max = TTS_NORM_INF; + } else { + item.min = parse_int(); + item.max = item.min; + if (peek_is(',')) { + p++; + item.max = peek_is('}') ? TTS_NORM_INF : parse_int(); + } + expect('}', "'}' to close quantifier"); + if (item.max < item.min) { + fail("quantifier max is smaller than min"); + } + } + if (!eof() && (peek() == '?' || peek() == '*' || peek() == '+' || peek() == '{')) { + fail("multiple quantifiers"); + } + } + + tts_norm_pattern_node parse_group() { + // '(' already consumed + tts_norm_pattern_node node; + node.op = tts_norm_op::GROUP; + if (peek_is('?')) { + p++; + if (eof()) { + fail("unterminated group"); + } + const uint32_t c = peek(); + if (c == ':') { + p++; + } else if (c == '=' || c == '!') { + p++; + node.op = tts_norm_op::LOOKAHEAD; + node.neg = c == '!'; + } else if (c == '<' && p + 1 < s.size() && (s[p + 1] == '=' || s[p + 1] == '!')) { + node.op = tts_norm_op::LOOKBEHIND; + node.neg = s[p + 1] == '!'; + p += 2; + tts_norm_item item; + if (eof() || peek() == ')') { + fail("lookbehind must contain exactly one codepoint matcher"); + } + item.atom = parse_atom(); + if (item.atom.op != tts_norm_op::LITERAL && item.atom.op != tts_norm_op::SET && item.atom.op != tts_norm_op::ANY) { + fail("lookbehind must contain exactly one codepoint matcher"); + } + if (!peek_is(')')) { + fail("lookbehind must contain exactly one codepoint matcher"); + } + p++; + node.alts.push_back(tts_norm_seq{}); + node.alts[0].push_back(std::move(item)); + return node; + } else if (c == 'i') { + fail("(?i) is only allowed at the start of the pattern"); + } else { + fail("unknown group type"); + } + } else { + node.capture = static_cast(n_caps++); + } + node.alts = parse_alts(); + expect(')', "')' to close group"); + return node; + } + + // after '%' inside or outside of a set + // returns true if it is a class (stored in cls), false if it is a literal (stored in lit) + bool parse_escape(bool in_set, tts_norm_class & cls, uint32_t & lit, bool & boundary) { + boundary = false; + if (eof()) { + fail("dangling '%'"); + } + const uint32_t e = s[p++]; + if (tts_norm_is_class_letter(e)) { + cls.kind = static_cast(tts_norm_ascii_lower(e)); + cls.neg = e >= 'A' && e <= 'Z'; + return true; + } + if (e == 'b' && !in_set) { + boundary = true; + return false; + } + if (tts_norm_is_ascii_alnum(e)) { + p--; + fail(std::string("unknown escape '%") + static_cast(e) + "'"); + } + lit = e; + return false; + } + + tts_norm_pattern_node parse_set() { + // '[' already consumed + tts_norm_pattern_node node; + node.op = tts_norm_op::SET; + if (peek_is('^')) { + node.set.neg = true; + p++; + } + bool empty = true; + while (true) { + if (eof()) { + fail("unterminated set"); + } + if (peek() == ']') { + p++; + break; + } + empty = false; + uint32_t lo = 0; + if (peek() == '%') { + p++; + tts_norm_class cls{}; + bool boundary = false; + if (parse_escape(true, cls, lo, boundary)) { + node.set.classes.push_back(cls); + continue; + } + } else { + lo = s[p++]; + } + uint32_t hi = lo; + if (peek_is('-') && p + 1 < s.size() && s[p + 1] != ']') { + p++; + if (peek() == '%') { + p++; + tts_norm_class cls{}; + bool boundary = false; + if (parse_escape(true, cls, hi, boundary)) { + fail("character class cannot be used in a range"); + } + } else { + hi = s[p++]; + } + if (hi < lo) { + fail("invalid range in set"); + } + } + node.set.ranges.emplace_back(lo, hi); + } + if (empty) { + fail("empty set"); + } + return node; + } + + tts_norm_pattern_node parse_atom() { + const uint32_t c = s[p++]; + tts_norm_pattern_node node; + switch (c) { + case '(': + return parse_group(); + case '[': + return parse_set(); + case '.': + node.op = tts_norm_op::ANY; + return node; + case '^': + node.op = tts_norm_op::START; + return node; + case '$': + node.op = tts_norm_op::END; + return node; + case '%': { + tts_norm_class cls{}; + uint32_t lit = 0; + bool boundary = false; + if (parse_escape(false, cls, lit, boundary)) { + node.op = tts_norm_op::SET; + node.set.classes.push_back(cls); + } else if (boundary) { + node.op = tts_norm_op::BOUNDARY; + } else { + node.op = tts_norm_op::LITERAL; + node.cp = lit; + } + return node; + } + case '?': + case '*': + case '+': + case '{': + p--; + fail("nothing to repeat"); + case ']': + case '}': + p--; + fail(std::string("unescaped '") + static_cast(c) + "', use '%" + static_cast(c) + "'"); + default: + node.op = tts_norm_op::LITERAL; + node.cp = c; + return node; + } + } +}; + +// +// matcher +// PEG-style: alternatives are ordered (the first one that matches wins) and quantifiers are greedy and possessive +// (they never give back what they consumed), so there is no backtracking +// + +struct tts_norm_matcher { + const std::vector & t; + bool icase; + std::vector> caps; + + bool is_w(size_t i) const { + return i < t.size() && tts_norm_class_match('w', t[i]); + } + + bool set_match(const tts_norm_set & st, uint32_t c) const { + auto test = [&](uint32_t x) { + for (const auto & r : st.ranges) { + if (x >= r.first && x <= r.second) { + return true; + } + } + for (const auto & k : st.classes) { + if (tts_norm_class_match(k.kind, x) != k.neg) { + return true; + } + } + return false; + }; + bool m = test(c); + if (!m && icase) { + const uint32_t o = tts_norm_ascii_swapcase(c); + if (o != c) { + m = test(o); + } + } + return m != st.neg; + } + + bool match_alts(const std::vector & alts, size_t i, size_t & end) { + for (const auto & seq : alts) { + auto saved = caps; + if (match_seq(seq, i, end)) { + return true; + } + caps = std::move(saved); + } + return false; + } + + bool match_seq(const tts_norm_seq & seq, size_t i, size_t & end) { + size_t pos = i; + for (const auto & it : seq) { + int count = 0; + while (count < it.max) { + const bool is_group = it.atom.op == tts_norm_op::GROUP; + std::vector> saved; + if (is_group) { + saved = caps; + } + size_t e = pos; + if (!match_once(it.atom, pos, e)) { + if (is_group) { + caps = std::move(saved); + } + break; + } + if (e == pos) { + // an empty iteration would repeat forever, it satisfies any remaining minimum + count = std::max(count + 1, it.min); + break; + } + pos = e; + count++; + } + if (count < it.min) { + return false; + } + } + end = pos; + return true; + } + + bool match_once(const tts_norm_pattern_node & node, size_t i, size_t & end) { + const size_t n = t.size(); + switch (node.op) { + case tts_norm_op::LITERAL: + if (i < n && (t[i] == node.cp || (icase && tts_norm_ascii_lower(t[i]) == tts_norm_ascii_lower(node.cp)))) { + end = i + 1; + return true; + } + return false; + case tts_norm_op::SET: + if (i < n && set_match(node.set, t[i])) { + end = i + 1; + return true; + } + return false; + case tts_norm_op::ANY: + if (i < n) { + end = i + 1; + return true; + } + return false; + case tts_norm_op::START: + end = i; + return i == 0; + case tts_norm_op::END: + end = i; + return i == n; + case tts_norm_op::BOUNDARY: + end = i; + return (i > 0 && is_w(i - 1)) != is_w(i); + case tts_norm_op::GROUP: { + size_t e = i; + if (!match_alts(node.alts, i, e)) { + return false; + } + if (node.capture >= 0) { + caps[node.capture] = {static_cast(i), static_cast(e)}; + } + end = e; + return true; + } + case tts_norm_op::LOOKAHEAD: { + // captures made inside a lookahead are discarded + auto saved = caps; + size_t e = i; + const bool ok = match_alts(node.alts, i, e); + caps = std::move(saved); + end = i; + return ok != node.neg; + } + case tts_norm_op::LOOKBEHIND: { + size_t e = 0; + const bool ok = i > 0 && match_once(node.alts[0][0].atom, i - 1, e) && e == i; + end = i; + return ok != node.neg; + } + } + return false; + } +}; + +// +// common_tts_norm_pattern +// + +common_tts_norm_pattern::common_tts_norm_pattern(const std::string & src) { + if (src.size() > TTS_NORM_MAX_PATTERN_LEN) { + throw std::runtime_error("invalid TTS norm pattern: pattern is longer than " + std::to_string(TTS_NORM_MAX_PATTERN_LEN) + " bytes"); + } + std::vector cpts; + std::vector offs; + tts_norm_decode_utf8(src, cpts, offs); + + tts_norm_parser parser(std::move(cpts), n_caps); + static const std::vector icase_prefix = {'(', '?', 'i', ')'}; + if (parser.s.size() >= 4 && std::equal(icase_prefix.begin(), icase_prefix.end(), parser.s.begin())) { + icase = true; + parser.p = 4; + } + + root = std::make_unique(); + root->op = tts_norm_op::GROUP; + root->alts = parser.parse_alts(); + if (!parser.eof()) { + parser.fail("unbalanced ')'"); + } +} + +common_tts_norm_pattern::~common_tts_norm_pattern() = default; +common_tts_norm_pattern::common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept = default; +common_tts_norm_pattern & common_tts_norm_pattern::operator=(common_tts_norm_pattern &&) noexcept = default; + +bool common_tts_norm_pattern::match_at(const std::vector & cpts, size_t pos, size_t & end, std::vector> & groups) const { + tts_norm_matcher m{cpts, icase, std::vector>(n_caps, {-1, -1})}; + size_t e = pos; + if (!m.match_alts(root->alts, pos, e)) { + return false; + } + end = e; + groups = std::move(m.caps); + return true; +} + +size_t common_tts_norm_pattern::n_groups() const { + return n_caps; +} + +// +// scan / sub +// + +std::vector common_tts_norm_scan(const std::string & text, const std::vector & rules) { + std::vector cpts; + std::vector offs; + tts_norm_decode_utf8(text, cpts, offs); + + std::vector out; + const size_t n = cpts.size(); + size_t i = 0; + size_t pending = 0; // start of the current unmatched run + std::vector> groups; + + while (i < n) { + bool matched = false; + for (size_t r = 0; r < rules.size(); r++) { + size_t end = i; + if (rules[r]->match_at(cpts, i, end, groups) && end > i) { + if (pending < i) { + out.push_back({-1, offs[pending], offs[i], {}}); + } + common_tts_norm_segment seg; + seg.rule = static_cast(r); + seg.start = offs[i]; + seg.end = offs[end]; + for (const auto & g : groups) { + if (g.first < 0) { + seg.groups.emplace_back(std::string::npos, std::string::npos); + } else { + seg.groups.emplace_back(offs[g.first], offs[g.second]); + } + } + out.push_back(std::move(seg)); + i = end; + pending = i; + matched = true; + break; + } + } + if (!matched) { + i++; + } + } + if (pending < n) { + out.push_back({-1, offs[pending], offs[n], {}}); + } + return out; +} + +struct tts_norm_repl_piece { + std::string literal; + int group = -1; // 0 = whole match, 1.. = groups +}; + +static std::vector tts_norm_parse_repl(const std::string & repl, const common_tts_norm_pattern & pattern) { + std::vector pieces; + std::string lit; + auto flush = [&]() { + if (!lit.empty()) { + pieces.push_back({lit, -1}); + lit.clear(); + } + }; + for (size_t i = 0; i < repl.size(); i++) { + if (repl[i] != '%') { + lit += repl[i]; + continue; + } + if (i + 1 >= repl.size()) { + throw std::runtime_error("invalid TTS norm replacement: dangling '%'"); + } + const char c = repl[++i]; + if (c == '%') { + lit += '%'; + } else if (c >= '0' && c <= '9') { + const int g = c - '0'; + if (static_cast(g) > pattern.n_groups()) { + throw std::runtime_error("invalid TTS norm replacement: group %" + std::string(1, c) + " does not exist"); + } + flush(); + pieces.push_back({"", g}); + } else { + throw std::runtime_error("invalid TTS norm replacement: unknown escape '%" + std::string(1, c) + "'"); + } + } + flush(); + return pieces; +} + +std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl) { + const auto pieces = tts_norm_parse_repl(repl, pattern); + std::string out; + for (const auto & seg : common_tts_norm_scan(text, {&pattern})) { + if (seg.rule < 0) { + out.append(text, seg.start, seg.end - seg.start); + continue; + } + for (const auto & piece : pieces) { + if (piece.group < 0) { + out += piece.literal; + } else if (piece.group == 0) { + out.append(text, seg.start, seg.end - seg.start); + } else { + const auto & g = seg.groups[piece.group - 1]; + if (g.first != std::string::npos) { + out.append(text, g.first, g.second - g.first); + } + } + } + } + return out; +} + +// +// template builtins +// + +// rules can be: a pattern string (rule name "match"), a dict {name: pattern}, or a list of [name, pattern] pairs +static std::vector> tts_norm_rules_from_value(const value & v) { + std::vector> rules; + if (is_val(v)) { + rules.emplace_back("match", v->as_string().str()); + } else if (is_val(v)) { + for (const auto & kv : v->as_ordered_object()) { + if (!is_val(kv.second)) { + throw std::runtime_error("tts_norm_scan: rule pattern must be a string"); + } + rules.emplace_back(kv.first->as_string().str(), kv.second->as_string().str()); + } + } else if (is_val(v)) { + for (const auto & item : v->as_array()) { + if (!is_val(item) || item->as_array().size() != 2 + || !is_val(item->as_array()[0]) || !is_val(item->as_array()[1])) { + throw std::runtime_error("tts_norm_scan: each rule must be a [name, pattern] pair of strings"); + } + rules.emplace_back(item->as_array()[0]->as_string().str(), item->as_array()[1]->as_string().str()); + } + } else { + throw std::runtime_error("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs, got " + v->type()); + } + return rules; +} + +static value tts_norm_builtin_scan(const func_args & args) { + args.ensure_count(2, 2); + args.ensure_val(args.get_pos(0)); + const string text_val = args.get_pos(0)->as_string(); + const std::string text = text_val.str(); + const bool is_input = text_val.all_parts_are_input(); + + const auto rule_srcs = tts_norm_rules_from_value(args.get_pos(1)); + std::vector patterns; + patterns.reserve(rule_srcs.size()); + std::vector ptrs; + for (const auto & r : rule_srcs) { + patterns.emplace_back(r.second); + } + for (const auto & p : patterns) { + ptrs.push_back(&p); + } + + auto mk_str = [&](size_t start, size_t end) -> value { + return mk_val(string(text.substr(start, end - start), is_input)); + }; + + auto out = mk_val(); + for (const auto & seg : common_tts_norm_scan(text, ptrs)) { + auto groups = mk_val(); + for (const auto & g : seg.groups) { + // groups that did not participate are empty strings, so they render as nothing and are falsy + groups->push_back(g.first == std::string::npos ? mk_str(0, 0) : mk_str(g.first, g.second)); + } + auto obj = mk_val(); + obj->insert("rule", seg.rule < 0 ? value(mk_val()) : value(mk_val(rule_srcs[seg.rule].first))); + obj->insert("text", mk_str(seg.start, seg.end)); + obj->insert("groups", groups); + out->push_back(obj); + } + return out; +} + +static value tts_norm_builtin_sub(const func_args & args) { + args.ensure_count(3, 3); + args.ensure_vals(); + const string text_val = args.get_pos(0)->as_string(); + const common_tts_norm_pattern pattern(args.get_pos(1)->as_string().str()); + const std::string res = common_tts_norm_sub(text_val.str(), pattern, args.get_pos(2)->as_string().str()); + return mk_val(string(res, text_val.all_parts_are_input())); +} + +void common_chat_tts_norm_add_builtins(context & ctx) { + ctx.set_val("tts_norm_scan", mk_val("tts_norm_scan", tts_norm_builtin_scan)); + ctx.set_val("tts_norm_sub", mk_val("tts_norm_sub", tts_norm_builtin_sub)); +} + diff --git a/common/chat-tts-norm.h b/common/chat-tts-norm.h new file mode 100644 index 000000000..14695bb52 --- /dev/null +++ b/common/chat-tts-norm.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include +#include + +// TTS normalization patterns: a small, fully specified matching language for TTS text normalization templates +// +// it is NOT a regex engine: the semantics are defined by llama.cpp (see docs/tts-norm-patterns.md), so that this +// implementation and the python reference (scripts/chat_tts_norm.py) give the same result for every input +// +// two functions are exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins(): +// tts_norm_scan(text, rules) -> list of segments {rule, text, groups} +// tts_norm_sub(text, pattern, repl) -> string + +namespace jinja { +struct context; +} + +struct tts_norm_pattern_node; + +class common_tts_norm_pattern { +public: + // throws std::runtime_error on invalid pattern + explicit common_tts_norm_pattern(const std::string & src); + ~common_tts_norm_pattern(); + common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept; + common_tts_norm_pattern & operator=(common_tts_norm_pattern &&) noexcept; + + // match anchored at codepoint index pos; on success, end is the codepoint index after the match + // groups[i] holds [start, end) codepoint indices of capture i+1, or {-1, -1} if the group did not participate + bool match_at(const std::vector & cpts, size_t pos, size_t & end, std::vector> & groups) const; + + size_t n_groups() const; + +private: + std::unique_ptr root; + size_t n_caps = 0; + bool icase = false; +}; + +// segment produced by common_tts_norm_scan(); byte offsets into the input text +struct common_tts_norm_segment { + int rule = -1; // index of the matched rule, -1 for unmatched text + size_t start = 0; + size_t end = 0; + std::vector> groups; // byte offsets; {npos, npos} for groups that did not participate +}; + +// split text into matched and unmatched segments +// at each position, rules are tried in order and the first non-empty match wins; otherwise one codepoint is skipped +std::vector common_tts_norm_scan(const std::string & text, const std::vector & rules); + +// replace every non-overlapping match of pattern with repl +// repl references: %0 (whole match), %1..%9 (groups), %% (literal percent) +std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl); + +// add tts_norm_scan() and tts_norm_sub() to the context of a task template +// they are not jinja global builtins on purpose: chat templates are not affected +void common_chat_tts_norm_add_builtins(jinja::context & ctx); diff --git a/common/chat.cpp b/common/chat.cpp index ef9b853f0..0e27f8197 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -3,6 +3,7 @@ #include "chat-auto-parser-helpers.h" #include "chat-auto-parser.h" #include "chat-peg-parser.h" +#include "chat-tts-norm.h" #include "common.h" #include "ggml.h" #include "json-schema-to-grammar.h" @@ -807,6 +808,8 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp // renders a task template (e.g. "tts") with the given variables static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) { jinja::context ctx(tmpl.source()); + // text normalization helpers: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md + common_chat_tts_norm_add_builtins(ctx); jinja::global_from_json(ctx, vars, false); ctx.set_val("bos_token", jinja::mk_val(tmpl.bos_token())); ctx.set_val("eos_token", jinja::mk_val(tmpl.eos_token())); diff --git a/docs/tts-norm-patterns.md b/docs/tts-norm-patterns.md new file mode 100644 index 000000000..802dde5ce --- /dev/null +++ b/docs/tts-norm-patterns.md @@ -0,0 +1,125 @@ +# TTS normalization patterns + +Text-to-speech models often need their input normalized before tokenization: numbers, currencies, dates and symbols spelled out, punctuation mapped, spacing fixed. The exact rules are model-specific (they must match what the model was trained on), so they live in the model's `tts` task template rather than in C++. + +Jinja has no regex support, and exposing `std::regex` / Python `re` would not help: the two differ in many details (`\d`, `\w`, `\b`, lookbehind, case folding, backtracking limits), so a template could give different results in llama.cpp and in Python. Instead, task templates get two functions with a small matching language whose semantics are defined here: + +- `tts_norm_scan(text, rules)`: split text into matched and unmatched segments +- `tts_norm_sub(text, pattern, repl)`: replace every match + +Implementations: +- C++: [common/chat-tts-norm.cpp](../common/chat-tts-norm.cpp), injected into task templates by `common_chat_template_render()` in [common/chat.cpp](../common/chat.cpp). They are not available to chat templates. +- Python reference: [scripts/chat_tts_norm.py](../scripts/chat_tts_norm.py), for use with `jinja2` (`env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan`) +- Both are checked against each other by `tests/test-jinja.cpp` (`test-jinja -py "tts norm patterns.*"`) + +## Example + +```jinja +{%- for s in tts_norm_scan(text, [ + ['money', '%$(%d+)(?:%.(%d%d))?'], + ['percent', '(%d+)%%'], + ['time', '(%d%d?):(%d%d)(?!%d)'], + ['neg', '(? bool: + return any(lo <= c <= hi for lo, hi in ranges) + + +def _is_alpha(c: int) -> bool: + if ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z"): + return True + if c in (0xAA, 0xB5, 0xBA): + return True + return c >= 0xC0 and not _in(TTS_NORM_SPACE, c) and not _in(TTS_NORM_PUNCT, c) and not _in(TTS_NORM_CONTROL, c) + + +def _class_match(kind: str, c: int) -> bool: + if kind == "d": + return ord("0") <= c <= ord("9") + if kind == "s": + return _in(TTS_NORM_SPACE, c) + if kind == "p": + return _in(TTS_NORM_PUNCT, c) + if kind == "a": + return _is_alpha(c) + if kind == "w": + return _is_alpha(c) or ord("0") <= c <= ord("9") + if kind == "u": + return ord("A") <= c <= ord("Z") + if kind == "l": + return ord("a") <= c <= ord("z") + if kind == "h": + return _in(TTS_NORM_HAN, c) + if kind == "c": + return _in(TTS_NORM_CONTROL, c) + return False + + +def _ascii_lower(c: int) -> int: + return c + 32 if ord("A") <= c <= ord("Z") else c + + +def _ascii_swapcase(c: int) -> int: + if ord("A") <= c <= ord("Z"): + return c + 32 + if ord("a") <= c <= ord("z"): + return c - 32 + return c + + +def _is_ascii_alnum(c: int) -> bool: + return ord("0") <= c <= ord("9") or ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z") + + +# AST + +LITERAL, SET, ANY, START, END, BOUNDARY, GROUP, LOOKAHEAD, LOOKBEHIND = range(9) +ZERO_WIDTH = (START, END, BOUNDARY, LOOKAHEAD, LOOKBEHIND) + + +@dataclass +class _Set: + neg: bool = False + ranges: list[tuple[int, int]] = field(default_factory=list) + classes: list[tuple[str, bool]] = field(default_factory=list) + + +@dataclass +class _Node: + op: int = GROUP + cp: int = 0 + set: _Set = field(default_factory=_Set) + alts: list[list[_Item]] = field(default_factory=list) + capture: int = -1 + neg: bool = False + + +@dataclass +class _Item: + atom: _Node + min: int = 1 + max: int = 1 + + +class PatternError(ValueError): + pass + + +class _Parser: + def __init__(self, s: list[int]): + self.s = s + self.p = 0 + self.n_caps = 0 + + def fail(self, msg: str): + raise PatternError(f"invalid TTS norm pattern: {msg} at position {self.p}") + + def eof(self) -> bool: + return self.p >= len(self.s) + + def peek_is(self, c: str) -> bool: + return not self.eof() and self.s[self.p] == ord(c) + + def expect(self, c: str, what: str): + if not self.peek_is(c): + self.fail(f"expected {what}") + self.p += 1 + + def parse_alts(self) -> list[list[_Item]]: + alts = [self.parse_seq()] + while self.peek_is("|"): + self.p += 1 + alts.append(self.parse_seq()) + return alts + + def parse_seq(self) -> list[_Item]: + seq: list[_Item] = [] + while not self.eof() and self.s[self.p] not in (ord("|"), ord(")")): + item = _Item(self.parse_atom()) + self.parse_quantifier(item) + seq.append(item) + return seq + + def parse_int(self) -> int: + if self.eof() or not (ord("0") <= self.s[self.p] <= ord("9")): + self.fail("expected number in quantifier") + v = 0 + while not self.eof() and ord("0") <= self.s[self.p] <= ord("9"): + v = v * 10 + self.s[self.p] - ord("0") + if v > MAX_REPEAT: + self.fail(f"repetition count exceeds {MAX_REPEAT}") + self.p += 1 + return v + + def parse_quantifier(self, item: _Item): + if self.eof(): + return + c = chr(self.s[self.p]) + if c not in "?*+{": + return + if item.atom.op in ZERO_WIDTH: + self.fail("zero-width assertion cannot be repeated") + self.p += 1 + if c == "?": + item.min, item.max = 0, 1 + elif c == "*": + item.min, item.max = 0, INF + elif c == "+": + item.min, item.max = 1, INF + else: + item.min = self.parse_int() + item.max = item.min + if self.peek_is(","): + self.p += 1 + item.max = INF if self.peek_is("}") else self.parse_int() + self.expect("}", "'}' to close quantifier") + if item.max < item.min: + self.fail("quantifier max is smaller than min") + if not self.eof() and chr(self.s[self.p]) in "?*+{": + self.fail("multiple quantifiers") + + def parse_group(self) -> _Node: + node = _Node(op=GROUP) + if self.peek_is("?"): + self.p += 1 + if self.eof(): + self.fail("unterminated group") + c = chr(self.s[self.p]) + if c == ":": + self.p += 1 + elif c in "=!": + self.p += 1 + node.op = LOOKAHEAD + node.neg = c == "!" + elif c == "<" and self.p + 1 < len(self.s) and self.s[self.p + 1] in (ord("="), ord("!")): + node.op = LOOKBEHIND + node.neg = self.s[self.p + 1] == ord("!") + self.p += 2 + if self.eof() or self.s[self.p] == ord(")"): + self.fail("lookbehind must contain exactly one codepoint matcher") + atom = self.parse_atom() + if atom.op not in (LITERAL, SET, ANY): + self.fail("lookbehind must contain exactly one codepoint matcher") + if not self.peek_is(")"): + self.fail("lookbehind must contain exactly one codepoint matcher") + self.p += 1 + node.alts = [[_Item(atom)]] + return node + elif c == "i": + self.fail("(?i) is only allowed at the start of the pattern") + else: + self.fail("unknown group type") + else: + node.capture = self.n_caps + self.n_caps += 1 + node.alts = self.parse_alts() + self.expect(")", "')' to close group") + return node + + # returns ("class", (kind, neg)) | ("boundary", None) | ("literal", cp) + def parse_escape(self, in_set: bool) -> tuple[str, Any]: + if self.eof(): + self.fail("dangling '%'") + e = self.s[self.p] + self.p += 1 + if e < 0x80 and chr(e).lower() in CLASS_LETTERS: + return "class", (chr(e).lower(), ord("A") <= e <= ord("Z")) + if e == ord("b") and not in_set: + return "boundary", None + if _is_ascii_alnum(e): + self.p -= 1 + self.fail(f"unknown escape '%{chr(e)}'") + return "literal", e + + def parse_set(self) -> _Node: + node = _Node(op=SET) + if self.peek_is("^"): + node.set.neg = True + self.p += 1 + empty = True + while True: + if self.eof(): + self.fail("unterminated set") + if self.s[self.p] == ord("]"): + self.p += 1 + break + empty = False + if self.s[self.p] == ord("%"): + self.p += 1 + kind, val = self.parse_escape(True) + if kind == "class": + node.set.classes.append(val) + continue + lo = val + else: + lo = self.s[self.p] + self.p += 1 + hi = lo + if self.peek_is("-") and self.p + 1 < len(self.s) and self.s[self.p + 1] != ord("]"): + self.p += 1 + if self.s[self.p] == ord("%"): + self.p += 1 + kind, val = self.parse_escape(True) + if kind == "class": + self.fail("character class cannot be used in a range") + hi = val + else: + hi = self.s[self.p] + self.p += 1 + if hi < lo: + self.fail("invalid range in set") + node.set.ranges.append((lo, hi)) + if empty: + self.fail("empty set") + return node + + def parse_atom(self) -> _Node: + c = chr(self.s[self.p]) + self.p += 1 + if c == "(": + return self.parse_group() + if c == "[": + return self.parse_set() + if c == ".": + return _Node(op=ANY) + if c == "^": + return _Node(op=START) + if c == "$": + return _Node(op=END) + if c == "%": + kind, val = self.parse_escape(False) + if kind == "class": + node = _Node(op=SET) + node.set.classes.append(val) + return node + if kind == "boundary": + return _Node(op=BOUNDARY) + return _Node(op=LITERAL, cp=val) + if c in "?*+{": + self.p -= 1 + self.fail("nothing to repeat") + if c in "]}": + self.p -= 1 + self.fail(f"unescaped '{c}', use '%{c}'") + return _Node(op=LITERAL, cp=ord(c)) + + +class _Matcher: + def __init__(self, t: list[int], icase: bool, n_groups: int): + self.t = t + self.icase = icase + self.caps: list[tuple[int, int]] = [(-1, -1)] * n_groups + + def is_w(self, i: int) -> bool: + return i < len(self.t) and _class_match("w", self.t[i]) + + def set_match(self, st: _Set, c: int) -> bool: + def test(x: int) -> bool: + if any(lo <= x <= hi for lo, hi in st.ranges): + return True + return any(_class_match(kind, x) != neg for kind, neg in st.classes) + + m = test(c) + if not m and self.icase: + o = _ascii_swapcase(c) + if o != c: + m = test(o) + return m != st.neg + + def match_alts(self, alts: list[list[_Item]], i: int) -> int: + """returns the end position, or -1 if no alternative matches""" + for seq in alts: + saved = list(self.caps) + end = self.match_seq(seq, i) + if end >= 0: + return end + self.caps = saved + return -1 + + def match_seq(self, seq: list[_Item], i: int) -> int: + pos = i + for it in seq: + count = 0 + while count < it.max: + is_group = it.atom.op == GROUP + saved = list(self.caps) if is_group else None + e = self.match_once(it.atom, pos) + if e < 0: + if saved is not None: + self.caps = saved + break + if e == pos: + # an empty iteration would repeat forever, it satisfies any remaining minimum + count = max(count + 1, it.min) + break + pos = e + count += 1 + if count < it.min: + return -1 + return pos + + def match_once(self, node: _Node, i: int) -> int: + """returns the end position, or -1 on failure""" + n = len(self.t) + op = node.op + if op == LITERAL: + if i < n and (self.t[i] == node.cp or (self.icase and _ascii_lower(self.t[i]) == _ascii_lower(node.cp))): + return i + 1 + return -1 + if op == SET: + return i + 1 if i < n and self.set_match(node.set, self.t[i]) else -1 + if op == ANY: + return i + 1 if i < n else -1 + if op == START: + return i if i == 0 else -1 + if op == END: + return i if i == n else -1 + if op == BOUNDARY: + return i if (i > 0 and self.is_w(i - 1)) != self.is_w(i) else -1 + if op == GROUP: + e = self.match_alts(node.alts, i) + if e < 0: + return -1 + if node.capture >= 0: + self.caps[node.capture] = (i, e) + return e + if op == LOOKAHEAD: + # captures made inside a lookahead are discarded + saved = list(self.caps) + ok = self.match_alts(node.alts, i) >= 0 + self.caps = saved + return i if ok != node.neg else -1 + if op == LOOKBEHIND: + ok = i > 0 and self.match_once(node.alts[0][0].atom, i - 1) == i + return i if ok != node.neg else -1 + return -1 + + +class Pattern: + def __init__(self, src: str): + if len(src.encode("utf-8")) > MAX_PATTERN_LEN: + raise PatternError(f"invalid TTS norm pattern: pattern is longer than {MAX_PATTERN_LEN} bytes") + self.icase = False + parser = _Parser([ord(c) for c in src]) + if src.startswith("(?i)"): + self.icase = True + parser.p = 4 + self.root = _Node(op=GROUP, alts=parser.parse_alts()) + if not parser.eof(): + parser.fail("unbalanced ')'") + self.n_caps = parser.n_caps + + def match_at(self, t: list[int], pos: int) -> tuple[int, list[tuple[int, int]]] | None: + m = _Matcher(t, self.icase, self.n_caps) + end = m.match_alts(self.root.alts, pos) + if end < 0: + return None + return end, m.caps + + +# (rule index or -1, start, end, groups) with codepoint offsets +Segment = tuple[int, int, int, list[tuple[int, int]]] + + +def scan_segments(text: str, rules: list[Pattern]) -> list[Segment]: + t = [ord(c) for c in text] + n = len(t) + out: list[Segment] = [] + i = 0 + pending = 0 + while i < n: + matched = False + for r, rule in enumerate(rules): + res = rule.match_at(t, i) + if res is not None and res[0] > i: + end, groups = res + if pending < i: + out.append((-1, pending, i, [])) + out.append((r, i, end, groups)) + i = end + pending = i + matched = True + break + if not matched: + i += 1 + if pending < n: + out.append((-1, pending, n, [])) + return out + + +def _rules_from_value(rules: Any) -> list[tuple[str, str]]: + if isinstance(rules, str): + return [("match", rules)] + if isinstance(rules, dict): + out = [] + for k, v in rules.items(): + if not isinstance(v, str): + raise TypeError("tts_norm_scan: rule pattern must be a string") + out.append((str(k), v)) + return out + if isinstance(rules, (list, tuple)): + out = [] + for item in rules: + if not isinstance(item, (list, tuple)) or len(item) != 2 or not isinstance(item[0], str) or not isinstance(item[1], str): + raise TypeError("tts_norm_scan: each rule must be a [name, pattern] pair of strings") + out.append((item[0], item[1])) + return out + raise TypeError("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs") + + +def tts_norm_scan(text: str, rules: Any) -> list[dict[str, Any]]: + rule_srcs = _rules_from_value(rules) + patterns = [Pattern(src) for _, src in rule_srcs] + out: list[dict[str, Any]] = [] + for r, start, end, groups in scan_segments(text, patterns): + # groups that did not participate are empty strings, so they render as nothing and are falsy + vals = ["" if gs < 0 else text[gs:ge] for gs, ge in groups] + out.append({"rule": rule_srcs[r][0] if r >= 0 else None, "text": text[start:end], "groups": vals}) + return out + + +def _parse_repl(repl: str, pattern: Pattern) -> list[str | int]: + pieces: list[str | int] = [] + lit = "" + i = 0 + while i < len(repl): + c = repl[i] + if c != "%": + lit += c + i += 1 + continue + if i + 1 >= len(repl): + raise PatternError("invalid TTS norm replacement: dangling '%'") + i += 1 + c = repl[i] + if c == "%": + lit += "%" + elif "0" <= c <= "9": + g = ord(c) - ord("0") + if g > pattern.n_caps: + raise PatternError(f"invalid TTS norm replacement: group %{c} does not exist") + if lit: + pieces.append(lit) + lit = "" + pieces.append(g) + else: + raise PatternError(f"invalid TTS norm replacement: unknown escape '%{c}'") + i += 1 + if lit: + pieces.append(lit) + return pieces + + +def tts_norm_sub(text: str, pattern: str, repl: str) -> str: + pat = Pattern(pattern) + pieces = _parse_repl(repl, pat) + out = [] + for r, start, end, groups in scan_segments(text, [pat]): + if r < 0: + out.append(text[start:end]) + continue + for piece in pieces: + if isinstance(piece, str): + out.append(piece) + elif piece == 0: + out.append(text[start:end]) + else: + gs, ge = groups[piece - 1] + if gs >= 0: + out.append(text[gs:ge]) + return "".join(out) diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp index ce5a9749b..29de01ea9 100644 --- a/tests/test-jinja.cpp +++ b/tests/test-jinja.cpp @@ -3,6 +3,7 @@ #include #include +#include "chat-tts-norm.h" #include "json.h" #include "subproc.h" @@ -16,8 +17,8 @@ using json = common_json; -static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect); -static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect); +static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm = false); +static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm = false); static void test_whitespace_control(testing & t); static void test_conditionals(testing & t); @@ -39,6 +40,7 @@ static void test_stats(testing & t); static void test_caps(testing & t); static void test_string_parts(testing & t); static void test_fuzzing(testing & t); +static void test_tts_norm_patterns(testing & t); static bool g_python_mode = false; @@ -75,6 +77,7 @@ int main(int argc, char *argv[]) { t.test("string methods", test_string_methods); t.test("array methods", test_array_methods); t.test("object methods", test_object_methods); + t.test("tts norm patterns", test_tts_norm_patterns); if (!g_python_mode) { t.test("hasher", test_hasher); t.test("stats", test_stats); @@ -2713,13 +2716,134 @@ static void test_string_parts(testing & t) { } -static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect) { - t.test(name, [&tmpl, &expect](testing & t) { +// TTS normalization patterns: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md +// these functions are only available to task templates (e.g. "tts"), so every test here passes tts_norm = true +static void test_tts_norm_patterns(testing & t) { + test_template(t, "not available without tts norm builtins", + "{{ tts_norm_scan is defined }}", + json::object(), + "False"); + + test_template(t, "tts_norm_scan splits text into matched and unmatched segments", + "{%- for s in tts_norm_scan(text, [['num', '%d+'], ['word', '%a+']]) -%}[{{ s.rule or '-' }}:{{ s.text }}]{%- endfor -%}", + {{"text", "abc 123, x9"}}, + "[word:abc][-: ][num:123][-:, ][word:x][num:9]", true); + + test_template(t, "tts_norm_scan groups, optional group that did not match is empty", + "{%- for s in tts_norm_scan(text, [['money', '%$(%d+)(?:%.(%d%d))?']]) -%}" + "{%- if s.rule -%}{%- set whole, cents = s.groups -%}{{ whole }}|{{ cents }}|{{ s.groups | length }};{%- endif -%}" + "{%- endfor -%}", + {{"text", "$3.50 and $7"}}, + "3|50|2;7||2;", true); + + test_template(t, "tts_norm_scan rules as dict, the first rule that matches wins", + "{%- for s in tts_norm_scan(text, {'ordinal': '%d+(?:st|nd|rd|th)%b', 'number': '%d+'}) -%}" + "{%- if s.rule -%}<{{ s.rule }}:{{ s.text }}>{%- else -%}{{ s.text }}{%- endif -%}" + "{%- endfor -%}", + {{"text", "the 3rd of 10 or 4three"}}, + "the of or three", true); + + // unlike regex, a failed sequence does not retry the other branches: (?:a|ab)c never matches "abc" + test_template(t, "ordered alternation commits to the first branch that matches", + "{%- for s in tts_norm_scan(text, '(?:a|ab)c') -%}[{{ s.rule or '-' }}:{{ s.text }}]{%- endfor -%}", + {{"text", "abc ac"}}, + "[-:abc ][match:ac]", true); + + // unlike regex, quantifiers never give back what they consumed: %d+0 never matches + test_template(t, "quantifiers are possessive", + "{{ tts_norm_sub(text, '%d+0', 'X') }}|{{ tts_norm_sub(text, '%d+', 'X') }}", + {{"text", "100 20"}}, + "100 20|X X", true); + + test_template(t, "quantifier ranges", + "{{ tts_norm_sub(text, '%d{3}-%d{2,}', '#') }}", + {{"text", "123-4 123-45 123-456789"}}, + "123-4 # #", true); + + test_template(t, "lookbehind: a minus sign is not part of a word", + "{{ tts_norm_sub(text, '(?') }}|{{ tts_norm_sub(text, '[^%w%s]', '') }}", + {{"text", "€ 5 & $6!"}}, + "<5> & <6>!| 5 6", true); + + test_template(t, "remove spaces between Han characters", + "{{ tts_norm_sub(text, '(%h) +(%h)', '%1%2') }}", + {{"text", "你 好 world 世界"}}, + "你好 world 世界", true); + + test_template(t, "mini text normalizer", + "{%- set ones = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten'," + " 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'] -%}\n" + "{%- set tens = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'] -%}\n" + "{%- macro num(n) -%}{{ ones[n] if n < 20 else tens[(n / 10) | int] ~ ('-' ~ ones[n % 10] if n % 10 else '') }}{%- endmacro -%}\n" + "{%- for s in tts_norm_scan(text, [\n" + " ['money', '%$(%d+)(?:%.(%d%d))?'],\n" + " ['percent', '(%d+)%%'],\n" + " ['time', '(%d%d?):(%d%d)(?!%d)'],\n" + " ['neg', '(?a)') }}", "unknown group type", true); + test_template_error(t, "error: wide lookbehind", "{{ tts_norm_scan('x', '(?<=ab)c') }}", "lookbehind must contain exactly one codepoint matcher", true); + test_template_error(t, "error: missing group in replacement", "{{ tts_norm_sub('x', '(a)', '%2') }}", "group %2 does not exist", true); + test_template_error(t, "error: bad rules", "{{ tts_norm_scan('x', 1) }}", "rules must be a string, a dict or a list", true); + } +} + +static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm) { + t.test(name, [&tmpl, &expect, tts_norm](testing & t) { std::string error; try { jinja::lexer lexer; jinja::program ast = jinja::parse_from_tokens(lexer.tokenize(tmpl)); jinja::context ctx(tmpl); + if (tts_norm) { + common_chat_tts_norm_add_builtins(ctx); + } jinja::runtime runtime(ctx); runtime.gather_string_parts(runtime.execute(ast)); } catch (const std::exception & e) { @@ -2729,14 +2853,17 @@ static void test_template_error(testing & t, const std::string & name, const std }); } -static void test_template_cpp(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) { - t.test(name, [&tmpl, &vars, &expect](testing & t) { +static void test_template_cpp(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) { + t.test(name, [&tmpl, &vars, &expect, tts_norm](testing & t) { jinja::lexer lexer; auto lexer_res = lexer.tokenize(tmpl); jinja::program ast = jinja::parse_from_tokens(lexer_res); jinja::context ctx(tmpl); + if (tts_norm) { + common_chat_tts_norm_add_builtins(ctx); + } jinja::global_from_json(ctx, vars, true); jinja::runtime runtime(ctx); @@ -2788,17 +2915,36 @@ env.filters["tojson"] = lambda x, ensure_ascii=False, indent=None, separators=No env.globals["strftime_now"] = lambda format: datetime.now().strftime(format) env.globals["raise_exception"] = raise_exception +if merged_input.get("tts_norm_ref"): + import importlib.util + spec = importlib.util.spec_from_file_location("chat_tts_norm", merged_input["tts_norm_ref"]) + chat_tts_norm = importlib.util.module_from_spec(spec) + sys.modules["chat_tts_norm"] = chat_tts_norm # required by dataclasses + spec.loader.exec_module(chat_tts_norm) + env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan + env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub + template = env.from_string(tmpl) result = template.render(**vars_json) sys.stdout.buffer.write(result.encode()) )"; -static void test_template_py(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) { - t.test(name, [&tmpl, &vars, &expect](testing & t) { +// python reference of the TTS normalization patterns, relative to this file +static std::string tts_norm_ref_path() { + std::string path = __FILE__; + const size_t pos = path.find_last_of("/\\"); + return (pos == std::string::npos ? std::string(".") : path.substr(0, pos)) + "/../scripts/chat_tts_norm.py"; +} + +static void test_template_py(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) { + t.test(name, [&tmpl, &vars, &expect, tts_norm](testing & t) { // Prepare arguments json merged; merged["tmpl"] = json(tmpl); merged["vars"] = vars; + if (tts_norm) { + merged["tts_norm_ref"] = tts_norm_ref_path(); + } #ifdef _WIN32 const char * python_executable = "python.exe"; @@ -2859,11 +3005,11 @@ static void test_template_py(testing & t, const std::string & name, const std::s }); } -static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) { +static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) { if (g_python_mode) { - test_template_py(t, name, tmpl, vars, expect); + test_template_py(t, name, tmpl, vars, expect, tts_norm); } else { - test_template_cpp(t, name, tmpl, vars, expect); + test_template_cpp(t, name, tmpl, vars, expect, tts_norm); } }