chat: allow TTS model to provide custom text normalizer via jinja (#32053)
* chat: allow TTS model to provide custom text normalizer via jinja * rm tts_norm_is_reserved_name
This commit is contained in:
1 parent
956591f457
commit
f7b886fe6f
7 files changed
+1769
-11
No files matched your search
@@ -66,6 +66,8 @@ add_library(${TARGET}
|
||||
chat-diff-analyzer.cpp
|
||||
chat-peg-parser.cpp
|
||||
chat-peg-parser.h
|
||||
chat-tts-norm.cpp
|
||||
chat-tts-norm.h
|
||||
chat.cpp
|
||||
chat.h
|
||||
common.cpp
|
||||
|
||||
@@ -0,0 +1,864 @@
|
||||
#include "chat-tts-norm.h"
|
||||
|
||||
#include "jinja/runtime.h"
|
||||
#include "jinja/value.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <climits>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// TTS normalization patterns, see docs/tts-norm-patterns.md for the specification
|
||||
// keep this file in sync with the python reference: scripts/chat_tts_norm.py
|
||||
|
||||
using namespace jinja;
|
||||
|
||||
//
|
||||
// character classes
|
||||
// they are defined with explicit codepoint ranges, independent of any unicode database version,
|
||||
// so that the C++ and python implementations agree on every codepoint
|
||||
//
|
||||
|
||||
struct tts_norm_range {
|
||||
uint32_t lo;
|
||||
uint32_t hi;
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_SPACE[] = {
|
||||
{0x0009, 0x000D}, {0x0020, 0x0020}, {0x0085, 0x0085}, {0x00A0, 0x00A0}, {0x1680, 0x1680},
|
||||
{0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F}, {0x205F, 0x205F}, {0x3000, 0x3000},
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_PUNCT[] = {
|
||||
{0x0021, 0x002F}, {0x003A, 0x0040}, {0x005B, 0x0060}, {0x007B, 0x007E},
|
||||
{0x00A1, 0x00A9}, {0x00AB, 0x00B4}, {0x00B6, 0x00B9}, {0x00BB, 0x00BF}, {0x00D7, 0x00D7}, {0x00F7, 0x00F7},
|
||||
{0x2010, 0x2027}, {0x2030, 0x205E}, {0x20A0, 0x20CF}, {0x2100, 0x218F}, {0x2190, 0x2BFF},
|
||||
{0x3001, 0x3003}, {0x3008, 0x3020}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30FB, 0x30FB},
|
||||
{0xFE10, 0xFE19}, {0xFE30, 0xFE4F}, {0xFE50, 0xFE6B},
|
||||
{0xFF01, 0xFF0F}, {0xFF1A, 0xFF20}, {0xFF3B, 0xFF40}, {0xFF5B, 0xFF65},
|
||||
{0x1F000, 0x1FAFF},
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_CONTROL[] = {
|
||||
{0x0000, 0x001F}, {0x007F, 0x009F}, {0x200B, 0x200F}, {0x2060, 0x206F}, {0xFEFF, 0xFEFF},
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_HAN[] = {
|
||||
{0x2E80, 0x2FDF}, {0x3005, 0x3005}, {0x3007, 0x3007}, {0x3021, 0x3029}, {0x3038, 0x303B},
|
||||
{0x3400, 0x4DBF}, {0x4E00, 0x9FFF}, {0xF900, 0xFAFF}, {0x20000, 0x323AF},
|
||||
};
|
||||
|
||||
template<size_t N>
|
||||
static bool tts_norm_in(const tts_norm_range (&ranges)[N], uint32_t c) {
|
||||
for (const auto & r : ranges) {
|
||||
if (c >= r.lo && c <= r.hi) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool tts_norm_is_alpha(uint32_t c) {
|
||||
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
|
||||
return true;
|
||||
}
|
||||
if (c == 0xAA || c == 0xB5 || c == 0xBA) {
|
||||
return true;
|
||||
}
|
||||
return c >= 0xC0 && !tts_norm_in(TTS_NORM_SPACE, c) && !tts_norm_in(TTS_NORM_PUNCT, c) && !tts_norm_in(TTS_NORM_CONTROL, c);
|
||||
}
|
||||
|
||||
// kind is one of "dspawulhc"
|
||||
static bool tts_norm_class_match(char kind, uint32_t c) {
|
||||
switch (kind) {
|
||||
case 'd': return c >= '0' && c <= '9';
|
||||
case 's': return tts_norm_in(TTS_NORM_SPACE, c);
|
||||
case 'p': return tts_norm_in(TTS_NORM_PUNCT, c);
|
||||
case 'a': return tts_norm_is_alpha(c);
|
||||
case 'w': return tts_norm_is_alpha(c) || (c >= '0' && c <= '9');
|
||||
case 'u': return c >= 'A' && c <= 'Z';
|
||||
case 'l': return c >= 'a' && c <= 'z';
|
||||
case 'h': return tts_norm_in(TTS_NORM_HAN, c);
|
||||
case 'c': return tts_norm_in(TTS_NORM_CONTROL, c);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool tts_norm_is_class_letter(uint32_t c) {
|
||||
if (c >= 'A' && c <= 'Z') {
|
||||
c += 'a' - 'A';
|
||||
}
|
||||
return c < 0x80 && std::string("dspawulhc").find(static_cast<char>(c)) != std::string::npos;
|
||||
}
|
||||
|
||||
static uint32_t tts_norm_ascii_lower(uint32_t c) {
|
||||
return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c;
|
||||
}
|
||||
|
||||
static uint32_t tts_norm_ascii_swapcase(uint32_t c) {
|
||||
if (c >= 'A' && c <= 'Z') return c + ('a' - 'A');
|
||||
if (c >= 'a' && c <= 'z') return c - ('a' - 'A');
|
||||
return c;
|
||||
}
|
||||
|
||||
static bool tts_norm_is_ascii_alnum(uint32_t c) {
|
||||
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
|
||||
}
|
||||
|
||||
// invalid bytes are decoded as U+FFFD, one per byte; offs[i] is the byte offset of codepoint i, offs[n] = text size
|
||||
static void tts_norm_decode_utf8(const std::string & s, std::vector<uint32_t> & cpts, std::vector<size_t> & offs) {
|
||||
cpts.clear();
|
||||
offs.clear();
|
||||
size_t i = 0;
|
||||
const size_t n = s.size();
|
||||
while (i < n) {
|
||||
offs.push_back(i);
|
||||
const unsigned char c = static_cast<unsigned char>(s[i]);
|
||||
uint32_t cp = 0;
|
||||
size_t len = 0;
|
||||
if (c < 0x80) {
|
||||
cp = c; len = 1;
|
||||
} else if ((c & 0xE0) == 0xC0) {
|
||||
cp = c & 0x1F; len = 2;
|
||||
} else if ((c & 0xF0) == 0xE0) {
|
||||
cp = c & 0x0F; len = 3;
|
||||
} else if ((c & 0xF8) == 0xF0) {
|
||||
cp = c & 0x07; len = 4;
|
||||
}
|
||||
bool ok = len > 0 && i + len <= n;
|
||||
for (size_t k = 1; ok && k < len; k++) {
|
||||
const unsigned char cc = static_cast<unsigned char>(s[i + k]);
|
||||
if ((cc & 0xC0) != 0x80) {
|
||||
ok = false;
|
||||
} else {
|
||||
cp = (cp << 6) | (cc & 0x3F);
|
||||
}
|
||||
}
|
||||
if (!ok) {
|
||||
cpts.push_back(0xFFFD);
|
||||
i += 1;
|
||||
} else {
|
||||
cpts.push_back(cp);
|
||||
i += len;
|
||||
}
|
||||
}
|
||||
offs.push_back(n);
|
||||
}
|
||||
|
||||
//
|
||||
// AST
|
||||
//
|
||||
|
||||
enum class tts_norm_op {
|
||||
LITERAL, // one codepoint
|
||||
SET, // one codepoint from a set; class escapes like %d are sets with a single class
|
||||
ANY, // any codepoint
|
||||
START, // ^
|
||||
END, // $
|
||||
BOUNDARY, // %b
|
||||
GROUP, // (...), (?:...), (?<name>...)
|
||||
LOOKAHEAD, // (?=...), (?!...)
|
||||
LOOKBEHIND, // (?<=c), (?<!c) - exactly one codepoint
|
||||
};
|
||||
|
||||
struct tts_norm_class {
|
||||
char kind;
|
||||
bool neg;
|
||||
};
|
||||
|
||||
struct tts_norm_set {
|
||||
bool neg = false;
|
||||
std::vector<std::pair<uint32_t, uint32_t>> ranges;
|
||||
std::vector<tts_norm_class> classes;
|
||||
};
|
||||
|
||||
struct tts_norm_item;
|
||||
using tts_norm_seq = std::vector<tts_norm_item>;
|
||||
|
||||
struct tts_norm_pattern_node {
|
||||
tts_norm_op op = tts_norm_op::GROUP;
|
||||
uint32_t cp = 0; // LITERAL
|
||||
tts_norm_set set; // SET
|
||||
std::vector<tts_norm_seq> alts; // GROUP, LOOKAHEAD; LOOKBEHIND holds a single item
|
||||
int capture = -1; // GROUP: capture index, -1 for non-capturing
|
||||
bool neg = false; // LOOKAHEAD, LOOKBEHIND
|
||||
};
|
||||
|
||||
static constexpr int TTS_NORM_INF = INT_MAX;
|
||||
static constexpr int TTS_NORM_MAX_REPEAT = 1000;
|
||||
static constexpr size_t TTS_NORM_MAX_PATTERN_LEN = 4096;
|
||||
|
||||
struct tts_norm_item {
|
||||
tts_norm_pattern_node atom;
|
||||
int min = 1;
|
||||
int max = 1;
|
||||
};
|
||||
|
||||
static bool tts_norm_is_zero_width(tts_norm_op op) {
|
||||
return op == tts_norm_op::START || op == tts_norm_op::END || op == tts_norm_op::BOUNDARY
|
||||
|| op == tts_norm_op::LOOKAHEAD || op == tts_norm_op::LOOKBEHIND;
|
||||
}
|
||||
|
||||
//
|
||||
// parser
|
||||
//
|
||||
|
||||
struct tts_norm_parser {
|
||||
std::vector<uint32_t> s;
|
||||
size_t p = 0;
|
||||
size_t & n_caps;
|
||||
|
||||
tts_norm_parser(std::vector<uint32_t> src, size_t & n_caps) : s(std::move(src)), n_caps(n_caps) {}
|
||||
|
||||
[[noreturn]] void fail(const std::string & msg) const {
|
||||
throw std::runtime_error("invalid TTS norm pattern: " + msg + " at position " + std::to_string(p));
|
||||
}
|
||||
|
||||
bool eof() const { return p >= s.size(); }
|
||||
uint32_t peek() const { return s[p]; }
|
||||
bool peek_is(uint32_t c) const { return !eof() && s[p] == c; }
|
||||
|
||||
void expect(uint32_t c, const char * what) {
|
||||
if (!peek_is(c)) {
|
||||
fail(std::string("expected ") + what);
|
||||
}
|
||||
p++;
|
||||
}
|
||||
|
||||
std::vector<tts_norm_seq> parse_alts() {
|
||||
std::vector<tts_norm_seq> alts;
|
||||
alts.push_back(parse_seq());
|
||||
while (peek_is('|')) {
|
||||
p++;
|
||||
alts.push_back(parse_seq());
|
||||
}
|
||||
return alts;
|
||||
}
|
||||
|
||||
tts_norm_seq parse_seq() {
|
||||
tts_norm_seq seq;
|
||||
while (!eof() && peek() != '|' && peek() != ')') {
|
||||
tts_norm_item item;
|
||||
item.atom = parse_atom();
|
||||
parse_quantifier(item);
|
||||
seq.push_back(std::move(item));
|
||||
}
|
||||
return seq;
|
||||
}
|
||||
|
||||
int parse_int() {
|
||||
if (eof() || peek() < '0' || peek() > '9') {
|
||||
fail("expected number in quantifier");
|
||||
}
|
||||
int v = 0;
|
||||
while (!eof() && peek() >= '0' && peek() <= '9') {
|
||||
v = v * 10 + static_cast<int>(peek() - '0');
|
||||
if (v > TTS_NORM_MAX_REPEAT) {
|
||||
fail("repetition count exceeds " + std::to_string(TTS_NORM_MAX_REPEAT));
|
||||
}
|
||||
p++;
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
void parse_quantifier(tts_norm_item & item) {
|
||||
if (eof()) {
|
||||
return;
|
||||
}
|
||||
const uint32_t c = peek();
|
||||
if (c != '?' && c != '*' && c != '+' && c != '{') {
|
||||
return;
|
||||
}
|
||||
if (tts_norm_is_zero_width(item.atom.op)) {
|
||||
fail("zero-width assertion cannot be repeated");
|
||||
}
|
||||
p++;
|
||||
if (c == '?') {
|
||||
item.min = 0; item.max = 1;
|
||||
} else if (c == '*') {
|
||||
item.min = 0; item.max = TTS_NORM_INF;
|
||||
} else if (c == '+') {
|
||||
item.min = 1; item.max = TTS_NORM_INF;
|
||||
} else {
|
||||
item.min = parse_int();
|
||||
item.max = item.min;
|
||||
if (peek_is(',')) {
|
||||
p++;
|
||||
item.max = peek_is('}') ? TTS_NORM_INF : parse_int();
|
||||
}
|
||||
expect('}', "'}' to close quantifier");
|
||||
if (item.max < item.min) {
|
||||
fail("quantifier max is smaller than min");
|
||||
}
|
||||
}
|
||||
if (!eof() && (peek() == '?' || peek() == '*' || peek() == '+' || peek() == '{')) {
|
||||
fail("multiple quantifiers");
|
||||
}
|
||||
}
|
||||
|
||||
tts_norm_pattern_node parse_group() {
|
||||
// '(' already consumed
|
||||
tts_norm_pattern_node node;
|
||||
node.op = tts_norm_op::GROUP;
|
||||
if (peek_is('?')) {
|
||||
p++;
|
||||
if (eof()) {
|
||||
fail("unterminated group");
|
||||
}
|
||||
const uint32_t c = peek();
|
||||
if (c == ':') {
|
||||
p++;
|
||||
} else if (c == '=' || c == '!') {
|
||||
p++;
|
||||
node.op = tts_norm_op::LOOKAHEAD;
|
||||
node.neg = c == '!';
|
||||
} else if (c == '<' && p + 1 < s.size() && (s[p + 1] == '=' || s[p + 1] == '!')) {
|
||||
node.op = tts_norm_op::LOOKBEHIND;
|
||||
node.neg = s[p + 1] == '!';
|
||||
p += 2;
|
||||
tts_norm_item item;
|
||||
if (eof() || peek() == ')') {
|
||||
fail("lookbehind must contain exactly one codepoint matcher");
|
||||
}
|
||||
item.atom = parse_atom();
|
||||
if (item.atom.op != tts_norm_op::LITERAL && item.atom.op != tts_norm_op::SET && item.atom.op != tts_norm_op::ANY) {
|
||||
fail("lookbehind must contain exactly one codepoint matcher");
|
||||
}
|
||||
if (!peek_is(')')) {
|
||||
fail("lookbehind must contain exactly one codepoint matcher");
|
||||
}
|
||||
p++;
|
||||
node.alts.push_back(tts_norm_seq{});
|
||||
node.alts[0].push_back(std::move(item));
|
||||
return node;
|
||||
} else if (c == 'i') {
|
||||
fail("(?i) is only allowed at the start of the pattern");
|
||||
} else {
|
||||
fail("unknown group type");
|
||||
}
|
||||
} else {
|
||||
node.capture = static_cast<int>(n_caps++);
|
||||
}
|
||||
node.alts = parse_alts();
|
||||
expect(')', "')' to close group");
|
||||
return node;
|
||||
}
|
||||
|
||||
// after '%' inside or outside of a set
|
||||
// returns true if it is a class (stored in cls), false if it is a literal (stored in lit)
|
||||
bool parse_escape(bool in_set, tts_norm_class & cls, uint32_t & lit, bool & boundary) {
|
||||
boundary = false;
|
||||
if (eof()) {
|
||||
fail("dangling '%'");
|
||||
}
|
||||
const uint32_t e = s[p++];
|
||||
if (tts_norm_is_class_letter(e)) {
|
||||
cls.kind = static_cast<char>(tts_norm_ascii_lower(e));
|
||||
cls.neg = e >= 'A' && e <= 'Z';
|
||||
return true;
|
||||
}
|
||||
if (e == 'b' && !in_set) {
|
||||
boundary = true;
|
||||
return false;
|
||||
}
|
||||
if (tts_norm_is_ascii_alnum(e)) {
|
||||
p--;
|
||||
fail(std::string("unknown escape '%") + static_cast<char>(e) + "'");
|
||||
}
|
||||
lit = e;
|
||||
return false;
|
||||
}
|
||||
|
||||
tts_norm_pattern_node parse_set() {
|
||||
// '[' already consumed
|
||||
tts_norm_pattern_node node;
|
||||
node.op = tts_norm_op::SET;
|
||||
if (peek_is('^')) {
|
||||
node.set.neg = true;
|
||||
p++;
|
||||
}
|
||||
bool empty = true;
|
||||
while (true) {
|
||||
if (eof()) {
|
||||
fail("unterminated set");
|
||||
}
|
||||
if (peek() == ']') {
|
||||
p++;
|
||||
break;
|
||||
}
|
||||
empty = false;
|
||||
uint32_t lo = 0;
|
||||
if (peek() == '%') {
|
||||
p++;
|
||||
tts_norm_class cls{};
|
||||
bool boundary = false;
|
||||
if (parse_escape(true, cls, lo, boundary)) {
|
||||
node.set.classes.push_back(cls);
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
lo = s[p++];
|
||||
}
|
||||
uint32_t hi = lo;
|
||||
if (peek_is('-') && p + 1 < s.size() && s[p + 1] != ']') {
|
||||
p++;
|
||||
if (peek() == '%') {
|
||||
p++;
|
||||
tts_norm_class cls{};
|
||||
bool boundary = false;
|
||||
if (parse_escape(true, cls, hi, boundary)) {
|
||||
fail("character class cannot be used in a range");
|
||||
}
|
||||
} else {
|
||||
hi = s[p++];
|
||||
}
|
||||
if (hi < lo) {
|
||||
fail("invalid range in set");
|
||||
}
|
||||
}
|
||||
node.set.ranges.emplace_back(lo, hi);
|
||||
}
|
||||
if (empty) {
|
||||
fail("empty set");
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
tts_norm_pattern_node parse_atom() {
|
||||
const uint32_t c = s[p++];
|
||||
tts_norm_pattern_node node;
|
||||
switch (c) {
|
||||
case '(':
|
||||
return parse_group();
|
||||
case '[':
|
||||
return parse_set();
|
||||
case '.':
|
||||
node.op = tts_norm_op::ANY;
|
||||
return node;
|
||||
case '^':
|
||||
node.op = tts_norm_op::START;
|
||||
return node;
|
||||
case '$':
|
||||
node.op = tts_norm_op::END;
|
||||
return node;
|
||||
case '%': {
|
||||
tts_norm_class cls{};
|
||||
uint32_t lit = 0;
|
||||
bool boundary = false;
|
||||
if (parse_escape(false, cls, lit, boundary)) {
|
||||
node.op = tts_norm_op::SET;
|
||||
node.set.classes.push_back(cls);
|
||||
} else if (boundary) {
|
||||
node.op = tts_norm_op::BOUNDARY;
|
||||
} else {
|
||||
node.op = tts_norm_op::LITERAL;
|
||||
node.cp = lit;
|
||||
}
|
||||
return node;
|
||||
}
|
||||
case '?':
|
||||
case '*':
|
||||
case '+':
|
||||
case '{':
|
||||
p--;
|
||||
fail("nothing to repeat");
|
||||
case ']':
|
||||
case '}':
|
||||
p--;
|
||||
fail(std::string("unescaped '") + static_cast<char>(c) + "', use '%" + static_cast<char>(c) + "'");
|
||||
default:
|
||||
node.op = tts_norm_op::LITERAL;
|
||||
node.cp = c;
|
||||
return node;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// matcher
|
||||
// PEG-style: alternatives are ordered (the first one that matches wins) and quantifiers are greedy and possessive
|
||||
// (they never give back what they consumed), so there is no backtracking
|
||||
//
|
||||
|
||||
struct tts_norm_matcher {
|
||||
const std::vector<uint32_t> & t;
|
||||
bool icase;
|
||||
std::vector<std::pair<int64_t, int64_t>> caps;
|
||||
|
||||
bool is_w(size_t i) const {
|
||||
return i < t.size() && tts_norm_class_match('w', t[i]);
|
||||
}
|
||||
|
||||
bool set_match(const tts_norm_set & st, uint32_t c) const {
|
||||
auto test = [&](uint32_t x) {
|
||||
for (const auto & r : st.ranges) {
|
||||
if (x >= r.first && x <= r.second) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
for (const auto & k : st.classes) {
|
||||
if (tts_norm_class_match(k.kind, x) != k.neg) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
bool m = test(c);
|
||||
if (!m && icase) {
|
||||
const uint32_t o = tts_norm_ascii_swapcase(c);
|
||||
if (o != c) {
|
||||
m = test(o);
|
||||
}
|
||||
}
|
||||
return m != st.neg;
|
||||
}
|
||||
|
||||
bool match_alts(const std::vector<tts_norm_seq> & alts, size_t i, size_t & end) {
|
||||
for (const auto & seq : alts) {
|
||||
auto saved = caps;
|
||||
if (match_seq(seq, i, end)) {
|
||||
return true;
|
||||
}
|
||||
caps = std::move(saved);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool match_seq(const tts_norm_seq & seq, size_t i, size_t & end) {
|
||||
size_t pos = i;
|
||||
for (const auto & it : seq) {
|
||||
int count = 0;
|
||||
while (count < it.max) {
|
||||
const bool is_group = it.atom.op == tts_norm_op::GROUP;
|
||||
std::vector<std::pair<int64_t, int64_t>> saved;
|
||||
if (is_group) {
|
||||
saved = caps;
|
||||
}
|
||||
size_t e = pos;
|
||||
if (!match_once(it.atom, pos, e)) {
|
||||
if (is_group) {
|
||||
caps = std::move(saved);
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (e == pos) {
|
||||
// an empty iteration would repeat forever, it satisfies any remaining minimum
|
||||
count = std::max(count + 1, it.min);
|
||||
break;
|
||||
}
|
||||
pos = e;
|
||||
count++;
|
||||
}
|
||||
if (count < it.min) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
end = pos;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool match_once(const tts_norm_pattern_node & node, size_t i, size_t & end) {
|
||||
const size_t n = t.size();
|
||||
switch (node.op) {
|
||||
case tts_norm_op::LITERAL:
|
||||
if (i < n && (t[i] == node.cp || (icase && tts_norm_ascii_lower(t[i]) == tts_norm_ascii_lower(node.cp)))) {
|
||||
end = i + 1;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case tts_norm_op::SET:
|
||||
if (i < n && set_match(node.set, t[i])) {
|
||||
end = i + 1;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case tts_norm_op::ANY:
|
||||
if (i < n) {
|
||||
end = i + 1;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case tts_norm_op::START:
|
||||
end = i;
|
||||
return i == 0;
|
||||
case tts_norm_op::END:
|
||||
end = i;
|
||||
return i == n;
|
||||
case tts_norm_op::BOUNDARY:
|
||||
end = i;
|
||||
return (i > 0 && is_w(i - 1)) != is_w(i);
|
||||
case tts_norm_op::GROUP: {
|
||||
size_t e = i;
|
||||
if (!match_alts(node.alts, i, e)) {
|
||||
return false;
|
||||
}
|
||||
if (node.capture >= 0) {
|
||||
caps[node.capture] = {static_cast<int64_t>(i), static_cast<int64_t>(e)};
|
||||
}
|
||||
end = e;
|
||||
return true;
|
||||
}
|
||||
case tts_norm_op::LOOKAHEAD: {
|
||||
// captures made inside a lookahead are discarded
|
||||
auto saved = caps;
|
||||
size_t e = i;
|
||||
const bool ok = match_alts(node.alts, i, e);
|
||||
caps = std::move(saved);
|
||||
end = i;
|
||||
return ok != node.neg;
|
||||
}
|
||||
case tts_norm_op::LOOKBEHIND: {
|
||||
size_t e = 0;
|
||||
const bool ok = i > 0 && match_once(node.alts[0][0].atom, i - 1, e) && e == i;
|
||||
end = i;
|
||||
return ok != node.neg;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// common_tts_norm_pattern
|
||||
//
|
||||
|
||||
common_tts_norm_pattern::common_tts_norm_pattern(const std::string & src) {
|
||||
if (src.size() > TTS_NORM_MAX_PATTERN_LEN) {
|
||||
throw std::runtime_error("invalid TTS norm pattern: pattern is longer than " + std::to_string(TTS_NORM_MAX_PATTERN_LEN) + " bytes");
|
||||
}
|
||||
std::vector<uint32_t> cpts;
|
||||
std::vector<size_t> offs;
|
||||
tts_norm_decode_utf8(src, cpts, offs);
|
||||
|
||||
tts_norm_parser parser(std::move(cpts), n_caps);
|
||||
static const std::vector<uint32_t> icase_prefix = {'(', '?', 'i', ')'};
|
||||
if (parser.s.size() >= 4 && std::equal(icase_prefix.begin(), icase_prefix.end(), parser.s.begin())) {
|
||||
icase = true;
|
||||
parser.p = 4;
|
||||
}
|
||||
|
||||
root = std::make_unique<tts_norm_pattern_node>();
|
||||
root->op = tts_norm_op::GROUP;
|
||||
root->alts = parser.parse_alts();
|
||||
if (!parser.eof()) {
|
||||
parser.fail("unbalanced ')'");
|
||||
}
|
||||
}
|
||||
|
||||
common_tts_norm_pattern::~common_tts_norm_pattern() = default;
|
||||
common_tts_norm_pattern::common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept = default;
|
||||
common_tts_norm_pattern & common_tts_norm_pattern::operator=(common_tts_norm_pattern &&) noexcept = default;
|
||||
|
||||
bool common_tts_norm_pattern::match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const {
|
||||
tts_norm_matcher m{cpts, icase, std::vector<std::pair<int64_t, int64_t>>(n_caps, {-1, -1})};
|
||||
size_t e = pos;
|
||||
if (!m.match_alts(root->alts, pos, e)) {
|
||||
return false;
|
||||
}
|
||||
end = e;
|
||||
groups = std::move(m.caps);
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t common_tts_norm_pattern::n_groups() const {
|
||||
return n_caps;
|
||||
}
|
||||
|
||||
//
|
||||
// scan / sub
|
||||
//
|
||||
|
||||
std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules) {
|
||||
std::vector<uint32_t> cpts;
|
||||
std::vector<size_t> offs;
|
||||
tts_norm_decode_utf8(text, cpts, offs);
|
||||
|
||||
std::vector<common_tts_norm_segment> out;
|
||||
const size_t n = cpts.size();
|
||||
size_t i = 0;
|
||||
size_t pending = 0; // start of the current unmatched run
|
||||
std::vector<std::pair<int64_t, int64_t>> groups;
|
||||
|
||||
while (i < n) {
|
||||
bool matched = false;
|
||||
for (size_t r = 0; r < rules.size(); r++) {
|
||||
size_t end = i;
|
||||
if (rules[r]->match_at(cpts, i, end, groups) && end > i) {
|
||||
if (pending < i) {
|
||||
out.push_back({-1, offs[pending], offs[i], {}});
|
||||
}
|
||||
common_tts_norm_segment seg;
|
||||
seg.rule = static_cast<int>(r);
|
||||
seg.start = offs[i];
|
||||
seg.end = offs[end];
|
||||
for (const auto & g : groups) {
|
||||
if (g.first < 0) {
|
||||
seg.groups.emplace_back(std::string::npos, std::string::npos);
|
||||
} else {
|
||||
seg.groups.emplace_back(offs[g.first], offs[g.second]);
|
||||
}
|
||||
}
|
||||
out.push_back(std::move(seg));
|
||||
i = end;
|
||||
pending = i;
|
||||
matched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!matched) {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
if (pending < n) {
|
||||
out.push_back({-1, offs[pending], offs[n], {}});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
struct tts_norm_repl_piece {
|
||||
std::string literal;
|
||||
int group = -1; // 0 = whole match, 1.. = groups
|
||||
};
|
||||
|
||||
static std::vector<tts_norm_repl_piece> tts_norm_parse_repl(const std::string & repl, const common_tts_norm_pattern & pattern) {
|
||||
std::vector<tts_norm_repl_piece> pieces;
|
||||
std::string lit;
|
||||
auto flush = [&]() {
|
||||
if (!lit.empty()) {
|
||||
pieces.push_back({lit, -1});
|
||||
lit.clear();
|
||||
}
|
||||
};
|
||||
for (size_t i = 0; i < repl.size(); i++) {
|
||||
if (repl[i] != '%') {
|
||||
lit += repl[i];
|
||||
continue;
|
||||
}
|
||||
if (i + 1 >= repl.size()) {
|
||||
throw std::runtime_error("invalid TTS norm replacement: dangling '%'");
|
||||
}
|
||||
const char c = repl[++i];
|
||||
if (c == '%') {
|
||||
lit += '%';
|
||||
} else if (c >= '0' && c <= '9') {
|
||||
const int g = c - '0';
|
||||
if (static_cast<size_t>(g) > pattern.n_groups()) {
|
||||
throw std::runtime_error("invalid TTS norm replacement: group %" + std::string(1, c) + " does not exist");
|
||||
}
|
||||
flush();
|
||||
pieces.push_back({"", g});
|
||||
} else {
|
||||
throw std::runtime_error("invalid TTS norm replacement: unknown escape '%" + std::string(1, c) + "'");
|
||||
}
|
||||
}
|
||||
flush();
|
||||
return pieces;
|
||||
}
|
||||
|
||||
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl) {
|
||||
const auto pieces = tts_norm_parse_repl(repl, pattern);
|
||||
std::string out;
|
||||
for (const auto & seg : common_tts_norm_scan(text, {&pattern})) {
|
||||
if (seg.rule < 0) {
|
||||
out.append(text, seg.start, seg.end - seg.start);
|
||||
continue;
|
||||
}
|
||||
for (const auto & piece : pieces) {
|
||||
if (piece.group < 0) {
|
||||
out += piece.literal;
|
||||
} else if (piece.group == 0) {
|
||||
out.append(text, seg.start, seg.end - seg.start);
|
||||
} else {
|
||||
const auto & g = seg.groups[piece.group - 1];
|
||||
if (g.first != std::string::npos) {
|
||||
out.append(text, g.first, g.second - g.first);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
//
|
||||
// template builtins
|
||||
//
|
||||
|
||||
// rules can be: a pattern string (rule name "match"), a dict {name: pattern}, or a list of [name, pattern] pairs
|
||||
static std::vector<std::pair<std::string, std::string>> tts_norm_rules_from_value(const value & v) {
|
||||
std::vector<std::pair<std::string, std::string>> rules;
|
||||
if (is_val<value_string>(v)) {
|
||||
rules.emplace_back("match", v->as_string().str());
|
||||
} else if (is_val<value_object>(v)) {
|
||||
for (const auto & kv : v->as_ordered_object()) {
|
||||
if (!is_val<value_string>(kv.second)) {
|
||||
throw std::runtime_error("tts_norm_scan: rule pattern must be a string");
|
||||
}
|
||||
rules.emplace_back(kv.first->as_string().str(), kv.second->as_string().str());
|
||||
}
|
||||
} else if (is_val<value_array>(v)) {
|
||||
for (const auto & item : v->as_array()) {
|
||||
if (!is_val<value_array>(item) || item->as_array().size() != 2
|
||||
|| !is_val<value_string>(item->as_array()[0]) || !is_val<value_string>(item->as_array()[1])) {
|
||||
throw std::runtime_error("tts_norm_scan: each rule must be a [name, pattern] pair of strings");
|
||||
}
|
||||
rules.emplace_back(item->as_array()[0]->as_string().str(), item->as_array()[1]->as_string().str());
|
||||
}
|
||||
} else {
|
||||
throw std::runtime_error("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs, got " + v->type());
|
||||
}
|
||||
return rules;
|
||||
}
|
||||
|
||||
static value tts_norm_builtin_scan(const func_args & args) {
|
||||
args.ensure_count(2, 2);
|
||||
args.ensure_val<value_string>(args.get_pos(0));
|
||||
const string text_val = args.get_pos(0)->as_string();
|
||||
const std::string text = text_val.str();
|
||||
const bool is_input = text_val.all_parts_are_input();
|
||||
|
||||
const auto rule_srcs = tts_norm_rules_from_value(args.get_pos(1));
|
||||
std::vector<common_tts_norm_pattern> patterns;
|
||||
patterns.reserve(rule_srcs.size());
|
||||
std::vector<const common_tts_norm_pattern *> ptrs;
|
||||
for (const auto & r : rule_srcs) {
|
||||
patterns.emplace_back(r.second);
|
||||
}
|
||||
for (const auto & p : patterns) {
|
||||
ptrs.push_back(&p);
|
||||
}
|
||||
|
||||
auto mk_str = [&](size_t start, size_t end) -> value {
|
||||
return mk_val<value_string>(string(text.substr(start, end - start), is_input));
|
||||
};
|
||||
|
||||
auto out = mk_val<value_array>();
|
||||
for (const auto & seg : common_tts_norm_scan(text, ptrs)) {
|
||||
auto groups = mk_val<value_array>();
|
||||
for (const auto & g : seg.groups) {
|
||||
// groups that did not participate are empty strings, so they render as nothing and are falsy
|
||||
groups->push_back(g.first == std::string::npos ? mk_str(0, 0) : mk_str(g.first, g.second));
|
||||
}
|
||||
auto obj = mk_val<value_object>();
|
||||
obj->insert("rule", seg.rule < 0 ? value(mk_val<value_none>()) : value(mk_val<value_string>(rule_srcs[seg.rule].first)));
|
||||
obj->insert("text", mk_str(seg.start, seg.end));
|
||||
obj->insert("groups", groups);
|
||||
out->push_back(obj);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static value tts_norm_builtin_sub(const func_args & args) {
|
||||
args.ensure_count(3, 3);
|
||||
args.ensure_vals<value_string, value_string, value_string>();
|
||||
const string text_val = args.get_pos(0)->as_string();
|
||||
const common_tts_norm_pattern pattern(args.get_pos(1)->as_string().str());
|
||||
const std::string res = common_tts_norm_sub(text_val.str(), pattern, args.get_pos(2)->as_string().str());
|
||||
return mk_val<value_string>(string(res, text_val.all_parts_are_input()));
|
||||
}
|
||||
|
||||
void common_chat_tts_norm_add_builtins(context & ctx) {
|
||||
ctx.set_val("tts_norm_scan", mk_val<value_func>("tts_norm_scan", tts_norm_builtin_scan));
|
||||
ctx.set_val("tts_norm_sub", mk_val<value_func>("tts_norm_sub", tts_norm_builtin_sub));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
// TTS normalization patterns: a small, fully specified matching language for TTS text normalization templates
|
||||
//
|
||||
// it is NOT a regex engine: the semantics are defined by llama.cpp (see docs/tts-norm-patterns.md), so that this
|
||||
// implementation and the python reference (scripts/chat_tts_norm.py) give the same result for every input
|
||||
//
|
||||
// two functions are exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():
|
||||
// tts_norm_scan(text, rules) -> list of segments {rule, text, groups}
|
||||
// tts_norm_sub(text, pattern, repl) -> string
|
||||
|
||||
namespace jinja {
|
||||
struct context;
|
||||
}
|
||||
|
||||
struct tts_norm_pattern_node;
|
||||
|
||||
class common_tts_norm_pattern {
|
||||
public:
|
||||
// throws std::runtime_error on invalid pattern
|
||||
explicit common_tts_norm_pattern(const std::string & src);
|
||||
~common_tts_norm_pattern();
|
||||
common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept;
|
||||
common_tts_norm_pattern & operator=(common_tts_norm_pattern &&) noexcept;
|
||||
|
||||
// match anchored at codepoint index pos; on success, end is the codepoint index after the match
|
||||
// groups[i] holds [start, end) codepoint indices of capture i+1, or {-1, -1} if the group did not participate
|
||||
bool match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const;
|
||||
|
||||
size_t n_groups() const;
|
||||
|
||||
private:
|
||||
std::unique_ptr<tts_norm_pattern_node> root;
|
||||
size_t n_caps = 0;
|
||||
bool icase = false;
|
||||
};
|
||||
|
||||
// segment produced by common_tts_norm_scan(); byte offsets into the input text
|
||||
struct common_tts_norm_segment {
|
||||
int rule = -1; // index of the matched rule, -1 for unmatched text
|
||||
size_t start = 0;
|
||||
size_t end = 0;
|
||||
std::vector<std::pair<size_t, size_t>> groups; // byte offsets; {npos, npos} for groups that did not participate
|
||||
};
|
||||
|
||||
// split text into matched and unmatched segments
|
||||
// at each position, rules are tried in order and the first non-empty match wins; otherwise one codepoint is skipped
|
||||
std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules);
|
||||
|
||||
// replace every non-overlapping match of pattern with repl
|
||||
// repl references: %0 (whole match), %1..%9 (groups), %% (literal percent)
|
||||
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl);
|
||||
|
||||
// add tts_norm_scan() and tts_norm_sub() to the context of a task template
|
||||
// they are not jinja global builtins on purpose: chat templates are not affected
|
||||
void common_chat_tts_norm_add_builtins(jinja::context & ctx);
|
||||
@@ -3,6 +3,7 @@
|
||||
#include "chat-auto-parser-helpers.h"
|
||||
#include "chat-auto-parser.h"
|
||||
#include "chat-peg-parser.h"
|
||||
#include "chat-tts-norm.h"
|
||||
#include "common.h"
|
||||
#include "ggml.h"
|
||||
#include "json-schema-to-grammar.h"
|
||||
@@ -807,6 +808,8 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
|
||||
// renders a task template (e.g. "tts") with the given variables
|
||||
static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) {
|
||||
jinja::context ctx(tmpl.source());
|
||||
// text normalization helpers: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md
|
||||
common_chat_tts_norm_add_builtins(ctx);
|
||||
jinja::global_from_json(ctx, vars, false);
|
||||
ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
|
||||
ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
# TTS normalization patterns
|
||||
|
||||
Text-to-speech models often need their input normalized before tokenization: numbers, currencies, dates and symbols spelled out, punctuation mapped, spacing fixed. The exact rules are model-specific (they must match what the model was trained on), so they live in the model's `tts` task template rather than in C++.
|
||||
|
||||
Jinja has no regex support, and exposing `std::regex` / Python `re` would not help: the two differ in many details (`\d`, `\w`, `\b`, lookbehind, case folding, backtracking limits), so a template could give different results in llama.cpp and in Python. Instead, task templates get two functions with a small matching language whose semantics are defined here:
|
||||
|
||||
- `tts_norm_scan(text, rules)`: split text into matched and unmatched segments
|
||||
- `tts_norm_sub(text, pattern, repl)`: replace every match
|
||||
|
||||
Implementations:
|
||||
- C++: [common/chat-tts-norm.cpp](../common/chat-tts-norm.cpp), injected into task templates by `common_chat_template_render()` in [common/chat.cpp](../common/chat.cpp). They are not available to chat templates.
|
||||
- Python reference: [scripts/chat_tts_norm.py](../scripts/chat_tts_norm.py), for use with `jinja2` (`env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan`)
|
||||
- Both are checked against each other by `tests/test-jinja.cpp` (`test-jinja -py "tts norm patterns.*"`)
|
||||
|
||||
## Example
|
||||
|
||||
```jinja
|
||||
{%- for s in tts_norm_scan(text, [
|
||||
['money', '%$(%d+)(?:%.(%d%d))?'],
|
||||
['percent', '(%d+)%%'],
|
||||
['time', '(%d%d?):(%d%d)(?!%d)'],
|
||||
['neg', '(?<!%w)-(%d+)'],
|
||||
['number', '%d+'],
|
||||
]) -%}
|
||||
{%- if s.rule == 'money' -%}{%- set whole, cents = s.groups -%}{{ num(whole) }} dollars{{ ' and ' ~ num(cents) ~ ' cents' if cents else '' }}
|
||||
{%- elif s.rule == 'percent' -%}{{ num(s.groups[0]) }} percent
|
||||
{%- elif s.rule == 'time' -%}{%- set h, m = s.groups -%}{{ num(h) }} {{ num(m) }}
|
||||
{%- elif s.rule == 'neg' -%}minus {{ num(s.groups[0]) }}
|
||||
{%- elif s.rule == 'number' -%}{{ num(s.text) }}
|
||||
{%- else -%}{{ s.text }}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
```
|
||||
|
||||
`It costs $3.50, 20% off, at 10:45, -5 degrees, 7 cats.` becomes `It costs three dollars and fifty cents, twenty percent off, at ten forty-five, minus five degrees, seven cats.` (with a `num` macro that spells numbers, see the "mini text normalizer" test in `tests/test-jinja.cpp`).
|
||||
|
||||
## Functions
|
||||
|
||||
### `tts_norm_scan(text, rules)`
|
||||
|
||||
`rules` is one of:
|
||||
- a list of `[name, pattern]` pairs (recommended, the order is explicit)
|
||||
- a dict `{name: pattern}`, in insertion order
|
||||
- a single pattern string, the rule is named `match`
|
||||
|
||||
Scanning starts at the first codepoint. At each position, the rules are tried in order and the first one that produces a **non-empty** match wins; the scan continues after the match. If no rule matches, one codepoint is skipped. Matches never overlap.
|
||||
|
||||
The result is a list of segments covering the whole text, in order. Consecutive unmatched codepoints are merged into one segment. Each segment is a dict:
|
||||
|
||||
| key | matched segment | unmatched segment |
|
||||
|---|---|---|
|
||||
| `rule` | rule name | `none` |
|
||||
| `text` | matched text | unmatched text |
|
||||
| `groups` | list of capture groups, in order of their `(` | `[]` |
|
||||
|
||||
There are no named groups, unpack `groups` instead: `{% set whole, cents = s.groups %}`.
|
||||
|
||||
A group that did not participate in the match (e.g. inside a `?` that matched nothing) is an empty string, so it renders as nothing and is falsy.
|
||||
|
||||
### `tts_norm_sub(text, pattern, repl)`
|
||||
|
||||
Replaces every match of `pattern`, with the same scanning rules as `tts_norm_scan`. `repl` is a literal string with references:
|
||||
|
||||
| reference | value |
|
||||
|---|---|
|
||||
| `%0` | whole match |
|
||||
| `%1` .. `%9` | group |
|
||||
| `%%` | a literal `%` |
|
||||
|
||||
Any other `%` sequence in `repl`, or a reference to a group that does not exist, is an error.
|
||||
|
||||
## Pattern syntax
|
||||
|
||||
Patterns operate on Unicode codepoints. Invalid UTF-8 bytes (C++ only) are seen as U+FFFD, one per byte, and preserved in the output.
|
||||
|
||||
| syntax | meaning |
|
||||
|---|---|
|
||||
| `x` | the literal codepoint `x` |
|
||||
| `%x` | escape: `%` followed by any non-alphanumeric codepoint is that literal codepoint (`%.`, `%$`, `%%`, `%(`, ...) |
|
||||
| `.` | any codepoint, including newlines |
|
||||
| `%d` `%s` `%p` `%a` `%w` `%u` `%l` `%h` `%c` | character classes, see below; the uppercase form (`%D`, ...) is the negation |
|
||||
| `[...]` | set: literal codepoints, ranges `a-z`, classes `%d`; `[^...]` is the negation |
|
||||
| `(...)` | capture group |
|
||||
| `(?:...)` | group without capture |
|
||||
| `a\|b` | ordered alternation |
|
||||
| `?` `*` `+` `{m}` `{m,}` `{m,n}` | quantifiers, greedy and possessive; `m`, `n` up to 1000 |
|
||||
| `^` `$` | start / end of the whole text |
|
||||
| `%b` | boundary: between a `%w` codepoint and a non-`%w` codepoint (the edges of the text count as non-`%w`) |
|
||||
| `(?=...)` `(?!...)` | lookahead, positive / negative |
|
||||
| `(?<=c)` `(?<!c)` | lookbehind of exactly one codepoint: `c` is a literal, a set, a class or `.` |
|
||||
| `(?i)` | at the very start of the pattern only: ignore ASCII case |
|
||||
|
||||
These characters must be escaped with `%` to be matched literally: `( ) [ ] { } . | ? * + ^ $ %`. Inside a set, only `]`, `%`, `-` (between two codepoints) and `^` (first position) are special.
|
||||
|
||||
### Matching semantics
|
||||
|
||||
The matcher is a PEG, not a backtracking regex:
|
||||
|
||||
- **Ordered alternation**: alternatives are tried left to right and the first one that matches is committed to. If what follows fails, the other alternatives are not retried. `(?:a|ab)c` never matches `abc`; write `(?:ab|a)c`.
|
||||
- **Possessive quantifiers**: quantifiers match as many repetitions as possible and never give any back. `%d+0` never matches; `.*x` never matches. Write patterns that describe exactly what to consume.
|
||||
- A repetition that matches the empty string ends the loop.
|
||||
- Captures inside lookaheads are discarded. A capture inside a repeated group holds its last repetition.
|
||||
|
||||
This gives a simple and linear-time implementation with identical results in C++ and Python.
|
||||
|
||||
### Character classes
|
||||
|
||||
Classes are defined by explicit codepoint ranges, not by a Unicode database, so the result does not depend on the Unicode version of the platform.
|
||||
|
||||
| class | definition |
|
||||
|---|---|
|
||||
| `%d` | ASCII digits `0-9` only |
|
||||
| `%s` | U+0009..U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000..U+200A, U+2028, U+2029, U+202F, U+205F, U+3000 |
|
||||
| `%p` | punctuation and symbols: ASCII punctuation, U+00A1..U+00BF (except U+00AA, U+00B5, U+00BA), U+00D7, U+00F7, U+2010..U+2027, U+2030..U+205E, U+20A0..U+20CF (currency), U+2100..U+2BFF (letterlike, number forms, arrows, math, technical, box drawing, shapes, misc symbols, dingbats), CJK punctuation (U+3001..U+3003, U+3008..U+3020, U+3030, U+303D, U+30FB), U+FE10..U+FE19, U+FE30..U+FE6B, fullwidth punctuation (U+FF01..U+FF0F, U+FF1A..U+FF20, U+FF3B..U+FF40, U+FF5B..U+FF65), U+1F000..U+1FAFF (emoji) |
|
||||
| `%c` | control and invisible format characters: U+0000..U+001F, U+007F..U+009F, U+200B..U+200F, U+2060..U+206F, U+FEFF |
|
||||
| `%a` | ASCII letters, U+00AA, U+00B5, U+00BA, and every codepoint from U+00C0 that is not in `%s`, `%p` or `%c` |
|
||||
| `%w` | `%a` or `%d` |
|
||||
| `%u` / `%l` | ASCII uppercase / lowercase letters only |
|
||||
| `%h` | Han: U+2E80..U+2FDF, U+3005, U+3007, U+3021..U+3029, U+3038..U+303B, U+3400..U+4DBF, U+4E00..U+9FFF, U+F900..U+FAFF, U+20000..U+323AF |
|
||||
|
||||
Classes may overlap (e.g. U+0085 is both `%s` and `%c`). `%a` intentionally includes all scripts, combining marks and CJK, so `(?<!%w)` and `%b` work as "not inside a word" for any language.
|
||||
|
||||
## Errors
|
||||
|
||||
Invalid patterns are rejected when the function is called, with a message and a position, for example: unbalanced parentheses, a quantifier with nothing to repeat, multiple quantifiers (`a+?`), repeating a zero-width assertion (`%b+`), unknown escapes (`%q`), unescaped `]` or `}`, an empty set, a lookbehind wider than one codepoint and `(?i)` not at the start.
|
||||
@@ -0,0 +1,556 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reference implementation of TTS normalization patterns, the tts_norm_scan() and tts_norm_sub() template functions.
|
||||
|
||||
The specification is in docs/tts-norm-patterns.md. This file must give the same result as the C++ implementation
|
||||
(common/chat-tts-norm.cpp) for every input; tests/test-jinja.cpp checks this in python mode (test-jinja -py).
|
||||
|
||||
Usage with jinja2:
|
||||
|
||||
env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan
|
||||
env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
# character classes, explicit codepoint ranges (keep in sync with common/chat-tts-norm.cpp)
|
||||
|
||||
TTS_NORM_SPACE = [
|
||||
(0x0009, 0x000D), (0x0020, 0x0020), (0x0085, 0x0085), (0x00A0, 0x00A0), (0x1680, 0x1680),
|
||||
(0x2000, 0x200A), (0x2028, 0x2029), (0x202F, 0x202F), (0x205F, 0x205F), (0x3000, 0x3000),
|
||||
]
|
||||
|
||||
TTS_NORM_PUNCT = [
|
||||
(0x0021, 0x002F), (0x003A, 0x0040), (0x005B, 0x0060), (0x007B, 0x007E),
|
||||
(0x00A1, 0x00A9), (0x00AB, 0x00B4), (0x00B6, 0x00B9), (0x00BB, 0x00BF), (0x00D7, 0x00D7), (0x00F7, 0x00F7),
|
||||
(0x2010, 0x2027), (0x2030, 0x205E), (0x20A0, 0x20CF), (0x2100, 0x218F), (0x2190, 0x2BFF),
|
||||
(0x3001, 0x3003), (0x3008, 0x3020), (0x3030, 0x3030), (0x303D, 0x303D), (0x30FB, 0x30FB),
|
||||
(0xFE10, 0xFE19), (0xFE30, 0xFE4F), (0xFE50, 0xFE6B),
|
||||
(0xFF01, 0xFF0F), (0xFF1A, 0xFF20), (0xFF3B, 0xFF40), (0xFF5B, 0xFF65),
|
||||
(0x1F000, 0x1FAFF),
|
||||
]
|
||||
|
||||
TTS_NORM_CONTROL = [
|
||||
(0x0000, 0x001F), (0x007F, 0x009F), (0x200B, 0x200F), (0x2060, 0x206F), (0xFEFF, 0xFEFF),
|
||||
]
|
||||
|
||||
TTS_NORM_HAN = [
|
||||
(0x2E80, 0x2FDF), (0x3005, 0x3005), (0x3007, 0x3007), (0x3021, 0x3029), (0x3038, 0x303B),
|
||||
(0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), (0x20000, 0x323AF),
|
||||
]
|
||||
|
||||
CLASS_LETTERS = "dspawulhc"
|
||||
INF = 1 << 31
|
||||
MAX_REPEAT = 1000
|
||||
MAX_PATTERN_LEN = 4096
|
||||
|
||||
|
||||
def _in(ranges: list[tuple[int, int]], c: int) -> bool:
|
||||
return any(lo <= c <= hi for lo, hi in ranges)
|
||||
|
||||
|
||||
def _is_alpha(c: int) -> bool:
|
||||
if ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z"):
|
||||
return True
|
||||
if c in (0xAA, 0xB5, 0xBA):
|
||||
return True
|
||||
return c >= 0xC0 and not _in(TTS_NORM_SPACE, c) and not _in(TTS_NORM_PUNCT, c) and not _in(TTS_NORM_CONTROL, c)
|
||||
|
||||
|
||||
def _class_match(kind: str, c: int) -> bool:
|
||||
if kind == "d":
|
||||
return ord("0") <= c <= ord("9")
|
||||
if kind == "s":
|
||||
return _in(TTS_NORM_SPACE, c)
|
||||
if kind == "p":
|
||||
return _in(TTS_NORM_PUNCT, c)
|
||||
if kind == "a":
|
||||
return _is_alpha(c)
|
||||
if kind == "w":
|
||||
return _is_alpha(c) or ord("0") <= c <= ord("9")
|
||||
if kind == "u":
|
||||
return ord("A") <= c <= ord("Z")
|
||||
if kind == "l":
|
||||
return ord("a") <= c <= ord("z")
|
||||
if kind == "h":
|
||||
return _in(TTS_NORM_HAN, c)
|
||||
if kind == "c":
|
||||
return _in(TTS_NORM_CONTROL, c)
|
||||
return False
|
||||
|
||||
|
||||
def _ascii_lower(c: int) -> int:
|
||||
return c + 32 if ord("A") <= c <= ord("Z") else c
|
||||
|
||||
|
||||
def _ascii_swapcase(c: int) -> int:
|
||||
if ord("A") <= c <= ord("Z"):
|
||||
return c + 32
|
||||
if ord("a") <= c <= ord("z"):
|
||||
return c - 32
|
||||
return c
|
||||
|
||||
|
||||
def _is_ascii_alnum(c: int) -> bool:
|
||||
return ord("0") <= c <= ord("9") or ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z")
|
||||
|
||||
|
||||
# AST
|
||||
|
||||
LITERAL, SET, ANY, START, END, BOUNDARY, GROUP, LOOKAHEAD, LOOKBEHIND = range(9)
|
||||
ZERO_WIDTH = (START, END, BOUNDARY, LOOKAHEAD, LOOKBEHIND)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Set:
|
||||
neg: bool = False
|
||||
ranges: list[tuple[int, int]] = field(default_factory=list)
|
||||
classes: list[tuple[str, bool]] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Node:
|
||||
op: int = GROUP
|
||||
cp: int = 0
|
||||
set: _Set = field(default_factory=_Set)
|
||||
alts: list[list[_Item]] = field(default_factory=list)
|
||||
capture: int = -1
|
||||
neg: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Item:
|
||||
atom: _Node
|
||||
min: int = 1
|
||||
max: int = 1
|
||||
|
||||
|
||||
class PatternError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class _Parser:
|
||||
def __init__(self, s: list[int]):
|
||||
self.s = s
|
||||
self.p = 0
|
||||
self.n_caps = 0
|
||||
|
||||
def fail(self, msg: str):
|
||||
raise PatternError(f"invalid TTS norm pattern: {msg} at position {self.p}")
|
||||
|
||||
def eof(self) -> bool:
|
||||
return self.p >= len(self.s)
|
||||
|
||||
def peek_is(self, c: str) -> bool:
|
||||
return not self.eof() and self.s[self.p] == ord(c)
|
||||
|
||||
def expect(self, c: str, what: str):
|
||||
if not self.peek_is(c):
|
||||
self.fail(f"expected {what}")
|
||||
self.p += 1
|
||||
|
||||
def parse_alts(self) -> list[list[_Item]]:
|
||||
alts = [self.parse_seq()]
|
||||
while self.peek_is("|"):
|
||||
self.p += 1
|
||||
alts.append(self.parse_seq())
|
||||
return alts
|
||||
|
||||
def parse_seq(self) -> list[_Item]:
|
||||
seq: list[_Item] = []
|
||||
while not self.eof() and self.s[self.p] not in (ord("|"), ord(")")):
|
||||
item = _Item(self.parse_atom())
|
||||
self.parse_quantifier(item)
|
||||
seq.append(item)
|
||||
return seq
|
||||
|
||||
def parse_int(self) -> int:
|
||||
if self.eof() or not (ord("0") <= self.s[self.p] <= ord("9")):
|
||||
self.fail("expected number in quantifier")
|
||||
v = 0
|
||||
while not self.eof() and ord("0") <= self.s[self.p] <= ord("9"):
|
||||
v = v * 10 + self.s[self.p] - ord("0")
|
||||
if v > MAX_REPEAT:
|
||||
self.fail(f"repetition count exceeds {MAX_REPEAT}")
|
||||
self.p += 1
|
||||
return v
|
||||
|
||||
def parse_quantifier(self, item: _Item):
|
||||
if self.eof():
|
||||
return
|
||||
c = chr(self.s[self.p])
|
||||
if c not in "?*+{":
|
||||
return
|
||||
if item.atom.op in ZERO_WIDTH:
|
||||
self.fail("zero-width assertion cannot be repeated")
|
||||
self.p += 1
|
||||
if c == "?":
|
||||
item.min, item.max = 0, 1
|
||||
elif c == "*":
|
||||
item.min, item.max = 0, INF
|
||||
elif c == "+":
|
||||
item.min, item.max = 1, INF
|
||||
else:
|
||||
item.min = self.parse_int()
|
||||
item.max = item.min
|
||||
if self.peek_is(","):
|
||||
self.p += 1
|
||||
item.max = INF if self.peek_is("}") else self.parse_int()
|
||||
self.expect("}", "'}' to close quantifier")
|
||||
if item.max < item.min:
|
||||
self.fail("quantifier max is smaller than min")
|
||||
if not self.eof() and chr(self.s[self.p]) in "?*+{":
|
||||
self.fail("multiple quantifiers")
|
||||
|
||||
def parse_group(self) -> _Node:
|
||||
node = _Node(op=GROUP)
|
||||
if self.peek_is("?"):
|
||||
self.p += 1
|
||||
if self.eof():
|
||||
self.fail("unterminated group")
|
||||
c = chr(self.s[self.p])
|
||||
if c == ":":
|
||||
self.p += 1
|
||||
elif c in "=!":
|
||||
self.p += 1
|
||||
node.op = LOOKAHEAD
|
||||
node.neg = c == "!"
|
||||
elif c == "<" and self.p + 1 < len(self.s) and self.s[self.p + 1] in (ord("="), ord("!")):
|
||||
node.op = LOOKBEHIND
|
||||
node.neg = self.s[self.p + 1] == ord("!")
|
||||
self.p += 2
|
||||
if self.eof() or self.s[self.p] == ord(")"):
|
||||
self.fail("lookbehind must contain exactly one codepoint matcher")
|
||||
atom = self.parse_atom()
|
||||
if atom.op not in (LITERAL, SET, ANY):
|
||||
self.fail("lookbehind must contain exactly one codepoint matcher")
|
||||
if not self.peek_is(")"):
|
||||
self.fail("lookbehind must contain exactly one codepoint matcher")
|
||||
self.p += 1
|
||||
node.alts = [[_Item(atom)]]
|
||||
return node
|
||||
elif c == "i":
|
||||
self.fail("(?i) is only allowed at the start of the pattern")
|
||||
else:
|
||||
self.fail("unknown group type")
|
||||
else:
|
||||
node.capture = self.n_caps
|
||||
self.n_caps += 1
|
||||
node.alts = self.parse_alts()
|
||||
self.expect(")", "')' to close group")
|
||||
return node
|
||||
|
||||
# returns ("class", (kind, neg)) | ("boundary", None) | ("literal", cp)
|
||||
def parse_escape(self, in_set: bool) -> tuple[str, Any]:
|
||||
if self.eof():
|
||||
self.fail("dangling '%'")
|
||||
e = self.s[self.p]
|
||||
self.p += 1
|
||||
if e < 0x80 and chr(e).lower() in CLASS_LETTERS:
|
||||
return "class", (chr(e).lower(), ord("A") <= e <= ord("Z"))
|
||||
if e == ord("b") and not in_set:
|
||||
return "boundary", None
|
||||
if _is_ascii_alnum(e):
|
||||
self.p -= 1
|
||||
self.fail(f"unknown escape '%{chr(e)}'")
|
||||
return "literal", e
|
||||
|
||||
def parse_set(self) -> _Node:
|
||||
node = _Node(op=SET)
|
||||
if self.peek_is("^"):
|
||||
node.set.neg = True
|
||||
self.p += 1
|
||||
empty = True
|
||||
while True:
|
||||
if self.eof():
|
||||
self.fail("unterminated set")
|
||||
if self.s[self.p] == ord("]"):
|
||||
self.p += 1
|
||||
break
|
||||
empty = False
|
||||
if self.s[self.p] == ord("%"):
|
||||
self.p += 1
|
||||
kind, val = self.parse_escape(True)
|
||||
if kind == "class":
|
||||
node.set.classes.append(val)
|
||||
continue
|
||||
lo = val
|
||||
else:
|
||||
lo = self.s[self.p]
|
||||
self.p += 1
|
||||
hi = lo
|
||||
if self.peek_is("-") and self.p + 1 < len(self.s) and self.s[self.p + 1] != ord("]"):
|
||||
self.p += 1
|
||||
if self.s[self.p] == ord("%"):
|
||||
self.p += 1
|
||||
kind, val = self.parse_escape(True)
|
||||
if kind == "class":
|
||||
self.fail("character class cannot be used in a range")
|
||||
hi = val
|
||||
else:
|
||||
hi = self.s[self.p]
|
||||
self.p += 1
|
||||
if hi < lo:
|
||||
self.fail("invalid range in set")
|
||||
node.set.ranges.append((lo, hi))
|
||||
if empty:
|
||||
self.fail("empty set")
|
||||
return node
|
||||
|
||||
def parse_atom(self) -> _Node:
|
||||
c = chr(self.s[self.p])
|
||||
self.p += 1
|
||||
if c == "(":
|
||||
return self.parse_group()
|
||||
if c == "[":
|
||||
return self.parse_set()
|
||||
if c == ".":
|
||||
return _Node(op=ANY)
|
||||
if c == "^":
|
||||
return _Node(op=START)
|
||||
if c == "$":
|
||||
return _Node(op=END)
|
||||
if c == "%":
|
||||
kind, val = self.parse_escape(False)
|
||||
if kind == "class":
|
||||
node = _Node(op=SET)
|
||||
node.set.classes.append(val)
|
||||
return node
|
||||
if kind == "boundary":
|
||||
return _Node(op=BOUNDARY)
|
||||
return _Node(op=LITERAL, cp=val)
|
||||
if c in "?*+{":
|
||||
self.p -= 1
|
||||
self.fail("nothing to repeat")
|
||||
if c in "]}":
|
||||
self.p -= 1
|
||||
self.fail(f"unescaped '{c}', use '%{c}'")
|
||||
return _Node(op=LITERAL, cp=ord(c))
|
||||
|
||||
|
||||
class _Matcher:
|
||||
def __init__(self, t: list[int], icase: bool, n_groups: int):
|
||||
self.t = t
|
||||
self.icase = icase
|
||||
self.caps: list[tuple[int, int]] = [(-1, -1)] * n_groups
|
||||
|
||||
def is_w(self, i: int) -> bool:
|
||||
return i < len(self.t) and _class_match("w", self.t[i])
|
||||
|
||||
def set_match(self, st: _Set, c: int) -> bool:
|
||||
def test(x: int) -> bool:
|
||||
if any(lo <= x <= hi for lo, hi in st.ranges):
|
||||
return True
|
||||
return any(_class_match(kind, x) != neg for kind, neg in st.classes)
|
||||
|
||||
m = test(c)
|
||||
if not m and self.icase:
|
||||
o = _ascii_swapcase(c)
|
||||
if o != c:
|
||||
m = test(o)
|
||||
return m != st.neg
|
||||
|
||||
def match_alts(self, alts: list[list[_Item]], i: int) -> int:
|
||||
"""returns the end position, or -1 if no alternative matches"""
|
||||
for seq in alts:
|
||||
saved = list(self.caps)
|
||||
end = self.match_seq(seq, i)
|
||||
if end >= 0:
|
||||
return end
|
||||
self.caps = saved
|
||||
return -1
|
||||
|
||||
def match_seq(self, seq: list[_Item], i: int) -> int:
|
||||
pos = i
|
||||
for it in seq:
|
||||
count = 0
|
||||
while count < it.max:
|
||||
is_group = it.atom.op == GROUP
|
||||
saved = list(self.caps) if is_group else None
|
||||
e = self.match_once(it.atom, pos)
|
||||
if e < 0:
|
||||
if saved is not None:
|
||||
self.caps = saved
|
||||
break
|
||||
if e == pos:
|
||||
# an empty iteration would repeat forever, it satisfies any remaining minimum
|
||||
count = max(count + 1, it.min)
|
||||
break
|
||||
pos = e
|
||||
count += 1
|
||||
if count < it.min:
|
||||
return -1
|
||||
return pos
|
||||
|
||||
def match_once(self, node: _Node, i: int) -> int:
|
||||
"""returns the end position, or -1 on failure"""
|
||||
n = len(self.t)
|
||||
op = node.op
|
||||
if op == LITERAL:
|
||||
if i < n and (self.t[i] == node.cp or (self.icase and _ascii_lower(self.t[i]) == _ascii_lower(node.cp))):
|
||||
return i + 1
|
||||
return -1
|
||||
if op == SET:
|
||||
return i + 1 if i < n and self.set_match(node.set, self.t[i]) else -1
|
||||
if op == ANY:
|
||||
return i + 1 if i < n else -1
|
||||
if op == START:
|
||||
return i if i == 0 else -1
|
||||
if op == END:
|
||||
return i if i == n else -1
|
||||
if op == BOUNDARY:
|
||||
return i if (i > 0 and self.is_w(i - 1)) != self.is_w(i) else -1
|
||||
if op == GROUP:
|
||||
e = self.match_alts(node.alts, i)
|
||||
if e < 0:
|
||||
return -1
|
||||
if node.capture >= 0:
|
||||
self.caps[node.capture] = (i, e)
|
||||
return e
|
||||
if op == LOOKAHEAD:
|
||||
# captures made inside a lookahead are discarded
|
||||
saved = list(self.caps)
|
||||
ok = self.match_alts(node.alts, i) >= 0
|
||||
self.caps = saved
|
||||
return i if ok != node.neg else -1
|
||||
if op == LOOKBEHIND:
|
||||
ok = i > 0 and self.match_once(node.alts[0][0].atom, i - 1) == i
|
||||
return i if ok != node.neg else -1
|
||||
return -1
|
||||
|
||||
|
||||
class Pattern:
|
||||
def __init__(self, src: str):
|
||||
if len(src.encode("utf-8")) > MAX_PATTERN_LEN:
|
||||
raise PatternError(f"invalid TTS norm pattern: pattern is longer than {MAX_PATTERN_LEN} bytes")
|
||||
self.icase = False
|
||||
parser = _Parser([ord(c) for c in src])
|
||||
if src.startswith("(?i)"):
|
||||
self.icase = True
|
||||
parser.p = 4
|
||||
self.root = _Node(op=GROUP, alts=parser.parse_alts())
|
||||
if not parser.eof():
|
||||
parser.fail("unbalanced ')'")
|
||||
self.n_caps = parser.n_caps
|
||||
|
||||
def match_at(self, t: list[int], pos: int) -> tuple[int, list[tuple[int, int]]] | None:
|
||||
m = _Matcher(t, self.icase, self.n_caps)
|
||||
end = m.match_alts(self.root.alts, pos)
|
||||
if end < 0:
|
||||
return None
|
||||
return end, m.caps
|
||||
|
||||
|
||||
# (rule index or -1, start, end, groups) with codepoint offsets
|
||||
Segment = tuple[int, int, int, list[tuple[int, int]]]
|
||||
|
||||
|
||||
def scan_segments(text: str, rules: list[Pattern]) -> list[Segment]:
|
||||
t = [ord(c) for c in text]
|
||||
n = len(t)
|
||||
out: list[Segment] = []
|
||||
i = 0
|
||||
pending = 0
|
||||
while i < n:
|
||||
matched = False
|
||||
for r, rule in enumerate(rules):
|
||||
res = rule.match_at(t, i)
|
||||
if res is not None and res[0] > i:
|
||||
end, groups = res
|
||||
if pending < i:
|
||||
out.append((-1, pending, i, []))
|
||||
out.append((r, i, end, groups))
|
||||
i = end
|
||||
pending = i
|
||||
matched = True
|
||||
break
|
||||
if not matched:
|
||||
i += 1
|
||||
if pending < n:
|
||||
out.append((-1, pending, n, []))
|
||||
return out
|
||||
|
||||
|
||||
def _rules_from_value(rules: Any) -> list[tuple[str, str]]:
|
||||
if isinstance(rules, str):
|
||||
return [("match", rules)]
|
||||
if isinstance(rules, dict):
|
||||
out = []
|
||||
for k, v in rules.items():
|
||||
if not isinstance(v, str):
|
||||
raise TypeError("tts_norm_scan: rule pattern must be a string")
|
||||
out.append((str(k), v))
|
||||
return out
|
||||
if isinstance(rules, (list, tuple)):
|
||||
out = []
|
||||
for item in rules:
|
||||
if not isinstance(item, (list, tuple)) or len(item) != 2 or not isinstance(item[0], str) or not isinstance(item[1], str):
|
||||
raise TypeError("tts_norm_scan: each rule must be a [name, pattern] pair of strings")
|
||||
out.append((item[0], item[1]))
|
||||
return out
|
||||
raise TypeError("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs")
|
||||
|
||||
|
||||
def tts_norm_scan(text: str, rules: Any) -> list[dict[str, Any]]:
|
||||
rule_srcs = _rules_from_value(rules)
|
||||
patterns = [Pattern(src) for _, src in rule_srcs]
|
||||
out: list[dict[str, Any]] = []
|
||||
for r, start, end, groups in scan_segments(text, patterns):
|
||||
# groups that did not participate are empty strings, so they render as nothing and are falsy
|
||||
vals = ["" if gs < 0 else text[gs:ge] for gs, ge in groups]
|
||||
out.append({"rule": rule_srcs[r][0] if r >= 0 else None, "text": text[start:end], "groups": vals})
|
||||
return out
|
||||
|
||||
|
||||
def _parse_repl(repl: str, pattern: Pattern) -> list[str | int]:
|
||||
pieces: list[str | int] = []
|
||||
lit = ""
|
||||
i = 0
|
||||
while i < len(repl):
|
||||
c = repl[i]
|
||||
if c != "%":
|
||||
lit += c
|
||||
i += 1
|
||||
continue
|
||||
if i + 1 >= len(repl):
|
||||
raise PatternError("invalid TTS norm replacement: dangling '%'")
|
||||
i += 1
|
||||
c = repl[i]
|
||||
if c == "%":
|
||||
lit += "%"
|
||||
elif "0" <= c <= "9":
|
||||
g = ord(c) - ord("0")
|
||||
if g > pattern.n_caps:
|
||||
raise PatternError(f"invalid TTS norm replacement: group %{c} does not exist")
|
||||
if lit:
|
||||
pieces.append(lit)
|
||||
lit = ""
|
||||
pieces.append(g)
|
||||
else:
|
||||
raise PatternError(f"invalid TTS norm replacement: unknown escape '%{c}'")
|
||||
i += 1
|
||||
if lit:
|
||||
pieces.append(lit)
|
||||
return pieces
|
||||
|
||||
|
||||
def tts_norm_sub(text: str, pattern: str, repl: str) -> str:
|
||||
pat = Pattern(pattern)
|
||||
pieces = _parse_repl(repl, pat)
|
||||
out = []
|
||||
for r, start, end, groups in scan_segments(text, [pat]):
|
||||
if r < 0:
|
||||
out.append(text[start:end])
|
||||
continue
|
||||
for piece in pieces:
|
||||
if isinstance(piece, str):
|
||||
out.append(piece)
|
||||
elif piece == 0:
|
||||
out.append(text[start:end])
|
||||
else:
|
||||
gs, ge = groups[piece - 1]
|
||||
if gs >= 0:
|
||||
out.append(text[gs:ge])
|
||||
return "".join(out)
|
||||
+157
-11
@@ -3,6 +3,7 @@
|
||||
#include <random>
|
||||
#include <cstdlib>
|
||||
|
||||
#include "chat-tts-norm.h"
|
||||
#include "json.h"
|
||||
#include "subproc.h"
|
||||
|
||||
@@ -16,8 +17,8 @@
|
||||
|
||||
using json = common_json;
|
||||
|
||||
static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect);
|
||||
static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect);
|
||||
static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm = false);
|
||||
static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm = false);
|
||||
|
||||
static void test_whitespace_control(testing & t);
|
||||
static void test_conditionals(testing & t);
|
||||
@@ -39,6 +40,7 @@ static void test_stats(testing & t);
|
||||
static void test_caps(testing & t);
|
||||
static void test_string_parts(testing & t);
|
||||
static void test_fuzzing(testing & t);
|
||||
static void test_tts_norm_patterns(testing & t);
|
||||
|
||||
static bool g_python_mode = false;
|
||||
|
||||
@@ -75,6 +77,7 @@ int main(int argc, char *argv[]) {
|
||||
t.test("string methods", test_string_methods);
|
||||
t.test("array methods", test_array_methods);
|
||||
t.test("object methods", test_object_methods);
|
||||
t.test("tts norm patterns", test_tts_norm_patterns);
|
||||
if (!g_python_mode) {
|
||||
t.test("hasher", test_hasher);
|
||||
t.test("stats", test_stats);
|
||||
@@ -2713,13 +2716,134 @@ static void test_string_parts(testing & t) {
|
||||
|
||||
}
|
||||
|
||||
static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect) {
|
||||
t.test(name, [&tmpl, &expect](testing & t) {
|
||||
// TTS normalization patterns: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md
|
||||
// these functions are only available to task templates (e.g. "tts"), so every test here passes tts_norm = true
|
||||
static void test_tts_norm_patterns(testing & t) {
|
||||
test_template(t, "not available without tts norm builtins",
|
||||
"{{ tts_norm_scan is defined }}",
|
||||
json::object(),
|
||||
"False");
|
||||
|
||||
test_template(t, "tts_norm_scan splits text into matched and unmatched segments",
|
||||
"{%- for s in tts_norm_scan(text, [['num', '%d+'], ['word', '%a+']]) -%}[{{ s.rule or '-' }}:{{ s.text }}]{%- endfor -%}",
|
||||
{{"text", "abc 123, x9"}},
|
||||
"[word:abc][-: ][num:123][-:, ][word:x][num:9]", true);
|
||||
|
||||
test_template(t, "tts_norm_scan groups, optional group that did not match is empty",
|
||||
"{%- for s in tts_norm_scan(text, [['money', '%$(%d+)(?:%.(%d%d))?']]) -%}"
|
||||
"{%- if s.rule -%}{%- set whole, cents = s.groups -%}{{ whole }}|{{ cents }}|{{ s.groups | length }};{%- endif -%}"
|
||||
"{%- endfor -%}",
|
||||
{{"text", "$3.50 and $7"}},
|
||||
"3|50|2;7||2;", true);
|
||||
|
||||
test_template(t, "tts_norm_scan rules as dict, the first rule that matches wins",
|
||||
"{%- for s in tts_norm_scan(text, {'ordinal': '%d+(?:st|nd|rd|th)%b', 'number': '%d+'}) -%}"
|
||||
"{%- if s.rule -%}<{{ s.rule }}:{{ s.text }}>{%- else -%}{{ s.text }}{%- endif -%}"
|
||||
"{%- endfor -%}",
|
||||
{{"text", "the 3rd of 10 or 4three"}},
|
||||
"the <ordinal:3rd> of <number:10> or <number:4>three", true);
|
||||
|
||||
// unlike regex, a failed sequence does not retry the other branches: (?:a|ab)c never matches "abc"
|
||||
test_template(t, "ordered alternation commits to the first branch that matches",
|
||||
"{%- for s in tts_norm_scan(text, '(?:a|ab)c') -%}[{{ s.rule or '-' }}:{{ s.text }}]{%- endfor -%}",
|
||||
{{"text", "abc ac"}},
|
||||
"[-:abc ][match:ac]", true);
|
||||
|
||||
// unlike regex, quantifiers never give back what they consumed: %d+0 never matches
|
||||
test_template(t, "quantifiers are possessive",
|
||||
"{{ tts_norm_sub(text, '%d+0', 'X') }}|{{ tts_norm_sub(text, '%d+', 'X') }}",
|
||||
{{"text", "100 20"}},
|
||||
"100 20|X X", true);
|
||||
|
||||
test_template(t, "quantifier ranges",
|
||||
"{{ tts_norm_sub(text, '%d{3}-%d{2,}', '#') }}",
|
||||
{{"text", "123-4 123-45 123-456789"}},
|
||||
"123-4 # #", true);
|
||||
|
||||
test_template(t, "lookbehind: a minus sign is not part of a word",
|
||||
"{{ tts_norm_sub(text, '(?<!%w)-(%d+)', 'minus %1') }}",
|
||||
{{"text", "GPT-5 is -5, (-3)"}},
|
||||
"GPT-5 is minus 5, (minus 3)", true);
|
||||
|
||||
test_template(t, "lookahead: clock time not followed by a digit",
|
||||
"{{ tts_norm_sub(text, '(%d%d?):(%d%d)(?!%d)', '%1h%2') }}",
|
||||
{{"text", "at 10:30 or 1:234"}},
|
||||
"at 10h30 or 1:234", true);
|
||||
|
||||
test_template(t, "tts_norm_sub references: group, whole match and literal percent",
|
||||
"{{ tts_norm_sub(text, '(%d+)%%', '%1 percent (%0) 100%%') }}",
|
||||
{{"text", "up 50%"}},
|
||||
"up 50 percent (50%) 100%", true);
|
||||
|
||||
test_template(t, "split letters and digits, non-ASCII letters included",
|
||||
"{{ tts_norm_sub(tts_norm_sub(text, '(%a)(%d)', '%1 %2'), '(%d)(%a)', '%1 %2') }}",
|
||||
{{"text", "R2D2 café2 4K"}},
|
||||
"R 2 D 2 café 2 4 K", true);
|
||||
|
||||
test_template(t, "case-insensitive pattern and word boundary",
|
||||
"{{ tts_norm_sub(text, '(?i)%bdr%.', 'doctor') }}",
|
||||
{{"text", "Dr. Who and dr. No, not Mdr."}},
|
||||
"doctor Who and doctor No, not Mdr.", true);
|
||||
|
||||
test_template(t, "sets with multi-byte codepoints, negated sets with classes",
|
||||
"{{ tts_norm_sub(text, '[$€£]%s*(%d+)', '<%1>') }}|{{ tts_norm_sub(text, '[^%w%s]', '') }}",
|
||||
{{"text", "€ 5 & $6!"}},
|
||||
"<5> & <6>!| 5 6", true);
|
||||
|
||||
test_template(t, "remove spaces between Han characters",
|
||||
"{{ tts_norm_sub(text, '(%h) +(%h)', '%1%2') }}",
|
||||
{{"text", "你 好 world 世界"}},
|
||||
"你好 world 世界", true);
|
||||
|
||||
test_template(t, "mini text normalizer",
|
||||
"{%- set ones = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten',"
|
||||
" 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'] -%}\n"
|
||||
"{%- set tens = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'] -%}\n"
|
||||
"{%- macro num(n) -%}{{ ones[n] if n < 20 else tens[(n / 10) | int] ~ ('-' ~ ones[n % 10] if n % 10 else '') }}{%- endmacro -%}\n"
|
||||
"{%- for s in tts_norm_scan(text, [\n"
|
||||
" ['money', '%$(%d+)(?:%.(%d%d))?'],\n"
|
||||
" ['percent', '(%d+)%%'],\n"
|
||||
" ['time', '(%d%d?):(%d%d)(?!%d)'],\n"
|
||||
" ['neg', '(?<!%w)-(%d+)'],\n"
|
||||
" ['number', '%d+'],\n"
|
||||
"]) -%}\n"
|
||||
"{%- if s.rule == 'money' -%}{%- set whole, cents = s.groups -%}{{ num(whole | int) }} dollars{{ ' and ' ~ num(cents | int) ~ ' cents' if cents else '' }}\n"
|
||||
"{%- elif s.rule == 'percent' -%}{{ num(s.groups[0] | int) }} percent\n"
|
||||
"{%- elif s.rule == 'time' -%}{%- set h, m = s.groups -%}{{ num(h | int) }} {{ num(m | int) }}\n"
|
||||
"{%- elif s.rule == 'neg' -%}minus {{ num(s.groups[0] | int) }}\n"
|
||||
"{%- elif s.rule == 'number' -%}{{ num(s.text | int) }}\n"
|
||||
"{%- else -%}{{ s.text }}\n"
|
||||
"{%- endif -%}\n"
|
||||
"{%- endfor -%}",
|
||||
{{"text", "It costs $3.50, 20% off, at 10:45, -5 degrees, 7 cats."}},
|
||||
"It costs three dollars and fifty cents, twenty percent off, at ten forty-five, minus five degrees, seven cats.", true);
|
||||
|
||||
if (!g_python_mode) {
|
||||
test_template_error(t, "error: unclosed group", "{{ tts_norm_scan('x', '(a') }}", "expected ')' to close group", true);
|
||||
test_template_error(t, "error: unbalanced parenthesis", "{{ tts_norm_scan('x', 'a)') }}", "unbalanced ')'", true);
|
||||
test_template_error(t, "error: nothing to repeat", "{{ tts_norm_scan('x', '*a') }}", "nothing to repeat", true);
|
||||
test_template_error(t, "error: multiple quantifiers", "{{ tts_norm_scan('x', 'a+?') }}", "multiple quantifiers", true);
|
||||
test_template_error(t, "error: bad quantifier range", "{{ tts_norm_scan('x', 'a{2,1}') }}", "quantifier max is smaller than min", true);
|
||||
test_template_error(t, "error: repeated assertion", "{{ tts_norm_scan('x', '%b+') }}", "zero-width assertion cannot be repeated", true);
|
||||
test_template_error(t, "error: unknown class", "{{ tts_norm_scan('x', '%q') }}", "unknown escape '%q'", true);
|
||||
test_template_error(t, "error: unescaped bracket", "{{ tts_norm_scan('x', 'a]') }}", "unescaped ']'", true);
|
||||
test_template_error(t, "error: no named groups", "{{ tts_norm_scan('x', '(?<n>a)') }}", "unknown group type", true);
|
||||
test_template_error(t, "error: wide lookbehind", "{{ tts_norm_scan('x', '(?<=ab)c') }}", "lookbehind must contain exactly one codepoint matcher", true);
|
||||
test_template_error(t, "error: missing group in replacement", "{{ tts_norm_sub('x', '(a)', '%2') }}", "group %2 does not exist", true);
|
||||
test_template_error(t, "error: bad rules", "{{ tts_norm_scan('x', 1) }}", "rules must be a string, a dict or a list", true);
|
||||
}
|
||||
}
|
||||
|
||||
static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm) {
|
||||
t.test(name, [&tmpl, &expect, tts_norm](testing & t) {
|
||||
std::string error;
|
||||
try {
|
||||
jinja::lexer lexer;
|
||||
jinja::program ast = jinja::parse_from_tokens(lexer.tokenize(tmpl));
|
||||
jinja::context ctx(tmpl);
|
||||
if (tts_norm) {
|
||||
common_chat_tts_norm_add_builtins(ctx);
|
||||
}
|
||||
jinja::runtime runtime(ctx);
|
||||
runtime.gather_string_parts(runtime.execute(ast));
|
||||
} catch (const std::exception & e) {
|
||||
@@ -2729,14 +2853,17 @@ static void test_template_error(testing & t, const std::string & name, const std
|
||||
});
|
||||
}
|
||||
|
||||
static void test_template_cpp(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) {
|
||||
t.test(name, [&tmpl, &vars, &expect](testing & t) {
|
||||
static void test_template_cpp(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) {
|
||||
t.test(name, [&tmpl, &vars, &expect, tts_norm](testing & t) {
|
||||
jinja::lexer lexer;
|
||||
auto lexer_res = lexer.tokenize(tmpl);
|
||||
|
||||
jinja::program ast = jinja::parse_from_tokens(lexer_res);
|
||||
|
||||
jinja::context ctx(tmpl);
|
||||
if (tts_norm) {
|
||||
common_chat_tts_norm_add_builtins(ctx);
|
||||
}
|
||||
jinja::global_from_json(ctx, vars, true);
|
||||
|
||||
jinja::runtime runtime(ctx);
|
||||
@@ -2788,17 +2915,36 @@ env.filters["tojson"] = lambda x, ensure_ascii=False, indent=None, separators=No
|
||||
env.globals["strftime_now"] = lambda format: datetime.now().strftime(format)
|
||||
env.globals["raise_exception"] = raise_exception
|
||||
|
||||
if merged_input.get("tts_norm_ref"):
|
||||
import importlib.util
|
||||
spec = importlib.util.spec_from_file_location("chat_tts_norm", merged_input["tts_norm_ref"])
|
||||
chat_tts_norm = importlib.util.module_from_spec(spec)
|
||||
sys.modules["chat_tts_norm"] = chat_tts_norm # required by dataclasses
|
||||
spec.loader.exec_module(chat_tts_norm)
|
||||
env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan
|
||||
env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub
|
||||
|
||||
template = env.from_string(tmpl)
|
||||
result = template.render(**vars_json)
|
||||
sys.stdout.buffer.write(result.encode())
|
||||
)";
|
||||
|
||||
static void test_template_py(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) {
|
||||
t.test(name, [&tmpl, &vars, &expect](testing & t) {
|
||||
// python reference of the TTS normalization patterns, relative to this file
|
||||
static std::string tts_norm_ref_path() {
|
||||
std::string path = __FILE__;
|
||||
const size_t pos = path.find_last_of("/\\");
|
||||
return (pos == std::string::npos ? std::string(".") : path.substr(0, pos)) + "/../scripts/chat_tts_norm.py";
|
||||
}
|
||||
|
||||
static void test_template_py(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) {
|
||||
t.test(name, [&tmpl, &vars, &expect, tts_norm](testing & t) {
|
||||
// Prepare arguments
|
||||
json merged;
|
||||
merged["tmpl"] = json(tmpl);
|
||||
merged["vars"] = vars;
|
||||
if (tts_norm) {
|
||||
merged["tts_norm_ref"] = tts_norm_ref_path();
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
const char * python_executable = "python.exe";
|
||||
@@ -2859,11 +3005,11 @@ static void test_template_py(testing & t, const std::string & name, const std::s
|
||||
});
|
||||
}
|
||||
|
||||
static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) {
|
||||
static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) {
|
||||
if (g_python_mode) {
|
||||
test_template_py(t, name, tmpl, vars, expect);
|
||||
test_template_py(t, name, tmpl, vars, expect, tts_norm);
|
||||
} else {
|
||||
test_template_cpp(t, name, tmpl, vars, expect);
|
||||
test_template_cpp(t, name, tmpl, vars, expect, tts_norm);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user