Files
llama.cpp/common/chat-tts-norm.cpp
T
Xuan-Son Nguyen f7b886fe6f chat: allow TTS model to provide custom text normalizer via jinja (#32053)
* chat: allow TTS model to provide custom text normalizer via jinja

* rm tts_norm_is_reserved_name
2026-11-11 17:34:55 +01:00

865 lines
29 KiB
C++

#include "chat-tts-norm.h"
#include "jinja/runtime.h"
#include "jinja/value.h"
#include <algorithm>
#include <climits>
#include <stdexcept>
#include <string>
#include <vector>
// TTS normalization patterns, see docs/tts-norm-patterns.md for the specification
// keep this file in sync with the python reference: scripts/chat_tts_norm.py
using namespace jinja;
//
// character classes
// they are defined with explicit codepoint ranges, independent of any unicode database version,
// so that the C++ and python implementations agree on every codepoint
//
struct tts_norm_range {
uint32_t lo;
uint32_t hi;
};
static const tts_norm_range TTS_NORM_SPACE[] = {
{0x0009, 0x000D}, {0x0020, 0x0020}, {0x0085, 0x0085}, {0x00A0, 0x00A0}, {0x1680, 0x1680},
{0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F}, {0x205F, 0x205F}, {0x3000, 0x3000},
};
static const tts_norm_range TTS_NORM_PUNCT[] = {
{0x0021, 0x002F}, {0x003A, 0x0040}, {0x005B, 0x0060}, {0x007B, 0x007E},
{0x00A1, 0x00A9}, {0x00AB, 0x00B4}, {0x00B6, 0x00B9}, {0x00BB, 0x00BF}, {0x00D7, 0x00D7}, {0x00F7, 0x00F7},
{0x2010, 0x2027}, {0x2030, 0x205E}, {0x20A0, 0x20CF}, {0x2100, 0x218F}, {0x2190, 0x2BFF},
{0x3001, 0x3003}, {0x3008, 0x3020}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30FB, 0x30FB},
{0xFE10, 0xFE19}, {0xFE30, 0xFE4F}, {0xFE50, 0xFE6B},
{0xFF01, 0xFF0F}, {0xFF1A, 0xFF20}, {0xFF3B, 0xFF40}, {0xFF5B, 0xFF65},
{0x1F000, 0x1FAFF},
};
static const tts_norm_range TTS_NORM_CONTROL[] = {
{0x0000, 0x001F}, {0x007F, 0x009F}, {0x200B, 0x200F}, {0x2060, 0x206F}, {0xFEFF, 0xFEFF},
};
static const tts_norm_range TTS_NORM_HAN[] = {
{0x2E80, 0x2FDF}, {0x3005, 0x3005}, {0x3007, 0x3007}, {0x3021, 0x3029}, {0x3038, 0x303B},
{0x3400, 0x4DBF}, {0x4E00, 0x9FFF}, {0xF900, 0xFAFF}, {0x20000, 0x323AF},
};
template<size_t N>
static bool tts_norm_in(const tts_norm_range (&ranges)[N], uint32_t c) {
for (const auto & r : ranges) {
if (c >= r.lo && c <= r.hi) {
return true;
}
}
return false;
}
static bool tts_norm_is_alpha(uint32_t c) {
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
return true;
}
if (c == 0xAA || c == 0xB5 || c == 0xBA) {
return true;
}
return c >= 0xC0 && !tts_norm_in(TTS_NORM_SPACE, c) && !tts_norm_in(TTS_NORM_PUNCT, c) && !tts_norm_in(TTS_NORM_CONTROL, c);
}
// kind is one of "dspawulhc"
static bool tts_norm_class_match(char kind, uint32_t c) {
switch (kind) {
case 'd': return c >= '0' && c <= '9';
case 's': return tts_norm_in(TTS_NORM_SPACE, c);
case 'p': return tts_norm_in(TTS_NORM_PUNCT, c);
case 'a': return tts_norm_is_alpha(c);
case 'w': return tts_norm_is_alpha(c) || (c >= '0' && c <= '9');
case 'u': return c >= 'A' && c <= 'Z';
case 'l': return c >= 'a' && c <= 'z';
case 'h': return tts_norm_in(TTS_NORM_HAN, c);
case 'c': return tts_norm_in(TTS_NORM_CONTROL, c);
}
return false;
}
static bool tts_norm_is_class_letter(uint32_t c) {
if (c >= 'A' && c <= 'Z') {
c += 'a' - 'A';
}
return c < 0x80 && std::string("dspawulhc").find(static_cast<char>(c)) != std::string::npos;
}
static uint32_t tts_norm_ascii_lower(uint32_t c) {
return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c;
}
static uint32_t tts_norm_ascii_swapcase(uint32_t c) {
if (c >= 'A' && c <= 'Z') return c + ('a' - 'A');
if (c >= 'a' && c <= 'z') return c - ('a' - 'A');
return c;
}
static bool tts_norm_is_ascii_alnum(uint32_t c) {
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
}
// invalid bytes are decoded as U+FFFD, one per byte; offs[i] is the byte offset of codepoint i, offs[n] = text size
static void tts_norm_decode_utf8(const std::string & s, std::vector<uint32_t> & cpts, std::vector<size_t> & offs) {
cpts.clear();
offs.clear();
size_t i = 0;
const size_t n = s.size();
while (i < n) {
offs.push_back(i);
const unsigned char c = static_cast<unsigned char>(s[i]);
uint32_t cp = 0;
size_t len = 0;
if (c < 0x80) {
cp = c; len = 1;
} else if ((c & 0xE0) == 0xC0) {
cp = c & 0x1F; len = 2;
} else if ((c & 0xF0) == 0xE0) {
cp = c & 0x0F; len = 3;
} else if ((c & 0xF8) == 0xF0) {
cp = c & 0x07; len = 4;
}
bool ok = len > 0 && i + len <= n;
for (size_t k = 1; ok && k < len; k++) {
const unsigned char cc = static_cast<unsigned char>(s[i + k]);
if ((cc & 0xC0) != 0x80) {
ok = false;
} else {
cp = (cp << 6) | (cc & 0x3F);
}
}
if (!ok) {
cpts.push_back(0xFFFD);
i += 1;
} else {
cpts.push_back(cp);
i += len;
}
}
offs.push_back(n);
}
//
// AST
//
enum class tts_norm_op {
LITERAL, // one codepoint
SET, // one codepoint from a set; class escapes like %d are sets with a single class
ANY, // any codepoint
START, // ^
END, // $
BOUNDARY, // %b
GROUP, // (...), (?:...), (?<name>...)
LOOKAHEAD, // (?=...), (?!...)
LOOKBEHIND, // (?<=c), (?<!c) - exactly one codepoint
};
struct tts_norm_class {
char kind;
bool neg;
};
struct tts_norm_set {
bool neg = false;
std::vector<std::pair<uint32_t, uint32_t>> ranges;
std::vector<tts_norm_class> classes;
};
struct tts_norm_item;
using tts_norm_seq = std::vector<tts_norm_item>;
struct tts_norm_pattern_node {
tts_norm_op op = tts_norm_op::GROUP;
uint32_t cp = 0; // LITERAL
tts_norm_set set; // SET
std::vector<tts_norm_seq> alts; // GROUP, LOOKAHEAD; LOOKBEHIND holds a single item
int capture = -1; // GROUP: capture index, -1 for non-capturing
bool neg = false; // LOOKAHEAD, LOOKBEHIND
};
static constexpr int TTS_NORM_INF = INT_MAX;
static constexpr int TTS_NORM_MAX_REPEAT = 1000;
static constexpr size_t TTS_NORM_MAX_PATTERN_LEN = 4096;
struct tts_norm_item {
tts_norm_pattern_node atom;
int min = 1;
int max = 1;
};
static bool tts_norm_is_zero_width(tts_norm_op op) {
return op == tts_norm_op::START || op == tts_norm_op::END || op == tts_norm_op::BOUNDARY
|| op == tts_norm_op::LOOKAHEAD || op == tts_norm_op::LOOKBEHIND;
}
//
// parser
//
struct tts_norm_parser {
std::vector<uint32_t> s;
size_t p = 0;
size_t & n_caps;
tts_norm_parser(std::vector<uint32_t> src, size_t & n_caps) : s(std::move(src)), n_caps(n_caps) {}
[[noreturn]] void fail(const std::string & msg) const {
throw std::runtime_error("invalid TTS norm pattern: " + msg + " at position " + std::to_string(p));
}
bool eof() const { return p >= s.size(); }
uint32_t peek() const { return s[p]; }
bool peek_is(uint32_t c) const { return !eof() && s[p] == c; }
void expect(uint32_t c, const char * what) {
if (!peek_is(c)) {
fail(std::string("expected ") + what);
}
p++;
}
std::vector<tts_norm_seq> parse_alts() {
std::vector<tts_norm_seq> alts;
alts.push_back(parse_seq());
while (peek_is('|')) {
p++;
alts.push_back(parse_seq());
}
return alts;
}
tts_norm_seq parse_seq() {
tts_norm_seq seq;
while (!eof() && peek() != '|' && peek() != ')') {
tts_norm_item item;
item.atom = parse_atom();
parse_quantifier(item);
seq.push_back(std::move(item));
}
return seq;
}
int parse_int() {
if (eof() || peek() < '0' || peek() > '9') {
fail("expected number in quantifier");
}
int v = 0;
while (!eof() && peek() >= '0' && peek() <= '9') {
v = v * 10 + static_cast<int>(peek() - '0');
if (v > TTS_NORM_MAX_REPEAT) {
fail("repetition count exceeds " + std::to_string(TTS_NORM_MAX_REPEAT));
}
p++;
}
return v;
}
void parse_quantifier(tts_norm_item & item) {
if (eof()) {
return;
}
const uint32_t c = peek();
if (c != '?' && c != '*' && c != '+' && c != '{') {
return;
}
if (tts_norm_is_zero_width(item.atom.op)) {
fail("zero-width assertion cannot be repeated");
}
p++;
if (c == '?') {
item.min = 0; item.max = 1;
} else if (c == '*') {
item.min = 0; item.max = TTS_NORM_INF;
} else if (c == '+') {
item.min = 1; item.max = TTS_NORM_INF;
} else {
item.min = parse_int();
item.max = item.min;
if (peek_is(',')) {
p++;
item.max = peek_is('}') ? TTS_NORM_INF : parse_int();
}
expect('}', "'}' to close quantifier");
if (item.max < item.min) {
fail("quantifier max is smaller than min");
}
}
if (!eof() && (peek() == '?' || peek() == '*' || peek() == '+' || peek() == '{')) {
fail("multiple quantifiers");
}
}
tts_norm_pattern_node parse_group() {
// '(' already consumed
tts_norm_pattern_node node;
node.op = tts_norm_op::GROUP;
if (peek_is('?')) {
p++;
if (eof()) {
fail("unterminated group");
}
const uint32_t c = peek();
if (c == ':') {
p++;
} else if (c == '=' || c == '!') {
p++;
node.op = tts_norm_op::LOOKAHEAD;
node.neg = c == '!';
} else if (c == '<' && p + 1 < s.size() && (s[p + 1] == '=' || s[p + 1] == '!')) {
node.op = tts_norm_op::LOOKBEHIND;
node.neg = s[p + 1] == '!';
p += 2;
tts_norm_item item;
if (eof() || peek() == ')') {
fail("lookbehind must contain exactly one codepoint matcher");
}
item.atom = parse_atom();
if (item.atom.op != tts_norm_op::LITERAL && item.atom.op != tts_norm_op::SET && item.atom.op != tts_norm_op::ANY) {
fail("lookbehind must contain exactly one codepoint matcher");
}
if (!peek_is(')')) {
fail("lookbehind must contain exactly one codepoint matcher");
}
p++;
node.alts.push_back(tts_norm_seq{});
node.alts[0].push_back(std::move(item));
return node;
} else if (c == 'i') {
fail("(?i) is only allowed at the start of the pattern");
} else {
fail("unknown group type");
}
} else {
node.capture = static_cast<int>(n_caps++);
}
node.alts = parse_alts();
expect(')', "')' to close group");
return node;
}
// after '%' inside or outside of a set
// returns true if it is a class (stored in cls), false if it is a literal (stored in lit)
bool parse_escape(bool in_set, tts_norm_class & cls, uint32_t & lit, bool & boundary) {
boundary = false;
if (eof()) {
fail("dangling '%'");
}
const uint32_t e = s[p++];
if (tts_norm_is_class_letter(e)) {
cls.kind = static_cast<char>(tts_norm_ascii_lower(e));
cls.neg = e >= 'A' && e <= 'Z';
return true;
}
if (e == 'b' && !in_set) {
boundary = true;
return false;
}
if (tts_norm_is_ascii_alnum(e)) {
p--;
fail(std::string("unknown escape '%") + static_cast<char>(e) + "'");
}
lit = e;
return false;
}
tts_norm_pattern_node parse_set() {
// '[' already consumed
tts_norm_pattern_node node;
node.op = tts_norm_op::SET;
if (peek_is('^')) {
node.set.neg = true;
p++;
}
bool empty = true;
while (true) {
if (eof()) {
fail("unterminated set");
}
if (peek() == ']') {
p++;
break;
}
empty = false;
uint32_t lo = 0;
if (peek() == '%') {
p++;
tts_norm_class cls{};
bool boundary = false;
if (parse_escape(true, cls, lo, boundary)) {
node.set.classes.push_back(cls);
continue;
}
} else {
lo = s[p++];
}
uint32_t hi = lo;
if (peek_is('-') && p + 1 < s.size() && s[p + 1] != ']') {
p++;
if (peek() == '%') {
p++;
tts_norm_class cls{};
bool boundary = false;
if (parse_escape(true, cls, hi, boundary)) {
fail("character class cannot be used in a range");
}
} else {
hi = s[p++];
}
if (hi < lo) {
fail("invalid range in set");
}
}
node.set.ranges.emplace_back(lo, hi);
}
if (empty) {
fail("empty set");
}
return node;
}
tts_norm_pattern_node parse_atom() {
const uint32_t c = s[p++];
tts_norm_pattern_node node;
switch (c) {
case '(':
return parse_group();
case '[':
return parse_set();
case '.':
node.op = tts_norm_op::ANY;
return node;
case '^':
node.op = tts_norm_op::START;
return node;
case '$':
node.op = tts_norm_op::END;
return node;
case '%': {
tts_norm_class cls{};
uint32_t lit = 0;
bool boundary = false;
if (parse_escape(false, cls, lit, boundary)) {
node.op = tts_norm_op::SET;
node.set.classes.push_back(cls);
} else if (boundary) {
node.op = tts_norm_op::BOUNDARY;
} else {
node.op = tts_norm_op::LITERAL;
node.cp = lit;
}
return node;
}
case '?':
case '*':
case '+':
case '{':
p--;
fail("nothing to repeat");
case ']':
case '}':
p--;
fail(std::string("unescaped '") + static_cast<char>(c) + "', use '%" + static_cast<char>(c) + "'");
default:
node.op = tts_norm_op::LITERAL;
node.cp = c;
return node;
}
}
};
//
// matcher
// PEG-style: alternatives are ordered (the first one that matches wins) and quantifiers are greedy and possessive
// (they never give back what they consumed), so there is no backtracking
//
struct tts_norm_matcher {
const std::vector<uint32_t> & t;
bool icase;
std::vector<std::pair<int64_t, int64_t>> caps;
bool is_w(size_t i) const {
return i < t.size() && tts_norm_class_match('w', t[i]);
}
bool set_match(const tts_norm_set & st, uint32_t c) const {
auto test = [&](uint32_t x) {
for (const auto & r : st.ranges) {
if (x >= r.first && x <= r.second) {
return true;
}
}
for (const auto & k : st.classes) {
if (tts_norm_class_match(k.kind, x) != k.neg) {
return true;
}
}
return false;
};
bool m = test(c);
if (!m && icase) {
const uint32_t o = tts_norm_ascii_swapcase(c);
if (o != c) {
m = test(o);
}
}
return m != st.neg;
}
bool match_alts(const std::vector<tts_norm_seq> & alts, size_t i, size_t & end) {
for (const auto & seq : alts) {
auto saved = caps;
if (match_seq(seq, i, end)) {
return true;
}
caps = std::move(saved);
}
return false;
}
bool match_seq(const tts_norm_seq & seq, size_t i, size_t & end) {
size_t pos = i;
for (const auto & it : seq) {
int count = 0;
while (count < it.max) {
const bool is_group = it.atom.op == tts_norm_op::GROUP;
std::vector<std::pair<int64_t, int64_t>> saved;
if (is_group) {
saved = caps;
}
size_t e = pos;
if (!match_once(it.atom, pos, e)) {
if (is_group) {
caps = std::move(saved);
}
break;
}
if (e == pos) {
// an empty iteration would repeat forever, it satisfies any remaining minimum
count = std::max(count + 1, it.min);
break;
}
pos = e;
count++;
}
if (count < it.min) {
return false;
}
}
end = pos;
return true;
}
bool match_once(const tts_norm_pattern_node & node, size_t i, size_t & end) {
const size_t n = t.size();
switch (node.op) {
case tts_norm_op::LITERAL:
if (i < n && (t[i] == node.cp || (icase && tts_norm_ascii_lower(t[i]) == tts_norm_ascii_lower(node.cp)))) {
end = i + 1;
return true;
}
return false;
case tts_norm_op::SET:
if (i < n && set_match(node.set, t[i])) {
end = i + 1;
return true;
}
return false;
case tts_norm_op::ANY:
if (i < n) {
end = i + 1;
return true;
}
return false;
case tts_norm_op::START:
end = i;
return i == 0;
case tts_norm_op::END:
end = i;
return i == n;
case tts_norm_op::BOUNDARY:
end = i;
return (i > 0 && is_w(i - 1)) != is_w(i);
case tts_norm_op::GROUP: {
size_t e = i;
if (!match_alts(node.alts, i, e)) {
return false;
}
if (node.capture >= 0) {
caps[node.capture] = {static_cast<int64_t>(i), static_cast<int64_t>(e)};
}
end = e;
return true;
}
case tts_norm_op::LOOKAHEAD: {
// captures made inside a lookahead are discarded
auto saved = caps;
size_t e = i;
const bool ok = match_alts(node.alts, i, e);
caps = std::move(saved);
end = i;
return ok != node.neg;
}
case tts_norm_op::LOOKBEHIND: {
size_t e = 0;
const bool ok = i > 0 && match_once(node.alts[0][0].atom, i - 1, e) && e == i;
end = i;
return ok != node.neg;
}
}
return false;
}
};
//
// common_tts_norm_pattern
//
common_tts_norm_pattern::common_tts_norm_pattern(const std::string & src) {
if (src.size() > TTS_NORM_MAX_PATTERN_LEN) {
throw std::runtime_error("invalid TTS norm pattern: pattern is longer than " + std::to_string(TTS_NORM_MAX_PATTERN_LEN) + " bytes");
}
std::vector<uint32_t> cpts;
std::vector<size_t> offs;
tts_norm_decode_utf8(src, cpts, offs);
tts_norm_parser parser(std::move(cpts), n_caps);
static const std::vector<uint32_t> icase_prefix = {'(', '?', 'i', ')'};
if (parser.s.size() >= 4 && std::equal(icase_prefix.begin(), icase_prefix.end(), parser.s.begin())) {
icase = true;
parser.p = 4;
}
root = std::make_unique<tts_norm_pattern_node>();
root->op = tts_norm_op::GROUP;
root->alts = parser.parse_alts();
if (!parser.eof()) {
parser.fail("unbalanced ')'");
}
}
common_tts_norm_pattern::~common_tts_norm_pattern() = default;
common_tts_norm_pattern::common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept = default;
common_tts_norm_pattern & common_tts_norm_pattern::operator=(common_tts_norm_pattern &&) noexcept = default;
bool common_tts_norm_pattern::match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const {
tts_norm_matcher m{cpts, icase, std::vector<std::pair<int64_t, int64_t>>(n_caps, {-1, -1})};
size_t e = pos;
if (!m.match_alts(root->alts, pos, e)) {
return false;
}
end = e;
groups = std::move(m.caps);
return true;
}
size_t common_tts_norm_pattern::n_groups() const {
return n_caps;
}
//
// scan / sub
//
std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules) {
std::vector<uint32_t> cpts;
std::vector<size_t> offs;
tts_norm_decode_utf8(text, cpts, offs);
std::vector<common_tts_norm_segment> out;
const size_t n = cpts.size();
size_t i = 0;
size_t pending = 0; // start of the current unmatched run
std::vector<std::pair<int64_t, int64_t>> groups;
while (i < n) {
bool matched = false;
for (size_t r = 0; r < rules.size(); r++) {
size_t end = i;
if (rules[r]->match_at(cpts, i, end, groups) && end > i) {
if (pending < i) {
out.push_back({-1, offs[pending], offs[i], {}});
}
common_tts_norm_segment seg;
seg.rule = static_cast<int>(r);
seg.start = offs[i];
seg.end = offs[end];
for (const auto & g : groups) {
if (g.first < 0) {
seg.groups.emplace_back(std::string::npos, std::string::npos);
} else {
seg.groups.emplace_back(offs[g.first], offs[g.second]);
}
}
out.push_back(std::move(seg));
i = end;
pending = i;
matched = true;
break;
}
}
if (!matched) {
i++;
}
}
if (pending < n) {
out.push_back({-1, offs[pending], offs[n], {}});
}
return out;
}
struct tts_norm_repl_piece {
std::string literal;
int group = -1; // 0 = whole match, 1.. = groups
};
static std::vector<tts_norm_repl_piece> tts_norm_parse_repl(const std::string & repl, const common_tts_norm_pattern & pattern) {
std::vector<tts_norm_repl_piece> pieces;
std::string lit;
auto flush = [&]() {
if (!lit.empty()) {
pieces.push_back({lit, -1});
lit.clear();
}
};
for (size_t i = 0; i < repl.size(); i++) {
if (repl[i] != '%') {
lit += repl[i];
continue;
}
if (i + 1 >= repl.size()) {
throw std::runtime_error("invalid TTS norm replacement: dangling '%'");
}
const char c = repl[++i];
if (c == '%') {
lit += '%';
} else if (c >= '0' && c <= '9') {
const int g = c - '0';
if (static_cast<size_t>(g) > pattern.n_groups()) {
throw std::runtime_error("invalid TTS norm replacement: group %" + std::string(1, c) + " does not exist");
}
flush();
pieces.push_back({"", g});
} else {
throw std::runtime_error("invalid TTS norm replacement: unknown escape '%" + std::string(1, c) + "'");
}
}
flush();
return pieces;
}
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl) {
const auto pieces = tts_norm_parse_repl(repl, pattern);
std::string out;
for (const auto & seg : common_tts_norm_scan(text, {&pattern})) {
if (seg.rule < 0) {
out.append(text, seg.start, seg.end - seg.start);
continue;
}
for (const auto & piece : pieces) {
if (piece.group < 0) {
out += piece.literal;
} else if (piece.group == 0) {
out.append(text, seg.start, seg.end - seg.start);
} else {
const auto & g = seg.groups[piece.group - 1];
if (g.first != std::string::npos) {
out.append(text, g.first, g.second - g.first);
}
}
}
}
return out;
}
//
// template builtins
//
// rules can be: a pattern string (rule name "match"), a dict {name: pattern}, or a list of [name, pattern] pairs
static std::vector<std::pair<std::string, std::string>> tts_norm_rules_from_value(const value & v) {
std::vector<std::pair<std::string, std::string>> rules;
if (is_val<value_string>(v)) {
rules.emplace_back("match", v->as_string().str());
} else if (is_val<value_object>(v)) {
for (const auto & kv : v->as_ordered_object()) {
if (!is_val<value_string>(kv.second)) {
throw std::runtime_error("tts_norm_scan: rule pattern must be a string");
}
rules.emplace_back(kv.first->as_string().str(), kv.second->as_string().str());
}
} else if (is_val<value_array>(v)) {
for (const auto & item : v->as_array()) {
if (!is_val<value_array>(item) || item->as_array().size() != 2
|| !is_val<value_string>(item->as_array()[0]) || !is_val<value_string>(item->as_array()[1])) {
throw std::runtime_error("tts_norm_scan: each rule must be a [name, pattern] pair of strings");
}
rules.emplace_back(item->as_array()[0]->as_string().str(), item->as_array()[1]->as_string().str());
}
} else {
throw std::runtime_error("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs, got " + v->type());
}
return rules;
}
static value tts_norm_builtin_scan(const func_args & args) {
args.ensure_count(2, 2);
args.ensure_val<value_string>(args.get_pos(0));
const string text_val = args.get_pos(0)->as_string();
const std::string text = text_val.str();
const bool is_input = text_val.all_parts_are_input();
const auto rule_srcs = tts_norm_rules_from_value(args.get_pos(1));
std::vector<common_tts_norm_pattern> patterns;
patterns.reserve(rule_srcs.size());
std::vector<const common_tts_norm_pattern *> ptrs;
for (const auto & r : rule_srcs) {
patterns.emplace_back(r.second);
}
for (const auto & p : patterns) {
ptrs.push_back(&p);
}
auto mk_str = [&](size_t start, size_t end) -> value {
return mk_val<value_string>(string(text.substr(start, end - start), is_input));
};
auto out = mk_val<value_array>();
for (const auto & seg : common_tts_norm_scan(text, ptrs)) {
auto groups = mk_val<value_array>();
for (const auto & g : seg.groups) {
// groups that did not participate are empty strings, so they render as nothing and are falsy
groups->push_back(g.first == std::string::npos ? mk_str(0, 0) : mk_str(g.first, g.second));
}
auto obj = mk_val<value_object>();
obj->insert("rule", seg.rule < 0 ? value(mk_val<value_none>()) : value(mk_val<value_string>(rule_srcs[seg.rule].first)));
obj->insert("text", mk_str(seg.start, seg.end));
obj->insert("groups", groups);
out->push_back(obj);
}
return out;
}
static value tts_norm_builtin_sub(const func_args & args) {
args.ensure_count(3, 3);
args.ensure_vals<value_string, value_string, value_string>();
const string text_val = args.get_pos(0)->as_string();
const common_tts_norm_pattern pattern(args.get_pos(1)->as_string().str());
const std::string res = common_tts_norm_sub(text_val.str(), pattern, args.get_pos(2)->as_string().str());
return mk_val<value_string>(string(res, text_val.all_parts_are_input()));
}
void common_chat_tts_norm_add_builtins(context & ctx) {
ctx.set_val("tts_norm_scan", mk_val<value_func>("tts_norm_scan", tts_norm_builtin_scan));
ctx.set_val("tts_norm_sub", mk_val<value_func>("tts_norm_sub", tts_norm_builtin_sub));
}