Files
llama.cpp/common/chat-tts-norm.h
T
Xuan-Son Nguyen f7b886fe6f chat: allow TTS model to provide custom text normalizer via jinja (#32053)
* chat: allow TTS model to provide custom text normalizer via jinja

* rm tts_norm_is_reserved_name
2026-11-11 17:34:55 +01:00

63 lines
2.7 KiB
C++

#pragma once
#include <cstdint>
#include <memory>
#include <string>
#include <utility>
#include <vector>
// TTS normalization patterns: a small, fully specified matching language for TTS text normalization templates
//
// it is NOT a regex engine: the semantics are defined by llama.cpp (see docs/tts-norm-patterns.md), so that this
// implementation and the python reference (scripts/chat_tts_norm.py) give the same result for every input
//
// two functions are exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():
// tts_norm_scan(text, rules) -> list of segments {rule, text, groups}
// tts_norm_sub(text, pattern, repl) -> string
namespace jinja {
struct context;
}
struct tts_norm_pattern_node;
class common_tts_norm_pattern {
public:
// throws std::runtime_error on invalid pattern
explicit common_tts_norm_pattern(const std::string & src);
~common_tts_norm_pattern();
common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept;
common_tts_norm_pattern & operator=(common_tts_norm_pattern &&) noexcept;
// match anchored at codepoint index pos; on success, end is the codepoint index after the match
// groups[i] holds [start, end) codepoint indices of capture i+1, or {-1, -1} if the group did not participate
bool match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const;
size_t n_groups() const;
private:
std::unique_ptr<tts_norm_pattern_node> root;
size_t n_caps = 0;
bool icase = false;
};
// segment produced by common_tts_norm_scan(); byte offsets into the input text
struct common_tts_norm_segment {
int rule = -1; // index of the matched rule, -1 for unmatched text
size_t start = 0;
size_t end = 0;
std::vector<std::pair<size_t, size_t>> groups; // byte offsets; {npos, npos} for groups that did not participate
};
// split text into matched and unmatched segments
// at each position, rules are tried in order and the first non-empty match wins; otherwise one codepoint is skipped
std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules);
// replace every non-overlapping match of pattern with repl
// repl references: %0 (whole match), %1..%9 (groups), %% (literal percent)
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl);
// add tts_norm_scan() and tts_norm_sub() to the context of a task template
// they are not jinja global builtins on purpose: chat templates are not affected
void common_chat_tts_norm_add_builtins(jinja::context & ctx);