Files
llama.cpp/common/trie.cpp
T

158 lines
4.8 KiB
C++

#include "trie.h"
#include "unicode.h"
#include <deque>
common_trie::step common_trie::next(std::string_view sv, const std::vector<int32_t> & token_map, size_t pos) const {
if (!tokens.empty() && pos < token_map.size()) {
const int32_t id = token_map[pos];
if (id >= 0 && tokens.find(id) != tokens.end()) {
size_t end = pos + 1;
while (end < token_map.size() && token_map[end] < 0) {
++end;
}
return { utf8_parse_result::SUCCESS, symbol::token(id), end - pos };
}
}
auto result = common_parse_utf8_codepoint(sv, pos);
return { result.status, symbol::codepoint(result.codepoint), result.bytes_consumed };
}
common_trie::match_result common_trie::check_at(std::string_view sv, size_t start_pos) const {
return check_at(sv, {}, start_pos);
}
common_trie::match_result common_trie::check_at(std::string_view sv, const std::vector<int32_t> & token_map, size_t start_pos) const {
size_t current = 0; // Start at root
size_t pos = start_pos;
// LOG_DBG("%s: checking at pos %zu, sv='%s'\n", __func__, start_pos, std::string(sv).c_str());
while (pos < sv.size()) {
auto step = next(sv, token_map, pos);
if (step.status != utf8_parse_result::SUCCESS) {
break;
}
auto it = nodes[current].children.find(step.symbol);
if (it == nodes[current].children.end()) {
// Can't continue matching
return match_result{match_result::NO_MATCH};
}
current = it->second;
pos += step.bytes_consumed;
// Check if we've matched a complete word
if (nodes[current].pattern >= 0) {
return match_result{match_result::COMPLETE_MATCH};
}
}
// Reached end of input while still in the trie (not at root)
if (current != 0) {
// We're in the middle of a potential match
return match_result{match_result::PARTIAL_MATCH};
}
// Reached end at root (no match)
return match_result{match_result::NO_MATCH};
}
int32_t common_trie::insert(const std::string & word) {
std::vector<symbol> symbols;
size_t pos = 0;
while (pos < word.length()) {
auto result = common_parse_utf8_codepoint(word, pos);
if (result.status != utf8_parse_result::SUCCESS) {
break;
}
symbols.push_back(symbol::codepoint(result.codepoint));
pos += result.bytes_consumed;
}
return insert(symbols);
}
int32_t common_trie::insert(const std::vector<symbol> & symbols) {
size_t current = 0;
for (symbol ch : symbols) {
current = extend(current, ch);
}
int32_t pattern = mark(current, n_patterns);
if (pattern == n_patterns) {
n_patterns++;
}
return pattern;
}
size_t common_trie::extend(size_t node, symbol ch) {
if (ch.is_token()) {
tokens.insert((int32_t) ch.value);
}
auto it = nodes[node].children.find(ch);
if (it != nodes[node].children.end()) {
return it->second;
}
size_t child = create_node();
nodes[node].children[ch] = child;
return child;
}
int32_t common_trie::mark(size_t node, int32_t value) {
if (nodes[node].pattern < 0) {
nodes[node].pattern = value;
}
return nodes[node].pattern;
}
common_aho_corasick::common_aho_corasick(common_trie trie) : t(std::move(trie)) {
const auto & nodes = t.nodes;
const size_t n = nodes.size();
fail.assign(n, 0);
order.reserve(n);
std::deque<size_t> queue{ 0 };
while (!queue.empty()) {
size_t u = queue.front();
queue.pop_front();
order.push_back(u);
for (const auto & [ch, v] : nodes[u].children) {
if (u != 0) {
size_t f = fail[u];
while (f && nodes[f].children.find(ch) == nodes[f].children.end()) {
f = fail[f];
}
auto it = nodes[f].children.find(ch);
fail[v] = (it != nodes[f].children.end() && it->second != v) ? it->second : 0;
}
queue.push_back(v);
}
}
// fail[u] points to a strictly shorter suffix, so the first pattern found on
// the fail chain (including u itself) is the longest pattern ending at u
match.assign(n, -1);
for (size_t u : order) {
match[u] = nodes[u].pattern >= 0 ? nodes[u].pattern : (u != 0 ? match[fail[u]] : -1);
}
for (const auto & node : nodes) {
for (const auto & [ch, v] : node.children) {
alphabet.insert(ch);
}
}
}
size_t common_aho_corasick::next(size_t state, common_trie::symbol ch) const {
const auto & nodes = t.nodes;
while (state && nodes[state].children.find(ch) == nodes[state].children.end()) {
state = fail[state];
}
auto it = nodes[state].children.find(ch);
return it != nodes[state].children.end() ? it->second : 0;
}