158 lines
4.8 KiB
C++
158 lines
4.8 KiB
C++
#include "trie.h"
|
|
|
|
#include "unicode.h"
|
|
|
|
#include <deque>
|
|
|
|
common_trie::step common_trie::next(std::string_view sv, const std::vector<int32_t> & token_map, size_t pos) const {
|
|
if (!tokens.empty() && pos < token_map.size()) {
|
|
const int32_t id = token_map[pos];
|
|
if (id >= 0 && tokens.find(id) != tokens.end()) {
|
|
size_t end = pos + 1;
|
|
while (end < token_map.size() && token_map[end] < 0) {
|
|
++end;
|
|
}
|
|
return { utf8_parse_result::SUCCESS, symbol::token(id), end - pos };
|
|
}
|
|
}
|
|
|
|
auto result = common_parse_utf8_codepoint(sv, pos);
|
|
return { result.status, symbol::codepoint(result.codepoint), result.bytes_consumed };
|
|
}
|
|
|
|
common_trie::match_result common_trie::check_at(std::string_view sv, size_t start_pos) const {
|
|
return check_at(sv, {}, start_pos);
|
|
}
|
|
|
|
common_trie::match_result common_trie::check_at(std::string_view sv, const std::vector<int32_t> & token_map, size_t start_pos) const {
|
|
size_t current = 0; // Start at root
|
|
size_t pos = start_pos;
|
|
|
|
// LOG_DBG("%s: checking at pos %zu, sv='%s'\n", __func__, start_pos, std::string(sv).c_str());
|
|
|
|
while (pos < sv.size()) {
|
|
auto step = next(sv, token_map, pos);
|
|
if (step.status != utf8_parse_result::SUCCESS) {
|
|
break;
|
|
}
|
|
|
|
auto it = nodes[current].children.find(step.symbol);
|
|
if (it == nodes[current].children.end()) {
|
|
// Can't continue matching
|
|
return match_result{match_result::NO_MATCH};
|
|
}
|
|
|
|
current = it->second;
|
|
pos += step.bytes_consumed;
|
|
|
|
// Check if we've matched a complete word
|
|
if (nodes[current].pattern >= 0) {
|
|
return match_result{match_result::COMPLETE_MATCH};
|
|
}
|
|
}
|
|
|
|
// Reached end of input while still in the trie (not at root)
|
|
if (current != 0) {
|
|
// We're in the middle of a potential match
|
|
return match_result{match_result::PARTIAL_MATCH};
|
|
}
|
|
|
|
// Reached end at root (no match)
|
|
return match_result{match_result::NO_MATCH};
|
|
}
|
|
|
|
int32_t common_trie::insert(const std::string & word) {
|
|
std::vector<symbol> symbols;
|
|
size_t pos = 0;
|
|
while (pos < word.length()) {
|
|
auto result = common_parse_utf8_codepoint(word, pos);
|
|
if (result.status != utf8_parse_result::SUCCESS) {
|
|
break;
|
|
}
|
|
|
|
symbols.push_back(symbol::codepoint(result.codepoint));
|
|
pos += result.bytes_consumed;
|
|
}
|
|
return insert(symbols);
|
|
}
|
|
|
|
int32_t common_trie::insert(const std::vector<symbol> & symbols) {
|
|
size_t current = 0;
|
|
for (symbol ch : symbols) {
|
|
current = extend(current, ch);
|
|
}
|
|
int32_t pattern = mark(current, n_patterns);
|
|
if (pattern == n_patterns) {
|
|
n_patterns++;
|
|
}
|
|
return pattern;
|
|
}
|
|
|
|
size_t common_trie::extend(size_t node, symbol ch) {
|
|
if (ch.is_token()) {
|
|
tokens.insert((int32_t) ch.value);
|
|
}
|
|
auto it = nodes[node].children.find(ch);
|
|
if (it != nodes[node].children.end()) {
|
|
return it->second;
|
|
}
|
|
size_t child = create_node();
|
|
nodes[node].children[ch] = child;
|
|
return child;
|
|
}
|
|
|
|
int32_t common_trie::mark(size_t node, int32_t value) {
|
|
if (nodes[node].pattern < 0) {
|
|
nodes[node].pattern = value;
|
|
}
|
|
return nodes[node].pattern;
|
|
}
|
|
|
|
common_aho_corasick::common_aho_corasick(common_trie trie) : t(std::move(trie)) {
|
|
const auto & nodes = t.nodes;
|
|
const size_t n = nodes.size();
|
|
|
|
fail.assign(n, 0);
|
|
order.reserve(n);
|
|
|
|
std::deque<size_t> queue{ 0 };
|
|
while (!queue.empty()) {
|
|
size_t u = queue.front();
|
|
queue.pop_front();
|
|
order.push_back(u);
|
|
for (const auto & [ch, v] : nodes[u].children) {
|
|
if (u != 0) {
|
|
size_t f = fail[u];
|
|
while (f && nodes[f].children.find(ch) == nodes[f].children.end()) {
|
|
f = fail[f];
|
|
}
|
|
auto it = nodes[f].children.find(ch);
|
|
fail[v] = (it != nodes[f].children.end() && it->second != v) ? it->second : 0;
|
|
}
|
|
queue.push_back(v);
|
|
}
|
|
}
|
|
|
|
// fail[u] points to a strictly shorter suffix, so the first pattern found on
|
|
// the fail chain (including u itself) is the longest pattern ending at u
|
|
match.assign(n, -1);
|
|
for (size_t u : order) {
|
|
match[u] = nodes[u].pattern >= 0 ? nodes[u].pattern : (u != 0 ? match[fail[u]] : -1);
|
|
}
|
|
|
|
for (const auto & node : nodes) {
|
|
for (const auto & [ch, v] : node.children) {
|
|
alphabet.insert(ch);
|
|
}
|
|
}
|
|
}
|
|
|
|
size_t common_aho_corasick::next(size_t state, common_trie::symbol ch) const {
|
|
const auto & nodes = t.nodes;
|
|
while (state && nodes[state].children.find(ch) == nodes[state].children.end()) {
|
|
state = fail[state];
|
|
}
|
|
auto it = nodes[state].children.find(ch);
|
|
return it != nodes[state].children.end() ? it->second : 0;
|
|
}
|