Files
llama.cpp/common/trie.h
T

115 lines
3.9 KiB
C++

#pragma once
#include "unicode.h"
#include <cstdint>
#include <map>
#include <set>
#include <string>
#include <string_view>
#include <vector>
// Trie for matching multiple literals.
// This is used in common_peg_until_parser and to build a GBNF exclusion grammar
struct common_trie {
// A symbol is either a UTF-8 codepoint or a token
struct symbol {
enum kind_type : uint8_t { CODEPOINT, TOKEN };
kind_type kind = CODEPOINT;
uint32_t value = 0;
static symbol codepoint(uint32_t cpt) { return { CODEPOINT, cpt }; }
static symbol token(int32_t id) { return { TOKEN, (uint32_t) id }; }
bool is_token() const { return kind == TOKEN; }
bool operator<(const symbol & other) const {
return kind != other.kind ? kind < other.kind : value < other.value;
}
bool operator==(const symbol & other) const {
return kind == other.kind && value == other.value;
}
};
struct step {
enum utf8_parse_result::status status;
common_trie::symbol symbol;
size_t bytes_consumed;
};
struct node {
std::map<symbol, size_t> children;
int32_t pattern = -1; // index of the pattern ending at this node, -1 if none
};
std::vector<node> nodes;
// every token id used by a pattern, the input is split so these match as single symbols
std::set<int32_t> tokens;
common_trie() {
create_node(); // root node
}
common_trie(const std::vector<std::string> & words) : common_trie() {
for (const auto & w : words) {
insert(w);
}
}
enum match_result { NO_MATCH, PARTIAL_MATCH, COMPLETE_MATCH };
// Read the symbol at pos. It is a token if a pattern uses the token at pos, else a UTF-8 codepoint.
step next(std::string_view sv, const std::vector<int32_t> & token_map, size_t pos) const;
// Check if a delimiter starts at the given position
match_result check_at(std::string_view sv, size_t start_pos) const;
match_result check_at(std::string_view sv, const std::vector<int32_t> & token_map, size_t start_pos) const;
// Insert a word as a sequence of Unicode codepoints, returns its pattern index
int32_t insert(const std::string & word);
// Insert a symbol sequence, returns its pattern index (insertion order, duplicates keep the first index)
int32_t insert(const std::vector<symbol> & symbols);
// Child of node on a symbol, created if missing
size_t extend(size_t node, symbol ch);
// Mark node as the end of a pattern with a value from the caller. Returns the value already there, or value if the node had none.
// A trie must take its values from mark or from insert, not both.
int32_t mark(size_t node, int32_t value);
private:
int32_t n_patterns = 0;
size_t create_node() {
size_t index = nodes.size();
nodes.emplace_back();
return index;
}
};
// Aho-Corasick automaton
struct common_aho_corasick {
common_trie t;
std::vector<size_t> fail; // failure links
std::vector<size_t> order; // states in BFS order
std::vector<int32_t> match; // longest pattern ending at each state (directly or via a suffix link), -1 if none
std::set<common_trie::symbol> alphabet; // every symbol with a transition
common_aho_corasick(common_trie trie);
common_aho_corasick(const std::vector<std::string> & strings)
: common_aho_corasick(common_trie(strings)) {}
size_t num_states() const { return t.nodes.size(); }
bool is_terminal(size_t s) const { return match[s] >= 0; }
// index of the longest pattern ending at this state, -1 if none
int32_t match_pattern(size_t s) const { return match[s]; }
// follow failure links until a transition on `ch` exists.
size_t next(size_t state, common_trie::symbol ch) const;
};