From acad5e405c63dc13c04d8149eb8cf0ff96e4db82 Mon Sep 17 00:00:00 2001 From: Hiroyuki Ikegami Date: Tue, 17 Nov 2026 23:36:15 +0900 Subject: [PATCH] vocab : speed up PLaMo-2/3 tokenizer table build (#32249) Assisted-by: Claude Code (Claude Opus 5.5) --- src/llama-vocab.cpp | 80 ++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 6f23a29d8..2bb183d21 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include // @@ -1480,9 +1481,16 @@ struct llm_tokenizer_plamo2 : llm_tokenizer { to_suffix_id_.clear(); table_.clear(); - // Build token list and byte mapping - std::unordered_map suffix_to_score; - std::unordered_map token_to_id; + // Suffixes and pieces are views into the token texts of the vocab, so no strings are built + std::unordered_map suffix_to_score; + std::unordered_map token_to_id; + suffix_to_score.reserve(vocab.n_tokens() * 4); + token_to_id.reserve(vocab.n_tokens()); + + // Byte offset of the next code point + auto next_cpt = [](std::string_view s, size_t offset) -> size_t { + return offset + std::min(s.size() - offset, unicode_len_utf8(s[offset])); + }; for (size_t token_id = 0; token_id < vocab.n_tokens(); ++token_id) { const auto & entry = vocab.get_token_data(token_id); @@ -1502,18 +1510,10 @@ struct llm_tokenizer_plamo2 : llm_tokenizer { } // Add token and all its suffixes to suffix_to_score - suffix_to_score[entry.text] = entry.score; - - // Extract suffixes character by character (UTF-8 aware) - std::vector cpts = unicode_cpts_from_utf8(entry.text); - for (size_t i = 1; i < cpts.size(); ++i) { - std::string suffix; - for (size_t j = i; j < cpts.size(); ++j) { - suffix += unicode_cpt_to_utf8(cpts[j]); - } - if (suffix_to_score.find(suffix) == suffix_to_score.end()) { - suffix_to_score[suffix] = std::numeric_limits::quiet_NaN(); - } + const std::string_view text = entry.text; + suffix_to_score[text] = entry.score; + for (size_t i = text.empty() ? 0 : next_cpt(text, 0); i < text.size(); i = next_cpt(text, i)) { + suffix_to_score.try_emplace(text.substr(i), std::numeric_limits::quiet_NaN()); } } @@ -1525,44 +1525,47 @@ struct llm_tokenizer_plamo2 : llm_tokenizer { } // Build suffix list in lexicographical order of reversed strings - std::vector suffixes; + std::vector suffixes; suffixes.reserve(suffix_to_score.size() + 1); for (const auto & pair : suffix_to_score) { suffixes.push_back(pair.first); } suffixes.push_back(""); // Empty suffix - std::sort(suffixes.begin(), suffixes.end(), [](const std::string & a, const std::string & b) { - std::string rev_a(a.rbegin(), a.rend()); - std::string rev_b(b.rbegin(), b.rend()); - return rev_a < rev_b; + std::sort(suffixes.begin(), suffixes.end(), [](std::string_view a, std::string_view b) { + return std::lexicographical_compare(a.rbegin(), a.rend(), b.rbegin(), b.rend(), [](char x, char y) { + return static_cast(x) < static_cast(y); + }); }); + // Byte length of each prefix of the suffix, by code point + std::vector prefix_ends; + auto get_prefix_ends = [&](std::string_view suffix) { + prefix_ends.clear(); + for (size_t i = 0; i < suffix.size();) { + i = next_cpt(suffix, i); + prefix_ends.push_back(i); + } + }; + // Build suffix_to_id and to_suffix_id_ - std::unordered_map suffix_to_id; + std::unordered_map suffix_to_id; + suffix_to_id.reserve(suffixes.size()); int32_t num_pieces = 0; for (const auto & suffix : suffixes) { suffix_to_id[suffix] = num_pieces; if (!suffix.empty()) { - std::vector cpts = unicode_cpts_from_utf8(suffix); + get_prefix_ends(suffix); + const uint32_t first_cpt = unicode_cpts_from_utf8(std::string(suffix.substr(0, prefix_ends[0])))[0]; - std::string remaining; - for (size_t i = 1; i < cpts.size(); ++i) { - remaining += unicode_cpt_to_utf8(cpts[i]); - } - - int64_t piece_code = (static_cast(cpts[0]) << 32) | suffix_to_id[remaining]; + int64_t piece_code = (static_cast(first_cpt) << 32) | suffix_to_id[suffix.substr(prefix_ends[0])]; to_suffix_id_[piece_code] = num_pieces; // Count number of pieces for this suffix int32_t pieces_for_suffix = 1; // sentinel row - for (int32_t piece_length = static_cast(cpts.size()); piece_length > 0; --piece_length) { - std::string piece; - for (int32_t i = 0; i < piece_length; ++i) { - piece += unicode_cpt_to_utf8(cpts[i]); - } - if (suffix_to_score.find(piece) != suffix_to_score.end()) { + for (size_t end : prefix_ends) { + if (suffix_to_score.find(suffix.substr(0, end)) != suffix_to_score.end()) { pieces_for_suffix++; } } @@ -1578,12 +1581,9 @@ struct llm_tokenizer_plamo2 : llm_tokenizer { for (const auto & suffix : suffixes) { // Add all prefixes of the suffix to the table (in decreasing order of length) - std::vector cpts = unicode_cpts_from_utf8(suffix); - for (int32_t piece_length = static_cast(cpts.size()); piece_length > 0; --piece_length) { - std::string piece; - for (int32_t i = 0; i < piece_length; ++i) { - piece += unicode_cpt_to_utf8(cpts[i]); - } + get_prefix_ends(suffix); + for (int32_t piece_length = static_cast(prefix_ends.size()); piece_length > 0; --piece_length) { + const std::string_view piece = suffix.substr(0, prefix_ends[piece_length - 1]); auto score_it = suffix_to_score.find(piece); if (score_it == suffix_to_score.end()) {