llama: fix exhausting CPU and MEM caused by dry sequence breaker (#31035)

This commit is contained in:
Trunk Trick authored and GitHub committed 2026-10-25 12:31:09 +01:00
1 parent 5c965fd8ca
commit 63412d8737
5 files changed
+98 -13

No files matched your search

+36 -12
View File
@@ -18,6 +18,7 @@
#include <numeric>
#include <random>
#include <unordered_map>
#include <unordered_set>
#include <stdexcept>
// the ring buffer works similarly to std::deque, but with a fixed capacity
@@ -3333,9 +3334,13 @@ struct llama_sampler_dry {
};
// Ported from Koboldcpp, original PR: https://github.com/LostRuins/koboldcpp/pull/982 (Original author: pi6am)
static void get_overlapping_token_sequences(const llama_vocab & vocab, const std::string& str, std::unordered_multimap<llama_token, std::vector<llama_token>>& token_sequences, int max_tail_len = -1) {
for (llama_token token_id = 0; token_id < (llama_token) vocab.n_tokens(); token_id++) {
std::string word = vocab.detokenize({token_id}, true);
static void get_overlapping_token_sequences(const llama_vocab & vocab, const std::vector<std::string> & vocab_words, const std::string& str, std::unordered_multimap<llama_token, std::vector<llama_token>>& token_sequences, int max_tail_len = -1) {
// the tail tokenization only depends on how many characters of str were matched, so compute each one at most once
std::vector<std::vector<llama_token>> tail_cache(str.size() + 1);
std::vector<bool> tail_cached(str.size() + 1, false);
for (llama_token token_id = 0; token_id < (llama_token) vocab_words.size(); token_id++) {
const std::string & word = vocab_words[token_id];
if (word.find(str) != std::string::npos) {
token_sequences.emplace(token_id, std::vector<llama_token>());
} else {
@@ -3352,10 +3357,14 @@ static void get_overlapping_token_sequences(const llama_vocab & vocab, const std
}
}
if (match) {
std::vector<llama_token> tokenization = vocab.tokenize(str.substr(i), false, false);
if (max_tail_len >= 0 && tokenization.size() > (size_t)max_tail_len) {
tokenization.resize(max_tail_len);
if (!tail_cached[i]) {
tail_cache[i] = vocab.tokenize(str.substr(i), false, false);
if (max_tail_len >= 0 && tail_cache[i].size() > (size_t)max_tail_len) {
tail_cache[i].resize(max_tail_len);
}
tail_cached[i] = true;
}
const std::vector<llama_token> & tokenization = tail_cache[i];
// Ensure we don't already have a duplicate matching tokenization
auto its = token_sequences.equal_range(token_id);
@@ -3656,6 +3665,16 @@ struct llama_sampler * llama_sampler_init_dry(const struct llama_vocab * vocab,
if (dry_enabled && seq_breakers != nullptr && num_breakers > 0) {
// Process sequence breakers
// each distinct breaker costs a full vocabulary scan: drop duplicates and stop at the cap
std::unordered_set<std::string> seen_breakers;
// detokenize the vocabulary once instead of rescanning it for every breaker
std::vector<std::string> vocab_words;
vocab_words.reserve(vocab->n_tokens());
for (llama_token token_id = 0; token_id < (llama_token) vocab->n_tokens(); ++token_id) {
vocab_words.push_back(vocab->detokenize({token_id}, true));
}
for (size_t i = 0; i < num_breakers; ++i) {
if (seq_breakers[i] == nullptr || std::strlen(seq_breakers[i]) == 0) {
LLAMA_LOG_WARN("skipping null or empty DRY sequence breaker at index %zu\n", i);
@@ -3663,17 +3682,22 @@ struct llama_sampler * llama_sampler_init_dry(const struct llama_vocab * vocab,
}
std::string sequence_break(seq_breakers[i]);
if (sequence_break.empty()) {
LLAMA_LOG_WARN("skipping empty DRY sequence breaker\n");
continue;
}
if (sequence_break.size() > MAX_CHAR_LEN) {
LLAMA_LOG_WARN("truncating DRY sequence breaker to %d characters\n", MAX_CHAR_LEN);
sequence_break.resize(MAX_CHAR_LEN);
}
get_overlapping_token_sequences(*vocab, sequence_break, processed_breakers, MAX_SEQ_LEN);
if (!seen_breakers.insert(sequence_break).second) {
continue;
}
if (seen_breakers.size() > LLAMA_DRY_MAX_SEQ_BREAKERS) {
LLAMA_LOG_WARN("ignoring DRY sequence breaker at index %zu: more than %d distinct breakers\n",
i, LLAMA_DRY_MAX_SEQ_BREAKERS);
break;
}
get_overlapping_token_sequences(*vocab, vocab_words, sequence_break, processed_breakers, MAX_SEQ_LEN);
}
}