llama: fix exhausting CPU and MEM caused by dry sequence breaker (#31035)
This commit is contained in:
1 parent
5c965fd8ca
commit
63412d8737
5 files changed
+98
-13
No files matched your search
+36
-12
@@ -18,6 +18,7 @@
|
||||
#include <numeric>
|
||||
#include <random>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <stdexcept>
|
||||
|
||||
// the ring buffer works similarly to std::deque, but with a fixed capacity
|
||||
@@ -3333,9 +3334,13 @@ struct llama_sampler_dry {
|
||||
};
|
||||
|
||||
// Ported from Koboldcpp, original PR: https://github.com/LostRuins/koboldcpp/pull/982 (Original author: pi6am)
|
||||
static void get_overlapping_token_sequences(const llama_vocab & vocab, const std::string& str, std::unordered_multimap<llama_token, std::vector<llama_token>>& token_sequences, int max_tail_len = -1) {
|
||||
for (llama_token token_id = 0; token_id < (llama_token) vocab.n_tokens(); token_id++) {
|
||||
std::string word = vocab.detokenize({token_id}, true);
|
||||
static void get_overlapping_token_sequences(const llama_vocab & vocab, const std::vector<std::string> & vocab_words, const std::string& str, std::unordered_multimap<llama_token, std::vector<llama_token>>& token_sequences, int max_tail_len = -1) {
|
||||
// the tail tokenization only depends on how many characters of str were matched, so compute each one at most once
|
||||
std::vector<std::vector<llama_token>> tail_cache(str.size() + 1);
|
||||
std::vector<bool> tail_cached(str.size() + 1, false);
|
||||
|
||||
for (llama_token token_id = 0; token_id < (llama_token) vocab_words.size(); token_id++) {
|
||||
const std::string & word = vocab_words[token_id];
|
||||
if (word.find(str) != std::string::npos) {
|
||||
token_sequences.emplace(token_id, std::vector<llama_token>());
|
||||
} else {
|
||||
@@ -3352,10 +3357,14 @@ static void get_overlapping_token_sequences(const llama_vocab & vocab, const std
|
||||
}
|
||||
}
|
||||
if (match) {
|
||||
std::vector<llama_token> tokenization = vocab.tokenize(str.substr(i), false, false);
|
||||
if (max_tail_len >= 0 && tokenization.size() > (size_t)max_tail_len) {
|
||||
tokenization.resize(max_tail_len);
|
||||
if (!tail_cached[i]) {
|
||||
tail_cache[i] = vocab.tokenize(str.substr(i), false, false);
|
||||
if (max_tail_len >= 0 && tail_cache[i].size() > (size_t)max_tail_len) {
|
||||
tail_cache[i].resize(max_tail_len);
|
||||
}
|
||||
tail_cached[i] = true;
|
||||
}
|
||||
const std::vector<llama_token> & tokenization = tail_cache[i];
|
||||
|
||||
// Ensure we don't already have a duplicate matching tokenization
|
||||
auto its = token_sequences.equal_range(token_id);
|
||||
@@ -3656,6 +3665,16 @@ struct llama_sampler * llama_sampler_init_dry(const struct llama_vocab * vocab,
|
||||
|
||||
if (dry_enabled && seq_breakers != nullptr && num_breakers > 0) {
|
||||
// Process sequence breakers
|
||||
// each distinct breaker costs a full vocabulary scan: drop duplicates and stop at the cap
|
||||
std::unordered_set<std::string> seen_breakers;
|
||||
|
||||
// detokenize the vocabulary once instead of rescanning it for every breaker
|
||||
std::vector<std::string> vocab_words;
|
||||
vocab_words.reserve(vocab->n_tokens());
|
||||
for (llama_token token_id = 0; token_id < (llama_token) vocab->n_tokens(); ++token_id) {
|
||||
vocab_words.push_back(vocab->detokenize({token_id}, true));
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < num_breakers; ++i) {
|
||||
if (seq_breakers[i] == nullptr || std::strlen(seq_breakers[i]) == 0) {
|
||||
LLAMA_LOG_WARN("skipping null or empty DRY sequence breaker at index %zu\n", i);
|
||||
@@ -3663,17 +3682,22 @@ struct llama_sampler * llama_sampler_init_dry(const struct llama_vocab * vocab,
|
||||
}
|
||||
|
||||
std::string sequence_break(seq_breakers[i]);
|
||||
if (sequence_break.empty()) {
|
||||
LLAMA_LOG_WARN("skipping empty DRY sequence breaker\n");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (sequence_break.size() > MAX_CHAR_LEN) {
|
||||
LLAMA_LOG_WARN("truncating DRY sequence breaker to %d characters\n", MAX_CHAR_LEN);
|
||||
sequence_break.resize(MAX_CHAR_LEN);
|
||||
}
|
||||
|
||||
get_overlapping_token_sequences(*vocab, sequence_break, processed_breakers, MAX_SEQ_LEN);
|
||||
if (!seen_breakers.insert(sequence_break).second) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (seen_breakers.size() > LLAMA_DRY_MAX_SEQ_BREAKERS) {
|
||||
LLAMA_LOG_WARN("ignoring DRY sequence breaker at index %zu: more than %d distinct breakers\n",
|
||||
i, LLAMA_DRY_MAX_SEQ_BREAKERS);
|
||||
break;
|
||||
}
|
||||
|
||||
get_overlapping_token_sequences(*vocab, vocab_words, sequence_break, processed_breakers, MAX_SEQ_LEN);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user