server: refactor, add server_slot_sched, prompt running hash (#32154)

* server: refactor, add server_slot_sched, prompt running hash

* use sha256 hash
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-11-15 09:37:17 +01:00
1 parent 58a4a6b49c
commit 429460dc0e
11 files changed
+790 -474

No files matched your search

+78 -2
View File
@@ -6,6 +6,7 @@
#include "mtmd-helper.h"
#include "chat.h"
#include "base64.hpp"
#include "hash/hash.h"
#include "server-common.h"
@@ -525,6 +526,7 @@ server_tokens::server_tokens(mtmd::input_chunks & mtmd_chunks, bool has_mtmd) :
}
server_tokens::server_tokens(const llama_tokens & tokens, bool has_mtmd) : has_mtmd(has_mtmd), tokens(tokens) {
update_hashes();
}
llama_pos server_tokens::pos_next(int64_t n_tokens) const {
@@ -641,6 +643,7 @@ void server_tokens::push_back(llama_token tok) {
throw std::runtime_error("Invalid token");
}
tokens.emplace_back(tok);
update_hashes();
}
void server_tokens::push_back(const mtmd_input_chunk * chunk) {
@@ -654,6 +657,7 @@ void server_tokens::push_back(const mtmd_input_chunk * chunk) {
}
mtmd::input_chunk_ptr new_chunk(mtmd_input_chunk_copy(chunk));
map_idx_to_media[start_idx] = std::move(new_chunk);
update_hashes();
} else if (type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
size_t n_tokens;
const auto * text_tokens = mtmd_input_chunk_get_tokens_text(chunk, &n_tokens);
@@ -677,6 +681,7 @@ void server_tokens::push_back_placeholder(const mtmd_input_chunk * chunk) {
tokens.emplace_back(LLAMA_TOKEN_NULL);
}
map_idx_to_media[start_idx] = std::move(new_chunk);
update_hashes();
} else {
push_back(chunk);
}
@@ -701,6 +706,7 @@ void server_tokens::push_back(server_tokens & tokens) {
void server_tokens::insert(const llama_tokens & inp_tokens) {
tokens.insert(tokens.end(), inp_tokens.begin(), inp_tokens.end());
update_hashes();
}
const llama_tokens & server_tokens::get_tokens() const {
@@ -768,6 +774,10 @@ server_tokens server_tokens::deserialize(const llama_tokens & packed, bool has_m
result.map_idx_to_media[start_idx] = std::move(chunk);
}
// the hashes from the constructor do not include the media chunks
result.hashes.clear();
result.update_hashes();
if (reader.remaining() >= sizeof(llama_token)) {
throw std::runtime_error("Trailing data in server tokens state");
}
@@ -789,6 +799,7 @@ llama_tokens server_tokens::get_text_tokens() const {
void server_tokens::set_token(llama_pos pos, llama_token id) {
GGML_ASSERT(!has_mtmd); // only allow this if mtmd is disabled
tokens[pos] = id;
hashes.resize(std::min(hashes.size(), (size_t) pos / hash_block_size));
}
void server_tokens::keep_first(size_t n) {
@@ -822,6 +833,7 @@ void server_tokens::keep_first(size_t n) {
}
}
tokens.resize(n);
hashes.resize(std::min(hashes.size(), n / hash_block_size));
}
std::string server_tokens::detokenize(const llama_context * ctx, bool special) const {
@@ -838,8 +850,11 @@ std::string server_tokens::detokenize(const llama_context * ctx, bool special) c
size_t server_tokens::get_common_prefix(const server_tokens & b) const {
const size_t max_idx = std::min(tokens.size(), b.tokens.size());
// the blocks with equal hashes are not compared again
const size_t min_idx = get_common_prefix_lower(b.get_prefix_hashes());
if (!has_mtmd) {
for (size_t i = 0; i < max_idx; ++i) {
for (size_t i = min_idx; i < max_idx; ++i) {
if (tokens[i] == b.tokens[i]) {
continue;
}
@@ -850,7 +865,7 @@ size_t server_tokens::get_common_prefix(const server_tokens & b) const {
return max_idx;
}
for (size_t i = 0; i < max_idx; ++i) {
for (size_t i = min_idx; i < max_idx; ++i) {
const llama_token ai = tokens[i];
const llama_token bi = b.tokens[i];
@@ -885,6 +900,66 @@ size_t server_tokens::get_common_prefix(const server_tokens & b) const {
return max_idx; // all tokens are equal
}
void server_tokens::update_hashes() {
const size_t n_blocks = tokens.size() / hash_block_size;
std::string buf;
const auto append = [&buf](const void * data, size_t len) {
buf.append((const char *) data, len);
};
for (size_t k = hashes.size(); k < n_blocks; k++) {
const size_t i_begin = k * hash_block_size;
const size_t i_end = i_begin + hash_block_size;
// the previous hash comes first, zeros for the first block
const prefix_hash prev = k > 0 ? hashes[k - 1] : prefix_hash{};
buf.clear();
append(prev.data(), prev.size());
append(tokens.data() + i_begin, hash_block_size * sizeof(llama_token));
// a media chunk is hashed in the block of its first token, it can span over many blocks
for (auto it = map_idx_to_media.lower_bound(i_begin); it != map_idx_to_media.end() && it->first < i_end; ++it) {
const std::string_view id = mtmd_input_chunk_get_id(it->second.get());
const uint64_t fields[] = { it->first, mtmd_input_chunk_get_n_tokens(it->second.get()), id.size() };
append(fields, sizeof(fields));
append(id.data(), id.size());
}
hashes.push_back(hash_sha256(buf.data(), buf.size()));
}
}
size_t server_tokens::get_common_prefix_lower(const std::vector<prefix_hash> & b_hashes) const {
const auto & a_hashes = hashes;
// a hash covers all the tokens before it, so the equal entries come first
size_t lo = 0;
size_t hi = std::min(a_hashes.size(), b_hashes.size());
while (lo < hi) {
const size_t mid = lo + (hi - lo) / 2;
if (a_hashes[mid] == b_hashes[mid]) {
lo = mid + 1;
} else {
hi = mid;
}
}
size_t n = lo * hash_block_size;
// do not cut a media chunk, go back to its start
auto it = map_idx_to_media.upper_bound(n);
if (it != map_idx_to_media.begin()) {
--it;
if (it->first < n && n < it->first + mtmd_input_chunk_get_n_tokens(it->second.get())) {
n = it->first;
}
}
return n;
}
common_chat_msg_spans server_tokens::find_message_spans(const common_chat_msg_delimiters & delims) const {
std::map<size_t, size_t> skips;
for (const auto & it : map_idx_to_media) {
@@ -933,6 +1008,7 @@ server_tokens server_tokens::clone() const {
server_tokens res;
res.has_mtmd = has_mtmd;
res.tokens = tokens;
res.hashes = hashes;
for (auto it = map_idx_to_media.begin(); it != map_idx_to_media.end(); ++it) {
size_t idx = it->first;
const mtmd::input_chunk_ptr & chunk = it->second;