server: disk prompt cache - text-only prompts with an mmproj loaded
The deploy of cb48bf224 with --cache-disk-path was rolled back: the first
request to the production 35B aborted in disk_cache_load().
server_tokens::get_tokens() has GGML_ASSERT(!has_mtmd), and has_mtmd is true
whenever an mmproj is loaded, even for a text-only prompt. Production's 35B
and gemma-26B always load one (-hf pulls it). disk_cache_collect() and
server_prompt_disk_cache::save() made the same call, so the shutdown save
would have aborted too. Every test had used --no-mmproj, which hid it.
disk_cache_text() takes the list from get_text_tokens(). A prompt is
text-only when that list is as long as size(), because media positions hold
LLAMA_TOKEN_NULL. So a text-only prompt is saved and restored with an mmproj
loaded, which is the main case (OpenWebUI chats on the 35B and gemma), and a
prompt with media is never saved or restored. When the slot itself holds
media, its reuse estimate is server_tokens::get_common_prefix(). These four
calls were the only get_tokens()/mtmd-unsafe accesses in the feature.
fork-validate --diskcache with the new mmproj cases, beside production:
15 PASS, 0 FAIL, GPU legs included. With the 0.8B's mmproj loaded, a text
chat is restored: prompt_n 58 of 596, output identical to the reference.
An image request is answered "red" with no restore, the text chat after it
is restored, and a SIGTERM with media in the slot copies no state.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cb48bf2240
commit
d9ca3278bf
2 files changed
+15
-6
No files matched your search
@@ -1043,6 +1043,12 @@ static bool add_mtmd_chunk(const server_slot & slot, const mtmd_input_chunk * ch
|
||||
// smaller prompts are not saved, and a smaller gain is not worth a file read
|
||||
static constexpr size_t DISK_CACHE_MIN_TOKENS = 256;
|
||||
|
||||
// the token list of a text-only prompt, false if it holds media (get_tokens() asserts when an mmproj is loaded)
|
||||
static bool disk_cache_text(const server_tokens & tokens, llama_tokens & out) {
|
||||
out = tokens.get_text_tokens();
|
||||
return out.size() == tokens.size();
|
||||
}
|
||||
|
||||
//
|
||||
// server_context_impl (private implementation)
|
||||
//
|
||||
@@ -1422,8 +1428,8 @@ private:
|
||||
}
|
||||
std::vector<server_slot *> todo;
|
||||
for (auto & slot : slots) {
|
||||
const llama_tokens & tokens = slot.prompt.tokens.get_tokens();
|
||||
if (tokens.size() < DISK_CACHE_MIN_TOKENS || std::find(tokens.begin(), tokens.end(), LLAMA_TOKEN_NULL) != tokens.end()) {
|
||||
llama_tokens tokens;
|
||||
if (!disk_cache_text(slot.prompt.tokens, tokens) || tokens.size() < DISK_CACHE_MIN_TOKENS) {
|
||||
continue;
|
||||
}
|
||||
// the last batch is not decoded if the shutdown came during a token wait
|
||||
@@ -1482,8 +1488,8 @@ private:
|
||||
if (vram_only && vram_cold) {
|
||||
return; // restore only while holding the GPU
|
||||
}
|
||||
const llama_tokens & tokens = tokens_new.get_tokens();
|
||||
if (std::find(tokens.begin(), tokens.end(), LLAMA_TOKEN_NULL) != tokens.end()) {
|
||||
llama_tokens tokens;
|
||||
if (!disk_cache_text(tokens_new, tokens)) {
|
||||
return;
|
||||
}
|
||||
// a recurrent or SWA state cannot be cut back to an arbitrary prefix (see update_slots)
|
||||
@@ -1492,7 +1498,9 @@ private:
|
||||
for (const auto & c : slot.prompt.checkpoints) {
|
||||
ckpt_n_tokens.push_back(c.n_tokens);
|
||||
}
|
||||
const size_t n_have = server_prompt_disk_cache::n_reuse(slot.prompt.tokens.get_tokens(), ckpt_n_tokens, tokens, can_trim);
|
||||
llama_tokens tokens_slot;
|
||||
const size_t n_have = disk_cache_text(slot.prompt.tokens, tokens_slot) ?
|
||||
server_prompt_disk_cache::n_reuse(tokens_slot, ckpt_n_tokens, tokens, can_trim) : slot.prompt.tokens.get_common_prefix(tokens_new);
|
||||
const int i = disk_cache->find(tokens, n_have, DISK_CACHE_MIN_TOKENS, can_trim);
|
||||
if (i < 0) {
|
||||
return;
|
||||
|
||||
Reference in new issue
Block a user