From d9ca3278bf74267b68aa5ca68fab3538c9a777ff Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Sun, 25 Oct 2026 21:38:58 +0100 Subject: [PATCH] server: disk prompt cache - text-only prompts with an mmproj loaded The deploy of cb48bf224 with --cache-disk-path was rolled back: the first request to the production 35B aborted in disk_cache_load(). server_tokens::get_tokens() has GGML_ASSERT(!has_mtmd), and has_mtmd is true whenever an mmproj is loaded, even for a text-only prompt. Production's 35B and gemma-26B always load one (-hf pulls it). disk_cache_collect() and server_prompt_disk_cache::save() made the same call, so the shutdown save would have aborted too. Every test had used --no-mmproj, which hid it. disk_cache_text() takes the list from get_text_tokens(). A prompt is text-only when that list is as long as size(), because media positions hold LLAMA_TOKEN_NULL. So a text-only prompt is saved and restored with an mmproj loaded, which is the main case (OpenWebUI chats on the 35B and gemma), and a prompt with media is never saved or restored. When the slot itself holds media, its reuse estimate is server_tokens::get_common_prefix(). These four calls were the only get_tokens()/mtmd-unsafe accesses in the feature. fork-validate --diskcache with the new mmproj cases, beside production: 15 PASS, 0 FAIL, GPU legs included. With the 0.8B's mmproj loaded, a text chat is restored: prompt_n 58 of 596, output identical to the reference. An image request is answered "red" with no restore, the text chat after it is restored, and a SIGTERM with media in the slot copies no state. Co-Authored-By: Claude Opus 5.5 --- tools/server/server-context.cpp | 18 +++++++++++++----- tools/server/server-task.cpp | 3 ++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index e259ef6e4..b6cf98353 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1043,6 +1043,12 @@ static bool add_mtmd_chunk(const server_slot & slot, const mtmd_input_chunk * ch // smaller prompts are not saved, and a smaller gain is not worth a file read static constexpr size_t DISK_CACHE_MIN_TOKENS = 256; +// the token list of a text-only prompt, false if it holds media (get_tokens() asserts when an mmproj is loaded) +static bool disk_cache_text(const server_tokens & tokens, llama_tokens & out) { + out = tokens.get_text_tokens(); + return out.size() == tokens.size(); +} + // // server_context_impl (private implementation) // @@ -1422,8 +1428,8 @@ private: } std::vector todo; for (auto & slot : slots) { - const llama_tokens & tokens = slot.prompt.tokens.get_tokens(); - if (tokens.size() < DISK_CACHE_MIN_TOKENS || std::find(tokens.begin(), tokens.end(), LLAMA_TOKEN_NULL) != tokens.end()) { + llama_tokens tokens; + if (!disk_cache_text(slot.prompt.tokens, tokens) || tokens.size() < DISK_CACHE_MIN_TOKENS) { continue; } // the last batch is not decoded if the shutdown came during a token wait @@ -1482,8 +1488,8 @@ private: if (vram_only && vram_cold) { return; // restore only while holding the GPU } - const llama_tokens & tokens = tokens_new.get_tokens(); - if (std::find(tokens.begin(), tokens.end(), LLAMA_TOKEN_NULL) != tokens.end()) { + llama_tokens tokens; + if (!disk_cache_text(tokens_new, tokens)) { return; } // a recurrent or SWA state cannot be cut back to an arbitrary prefix (see update_slots) @@ -1492,7 +1498,9 @@ private: for (const auto & c : slot.prompt.checkpoints) { ckpt_n_tokens.push_back(c.n_tokens); } - const size_t n_have = server_prompt_disk_cache::n_reuse(slot.prompt.tokens.get_tokens(), ckpt_n_tokens, tokens, can_trim); + llama_tokens tokens_slot; + const size_t n_have = disk_cache_text(slot.prompt.tokens, tokens_slot) ? + server_prompt_disk_cache::n_reuse(tokens_slot, ckpt_n_tokens, tokens, can_trim) : slot.prompt.tokens.get_common_prefix(tokens_new); const int i = disk_cache->find(tokens, n_have, DISK_CACHE_MIN_TOKENS, can_trim); if (i < 0) { return; diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 96253bb17..08fec6cac 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -2239,7 +2239,8 @@ bool server_prompt_disk_cache::load(int i, server_prompt & prompt, std::vector & data) { - const llama_tokens & tokens = prompt.tokens.get_tokens(); + // text only, see disk_cache_collect + const llama_tokens tokens = prompt.tokens.get_text_tokens(); std::vector ckpts; for (const auto & c : prompt.checkpoints) {