diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index ba1e462aa..b85e4755d 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1073,6 +1073,12 @@ static bool add_mtmd_chunk(const server_slot & slot, const mtmd_input_chunk * ch // smaller prompts are not saved, and a smaller gain is not worth a file read static constexpr size_t DISK_CACHE_MIN_TOKENS = 256; +// the token list of a text-only prompt, false if it holds media (get_tokens() asserts when an mmproj is loaded) +static bool disk_cache_text(const server_tokens & tokens, llama_tokens & out) { + out = tokens.get_text_tokens(); + return out.size() == tokens.size(); +} + // // server_context_impl (private implementation) // @@ -1456,8 +1462,8 @@ private: } std::vector todo; for (auto & slot : slots) { - const llama_tokens & tokens = slot.prompt.tokens.get_tokens(); - if (tokens.size() < DISK_CACHE_MIN_TOKENS || std::find(tokens.begin(), tokens.end(), LLAMA_TOKEN_NULL) != tokens.end()) { + llama_tokens tokens; + if (!disk_cache_text(slot.prompt.tokens, tokens) || tokens.size() < DISK_CACHE_MIN_TOKENS) { continue; } // the lora config is not in the file, only the server one can be restored @@ -1520,8 +1526,8 @@ private: if (vram_only && vram_cold) { return; // restore only while holding the GPU } - const llama_tokens & tokens = tokens_new.get_tokens(); - if (std::find(tokens.begin(), tokens.end(), LLAMA_TOKEN_NULL) != tokens.end()) { + llama_tokens tokens; + if (!disk_cache_text(tokens_new, tokens)) { return; } // a recurrent or SWA state cannot be cut back to an arbitrary prefix (see update_slots) @@ -1530,7 +1536,9 @@ private: for (const auto & c : slot.prompt.checkpoints) { ckpt_n_tokens.push_back(c->n_tokens); } - const size_t n_have = server_prompt_disk_cache::n_reuse(slot.prompt.tokens.get_tokens(), ckpt_n_tokens, tokens, can_trim); + llama_tokens tokens_slot; + const size_t n_have = disk_cache_text(slot.prompt.tokens, tokens_slot) ? + server_prompt_disk_cache::n_reuse(tokens_slot, ckpt_n_tokens, tokens, can_trim) : slot.prompt.tokens.get_common_prefix(tokens_new); const int i = disk_cache->find(tokens, n_have, DISK_CACHE_MIN_TOKENS, can_trim); if (i < 0) { return; diff --git a/tools/server/server-slot-sched.cpp b/tools/server/server-slot-sched.cpp index eaa1037a5..70481918f 100644 --- a/tools/server/server-slot-sched.cpp +++ b/tools/server/server-slot-sched.cpp @@ -739,7 +739,8 @@ bool server_prompt_disk_cache::load(int i, server_prompt & prompt, common_state_ } bool server_prompt_disk_cache::save(const server_prompt & prompt, const common_state_data & data) { - const llama_tokens & tokens = prompt.tokens.get_tokens(); + // text only, see disk_cache_collect + const llama_tokens tokens = prompt.tokens.get_text_tokens(); std::vector ckpts; for (const auto & c : prompt.checkpoints) {