diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index e259ef6e4..b6cf98353 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1043,6 +1043,12 @@ static bool add_mtmd_chunk(const server_slot & slot, const mtmd_input_chunk * ch // smaller prompts are not saved, and a smaller gain is not worth a file read static constexpr size_t DISK_CACHE_MIN_TOKENS = 256; +// the token list of a text-only prompt, false if it holds media (get_tokens() asserts when an mmproj is loaded) +static bool disk_cache_text(const server_tokens & tokens, llama_tokens & out) { + out = tokens.get_text_tokens(); + return out.size() == tokens.size(); +} + // // server_context_impl (private implementation) // @@ -1422,8 +1428,8 @@ private: } std::vector todo; for (auto & slot : slots) { - const llama_tokens & tokens = slot.prompt.tokens.get_tokens(); - if (tokens.size() < DISK_CACHE_MIN_TOKENS || std::find(tokens.begin(), tokens.end(), LLAMA_TOKEN_NULL) != tokens.end()) { + llama_tokens tokens; + if (!disk_cache_text(slot.prompt.tokens, tokens) || tokens.size() < DISK_CACHE_MIN_TOKENS) { continue; } // the last batch is not decoded if the shutdown came during a token wait @@ -1482,8 +1488,8 @@ private: if (vram_only && vram_cold) { return; // restore only while holding the GPU } - const llama_tokens & tokens = tokens_new.get_tokens(); - if (std::find(tokens.begin(), tokens.end(), LLAMA_TOKEN_NULL) != tokens.end()) { + llama_tokens tokens; + if (!disk_cache_text(tokens_new, tokens)) { return; } // a recurrent or SWA state cannot be cut back to an arbitrary prefix (see update_slots) @@ -1492,7 +1498,9 @@ private: for (const auto & c : slot.prompt.checkpoints) { ckpt_n_tokens.push_back(c.n_tokens); } - const size_t n_have = server_prompt_disk_cache::n_reuse(slot.prompt.tokens.get_tokens(), ckpt_n_tokens, tokens, can_trim); + llama_tokens tokens_slot; + const size_t n_have = disk_cache_text(slot.prompt.tokens, tokens_slot) ? + server_prompt_disk_cache::n_reuse(tokens_slot, ckpt_n_tokens, tokens, can_trim) : slot.prompt.tokens.get_common_prefix(tokens_new); const int i = disk_cache->find(tokens, n_have, DISK_CACHE_MIN_TOKENS, can_trim); if (i < 0) { return; diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 96253bb17..08fec6cac 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -2239,7 +2239,8 @@ bool server_prompt_disk_cache::load(int i, server_prompt & prompt, std::vector & data) { - const llama_tokens & tokens = prompt.tokens.get_tokens(); + // text only, see disk_cache_collect + const llama_tokens tokens = prompt.tokens.get_text_tokens(); std::vector ckpts; for (const auto & c : prompt.checkpoints) {