#include "server-slot-sched.h" #include "common.h" #include "llama.h" #include "log.h" #include // // server_prompt // server_prompt_reuse server_prompt::get_reuse(int32_t n_prefix, bool has_new_tokens, int32_t n_swa, llama_pos pos_min) const { server_prompt_reuse res; res.n_past = n_prefix; res.pos_next = tokens.pos_next(n_prefix); // the first evaluated token attends to the positions > pos_eval - n_swa, so a SWA state // starting at pos_eval - n_swa + 1 is enough, while a state without SWA must end before pos_eval const llama_pos pos_eval = res.pos_next - (has_new_tokens ? 0 : 1); // a state or a checkpoint is useful when its pos_min is below this threshold const auto pos_min_thold = std::max(0, n_swa > 0 ? pos_eval - n_swa + 2 : pos_eval); if (pos_min < pos_min_thold) { return res; } // search for a context checkpoint const auto it = std::find_if( checkpoints.rbegin(), checkpoints.rend(), [&](const auto & ckpt) { const auto & cur = *ckpt; // guarantee that a checkpoint will result in at least one token being processed [TAG_PROMPT_LOGITS] SRV_TRC("checking checkpoint with [%d, %d] against %d...\n", cur.pos_min, cur.pos_max, pos_min_thold); // workaround for [TAG_CHECKPOINTS_FIX_POS_MIN] if (cur.pos_max > res.pos_next) { return false; } return cur.pos_min < pos_min_thold || cur.pos_min == 0; } ); if (it == checkpoints.rend()) { res.n_past = 0; res.pos_next = 0; return res; } const auto & cur = **it; res.pos_next = std::min(res.pos_next, std::max(cur.pos_min + 1, cur.pos_max)); res.n_past = std::min(tokens.size_up_to_pos(res.pos_next), (size_t) cur.n_tokens); res.ckpt = *it; return res; } // // server_prompt_cache // size_t server_prompt_cache::size() const { size_t res = 0; for (const auto & state : states) { res += state.size(); } return res; } size_t server_prompt_cache::n_tokens() const { size_t res = 0; for (const auto & state : states) { res += state.prompt.n_tokens(); } return res; } server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & prompt, size_t state_size_tgt, size_t state_size_dft) { // first check if the current state is contained fully in the cache for (auto it = states.begin(); it != states.end(); ++it) { const int cur_lcp_len = it->prompt.tokens.get_common_prefix(prompt.tokens); if (cur_lcp_len == (int) prompt.tokens.size() && are_lora_equal(it->prompt.lora, prompt.lora)) { SRV_TRC("%s", " - prompt is already in the cache, skipping\n"); return nullptr; } } // calculate checkpoints size to see if it will fit with the prompt size_t checkpoints_size = 0; for (const auto & ckpt : prompt.checkpoints) { checkpoints_size += ckpt->size(); } const size_t state_size_new = state_size_tgt + state_size_dft + checkpoints_size; // skip over-limit entries to avoid disturbing the cache if (limit_size > 0 && state_size_new > limit_size) { SRV_WRN(" - prompt state size %.3f MiB exceeds cache size limit %.3f MiB, skipping\n", state_size_new / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0)); return nullptr; } // remove any cached prompts that are fully contained in the current prompt for (auto it = states.begin(); it != states.end();) { const int len = it->prompt.tokens.get_common_prefix(prompt.tokens); if (len == (int) it->prompt.tokens.size() && are_lora_equal(it->prompt.lora, prompt.lora)) { SRV_TRC(" - removing obsolete cached prompt with length %d\n", len); it = states.erase(it); } else { ++it; } } if (limit_size > 0) { // make room before allocating the new vectors to avoid breaching the limit while (!states.empty() && size() + state_size_new > limit_size) { SRV_WRN(" - making room for prompt cache entry, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0)); states.pop_front(); } } common_state_data state_data_tgt; common_state_data state_data_dft; // check if we can allocate enough memory for the new state try { state_data_tgt.resize(state_size_tgt); state_data_dft.resize(state_size_dft); } catch (const std::bad_alloc & e) { SRV_ERR("failed to allocate memory for prompt cache state: %s\n", e.what()); limit_size = std::max(1, 0.4*size()); SRV_WRN(" - cache size limit reduced to %.3f MiB\n", limit_size / (1024.0 * 1024.0)); update(); return nullptr; } states.push_back({ /*.prompt =*/ { /*.tokens =*/ prompt.tokens.clone(), /*.checkpoints =*/ prompt.checkpoints, /*.lora =*/ prompt.lora, }, /*.data =*/ { /*.main =*/ std::move(state_data_tgt), /*.drft =*/ std::move(state_data_dft), }, }); return &states.back(); } bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, const std::vector & lora_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) { // a prompt made with another lora config is cleared before the new one starts, so it counts as empty const bool is_empty = prompt.tokens.empty() || lora_should_clear_cache(prompt.lora, lora_new); int lcp_best = is_empty ? 0 : prompt.tokens.get_common_prefix(tokens_new); float f_keep_best = is_empty ? -1.0f : float(lcp_best) / prompt.tokens.size(); // empty slot: any cache entry wins float f_sim_best = float(lcp_best) / tokens_new.size(); SRV_TRC(" - looking for better prompt, base f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best); auto it_best = states.end(); // find the most similar cached prompt, that would also preserve the most context for (auto it = states.begin(); it != states.end(); ++it) { if (lora_should_clear_cache(it->prompt.lora, lora_new)) { SRV_TRC(" - prompt with length %7zu, skipping, other lora config\n", it->prompt.tokens.size()); continue; } const int lcp_cur = it->prompt.tokens.get_common_prefix(tokens_new); const float f_keep_cur = float(lcp_cur) / it->prompt.tokens.size(); const float f_sim_cur = float(lcp_cur) / tokens_new.size(); SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur); // don't trash large prompts if (f_keep_cur < 0.25f) { continue; } if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) { f_keep_best = f_keep_cur; f_sim_best = f_sim_cur; lcp_best = lcp_cur; it_best = it; } } if (it_best != states.end()) { SRV_TRC(" - found better prompt with f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best); { auto & data = it_best->data.main; const size_t size = data.size(); const size_t n = llama_state_seq_set_data_ext(ctx_tgt, data.data(), size, id_slot, 0); if (n != size) { SRV_ERR("failed to restore state with size %zu\n", size); return false; } data.clear(); data.shrink_to_fit(); } { auto & data = it_best->data.drft; if (!data.empty()) { GGML_ASSERT(ctx_dft); const size_t size = data.size(); const size_t n = llama_state_seq_set_data_ext(ctx_dft, data.data(), size, id_slot, 0); if (n != size) { SRV_WRN("failed to restore state with size %zu\n", size); return false; } data.clear(); data.shrink_to_fit(); } } SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_RESTORE__\n"); // the whole cached prompt is a prefix of the new one if (lcp_best == (int) it_best->prompt.tokens.size()) { SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_FULL_RESTORE__\n"); } prompt = std::move(it_best->prompt); states.erase(it_best); } return true; } void server_prompt_cache::update() { if (limit_size > 0) { while (!states.empty() && size() > limit_size) { SRV_WRN(" - cache size limit reached, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0)); states.pop_front(); } } // average size per token const float size_per_token = std::max(1.0f, float(size()) / (std::max(1, n_tokens()))); // dynamically increase the token limit if it can fit in the memory limit const size_t limit_tokens_cur = limit_size > 0 ? std::max(limit_tokens, limit_size/size_per_token) : limit_tokens; if (limit_tokens > 0) { while (!states.empty() && n_tokens() > limit_tokens_cur) { SRV_WRN(" - cache token limit (%zu, est: %zu) reached, removing oldest entry (size = %.3f MiB)\n", limit_tokens, limit_tokens_cur, states.front().size() / (1024.0 * 1024.0)); states.pop_front(); } } SRV_TRC(" - cache state: %zu prompts, %.3f MiB (limits: %.3f MiB, %zu tokens, %zu est)\n", states.size(), size() / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0), limit_tokens, limit_tokens_cur); for (const auto & state : states) { SRV_TRC(" - prompt %p: %7d tokens, checkpoints: %2zu, %9.3f MiB\n", (const void *)&state, state.prompt.n_tokens(), state.prompt.checkpoints.size(), state.size() / (1024.0 * 1024.0)); } } // // server_slot_sched // static std::vector get_free_slot_infos(const std::vector & slots, size_t n_slots_needed, int exclude_id_slot) { std::vector free_slots; for (const auto & slot : slots) { if (!slot.is_processing && slot.id != exclude_id_slot) { free_slots.push_back(&slot); } } std::stable_sort(free_slots.begin(), free_slots.end(), [](const server_slot_info * a, const server_slot_info * b) { if ((a->prompt->n_tokens() == 0) != (b->prompt->n_tokens() == 0)) { return a->prompt->n_tokens() == 0; } return a->t_last_used < b->t_last_used; }); free_slots.resize(std::min(free_slots.size(), n_slots_needed)); return free_slots; } std::vector server_slot_sched::get_free_slots(const std::vector & slots, size_t n_slots_needed, int exclude_id_slot) const { std::vector res; for (const auto * slot : get_free_slot_infos(slots, n_slots_needed, exclude_id_slot)) { res.push_back(slot->id); } return res; } server_slot_plan server_slot_sched::select(const std::vector & slots, const server_task & task, const std::vector & lora) const { const server_slot_info * ret = nullptr; bool update_cache = false; // if a specific slot is requested, use it (still goes through cache update logic below) if (task.id_slot != -1 && !slots.empty()) { // note: allow id_slot to be out of bounds (wrap around) const int id_slot = task.id_slot % slots.size(); for (const auto & slot : slots) { if (slot.id == id_slot) { ret = &slot; break; } } if (ret) { // a busy slot is returned untouched, the caller defers the task if (ret->is_processing) { return { ret->id, false }; } SLT_INF(*ret, "selected slot by id (%d)\n", task.id_slot); } } // find the slot that has at least n% prompt similarity if (slot_prompt_similarity != 0.0f) { float f_sim_best = 0; for (const auto & slot : slots) { if (task.id_slot != -1 && slot.id != task.id_slot) { continue; } // skip the slot if it is not available if (slot.is_processing) { SLT_TRC(slot, " - skipping, is_processing = %d\n", slot.is_processing); continue; } const auto & tokens = slot.prompt->tokens; // skip the slot if it does not contains cached tokens if (tokens.empty()) { SLT_TRC(slot, "%s", " - skipping, slot is empty\n"); continue; } // the task would clear this slot, its state cannot be reused with another lora config if (lora_should_clear_cache(slot.prompt->lora, lora)) { SLT_TRC(slot, "%s", " - skipping, other lora config\n"); continue; } // fraction of the Longest Common Prefix length with respect to the input prompt length const size_t lcp_len = tokens.get_common_prefix(task.tokens); const float f_sim_cur = float(lcp_len) / task.tokens.size(); SLT_TRC(slot, " - checking sim = %.3f (%zu/%zu) > %.3f\n", f_sim_cur, lcp_len, task.tokens.size(), slot_prompt_similarity); // select the current slot if the criteria match if (f_sim_cur > f_sim_best && f_sim_cur > slot_prompt_similarity) { f_sim_best = f_sim_cur; ret = &slot; } } if (ret != nullptr) { const float f_keep = (f_sim_best*task.tokens.size()) / ret->prompt->tokens.size(); if (task.id_slot == -1) { SLT_INF(*ret, "selected slot by LCP similarity, f_sim_best = %.3f (> %.3f thold), f_keep = %.3f\n", f_sim_best, slot_prompt_similarity, f_keep); } // if we are about to lose a large portion of the existing context - save it in the prompt cache if (f_keep < 0.5f) { update_cache = true; } } } // find an empty slot, or the one that has been least recently used if (ret == nullptr) { const auto free_slots = get_free_slot_infos(slots, 1, -1); if (!free_slots.empty()) { ret = free_slots[0]; SLT_INF(*ret, "selected slot by LRU, n_tokens = %d, t_last = %" PRId64 "\n", ret->prompt->n_tokens(), ret->t_last_used); update_cache = true; } } if (ret == nullptr) { return {}; } // cache prompts only for the tasks that reuse them update_cache = update_cache && !task.is_stateless(); return { ret->id, update_cache }; }