server: refactor, add server_slot_sched, prompt running hash (#32154)

* server: refactor, add server_slot_sched, prompt running hash

* use sha256 hash
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-11-15 09:37:17 +01:00
1 parent 58a4a6b49c
commit 429460dc0e
11 files changed
+790 -474

No files matched your search

+58 -174
View File
@@ -10,6 +10,7 @@
#include "server-queue.h"
#include "server-realtime.h"
#include "server-schema.h"
#include "server-slot-sched.h"
#include "server-stream.h"
#include "build-info.h"
@@ -1124,8 +1125,7 @@ private:
json json_ui_settings = json::object();
// Necessary similarity of prompt for slot selection
float slot_prompt_similarity = 0.0f;
server_slot_sched slot_sched;
std::string model_name; // name of the loaded model, to be used by API
std::set<std::string> model_aliases; // additional names for the model
@@ -1441,7 +1441,7 @@ private:
n_swa = params_base.swa_full ? 0 : llama_model_n_swa(model_tgt);
// Necessary similarity of prompt for slot selection
slot_prompt_similarity = params_base.slot_prompt_similarity;
slot_sched.slot_prompt_similarity = params_base.slot_prompt_similarity;
const int n_ctx_train = llama_model_n_ctx_train(model_tgt);
@@ -1800,113 +1800,29 @@ private:
return nullptr;
}
server_slot * get_available_slot(const server_task & task) {
server_slot * ret = nullptr;
std::vector<server_slot_info> get_slot_infos() const {
std::vector<server_slot_info> res;
for (const server_slot & slot : slots) {
res.push_back({ slot.id, slot.task.get(), slot.is_processing(), slot.t_last_used, &slot.prompt });
}
return res;
}
bool update_cache = false;
// call only when the task is sure to launch on the slot
void update_prompt_cache(server_slot & slot, const server_task & task) {
SRV_TRC("%s", "updating prompt cache\n");
// if a specific slot is requested, use it (still goes through cache update logic below)
if (task.id_slot != -1) {
ret = get_slot_by_id(task.id_slot);
if (ret) {
// a busy slot is returned untouched, the caller defers the task
if (ret->is_processing()) {
return ret;
}
const int64_t t_start = ggml_time_us();
SLT_INF(*ret, "selected slot by id (%d)\n", task.id_slot);
}
slot.prompt_save(*prompt_cache);
if (!slot.prompt_load(*prompt_cache, task.tokens)) {
slot.prompt_clear();
}
// find the slot that has at least n% prompt similarity
if (slot_prompt_similarity != 0.0f) {
float f_sim_best = 0;
prompt_cache->update();
for (server_slot & slot : slots) {
if (task.id_slot != -1 && slot.id != task.id_slot) {
continue;
}
// skip the slot if it is not available
if (slot.is_processing()) {
SLT_TRC(slot, " - skipping, is_processing = %d\n", slot.is_processing());
continue;
}
const auto & tokens = slot.prompt.tokens;
// skip the slot if it does not contains cached tokens
if (tokens.empty()) {
SLT_TRC(slot, "%s", " - skipping, slot is empty\n");
continue;
}
// fraction of the Longest Common Prefix length with respect to the input prompt length
const size_t lcp_len = tokens.get_common_prefix(task.tokens);
const float f_sim_cur = float(lcp_len) / task.tokens.size();
SLT_TRC(slot, " - checking sim = %.3f (%zu/%zu) > %.3f\n", f_sim_cur, lcp_len, task.tokens.size(), slot_prompt_similarity);
// select the current slot if the criteria match
if (f_sim_cur > f_sim_best && f_sim_cur > slot_prompt_similarity) {
f_sim_best = f_sim_cur;
ret = &slot;
}
}
if (ret != nullptr) {
const float f_keep = (f_sim_best*task.tokens.size()) / ret->prompt.tokens.size();
if (task.id_slot == -1) {
SLT_INF(*ret, "selected slot by LCP similarity, f_sim_best = %.3f (> %.3f thold), f_keep = %.3f\n",
f_sim_best, slot_prompt_similarity, f_keep);
}
// if we are about to lose a large portion of the existing context - save it in the prompt cache
if (f_keep < 0.5f) {
update_cache = true;
}
}
}
// find an empty slot, or the one that has been least recently used
if (ret == nullptr) {
const auto free_slots = get_free_slots(1, -1);
if (!free_slots.empty()) {
ret = free_slots[0];
SLT_INF(*ret, "selected slot by LRU, n_tokens = %d, t_last = %" PRId64 "\n", ret->prompt.n_tokens(), ret->t_last_used);
update_cache = true;
}
}
if (ret) {
update_cache = update_cache && prompt_cache;
// cache prompts only for the tasks that reuse them
update_cache = update_cache && !task.is_stateless();
if (update_cache) {
SRV_TRC("%s", "updating prompt cache\n");
const int64_t t_start = ggml_time_us();
ret->prompt_save(*prompt_cache);
if (!ret->prompt_load(*prompt_cache, task.tokens)) {
ret->prompt_clear();
}
prompt_cache->update();
SRV_TRC("prompt cache update took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0);
}
}
return ret;
SRV_TRC("prompt cache update took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0);
}
// return true if at least one slot has been cleared
@@ -2651,24 +2567,11 @@ private:
return true;
}
// idle slots that lose the least by being reused: empty slots first, then the least recently used ones
std::vector<server_slot *> get_free_slots(size_t n_slots_needed, int exclude_id_slot) {
std::vector<server_slot *> free_slots;
for (auto & slot : slots) {
if (!slot.is_processing() && slot.id != exclude_id_slot) {
free_slots.push_back(&slot);
}
for (int id : slot_sched.get_free_slots(get_slot_infos(), n_slots_needed, exclude_id_slot)) {
free_slots.push_back(get_slot_by_id(id));
}
std::stable_sort(free_slots.begin(), free_slots.end(), [](const server_slot * a, const server_slot * b) {
if ((a->prompt.n_tokens() == 0) != (b->prompt.n_tokens() == 0)) {
return a->prompt.n_tokens() == 0;
}
return a->t_last_used < b->t_last_used;
});
free_slots.resize(std::min(free_slots.size(), n_slots_needed));
return free_slots;
}
@@ -3040,7 +2943,9 @@ private:
const int id_task = task.id;
server_slot * slot = get_available_slot(task);
const auto plan = slot_sched.select(get_slot_infos(), task);
server_slot * slot = plan.id_slot == -1 ? nullptr : get_slot_by_id(plan.id_slot);
//
// slot scheduling logic
@@ -3060,15 +2965,24 @@ private:
break;
}
std::vector<server_slot *> child_slots;
if (task.is_parent()) {
// try getting free slots for all child tasks
size_t n_child_tasks = task.child_tasks.size();
std::vector<server_slot *> child_slots = get_free_slots(n_child_tasks, slot->id);
child_slots = get_free_slots(n_child_tasks, slot->id);
if (child_slots.size() < n_child_tasks) {
SRV_DBG("not enough free slots for child tasks, n_free = %zu, n_children = %zu, defer task, id_task = %d\n", child_slots.size(), n_child_tasks, id_task);
queue_tasks.defer(std::move(task));
break;
}
}
// the task is not deferred past this point
if (plan.update_cache && prompt_cache) {
update_prompt_cache(*slot, task);
}
if (task.is_parent()) {
if (!launch_slots_with_parent_task(*slot, child_slots, std::move(task))) {
SRV_ERR("failed to launch slot with parent task, id_task = %d\n", id_task);
break; // drop the task
@@ -4209,13 +4123,6 @@ private:
// ref: https://github.com/ggml-org/llama.cpp/pull/24110
const bool has_new_tokens = (n_past < slot.task->n_tokens());
// the first evaluated token attends to the positions > pos_eval - n_swa, so a SWA state
// starting at pos_eval - n_swa + 1 is enough, while a state without SWA must end before pos_eval
const llama_pos pos_eval = pos_next - (has_new_tokens ? 0 : 1);
// a state or a checkpoint is useful when its pos_min is below this threshold
const auto pos_min_thold = std::max(0, n_swa > 0 ? pos_eval - n_swa + 2 : pos_eval);
if (n_past > 0 && n_past <= slot.prompt.n_tokens()) {
const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id);
if (pos_min == -1) {
@@ -4266,58 +4173,35 @@ private:
SLT_WRN(slot, "%s\n", st1.str().c_str());
}
if (pos_min >= pos_min_thold) {
// search for a context checkpoint
const auto it = std::find_if(
slot.prompt.checkpoints.rbegin(),
slot.prompt.checkpoints.rend(),
[&](const auto & ckpt) {
const auto & cur = *ckpt;
// guarantee that a checkpoint will result in at least one token being processed [TAG_PROMPT_LOGITS]
SLT_TRC(slot, "checking checkpoint with [%d, %d] against %d...\n", cur.pos_min, cur.pos_max, pos_min_thold);
// workaround for [TAG_CHECKPOINTS_FIX_POS_MIN]
if (cur.pos_max > pos_next) {
return false;
}
return cur.pos_min < pos_min_thold || cur.pos_min == 0;
auto reuse = slot.prompt.get_reuse(n_past, has_new_tokens, n_swa, pos_min);
if (reuse.ckpt) {
const auto & cur = *reuse.ckpt;
// restore the context checkpoint
if (!cur.load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) ||
!cur.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY)) {
if (cur.id_task != -1) {
GGML_ABORT("failed to restore context checkpoint\n");
}
);
bool do_reset = it == slot.prompt.checkpoints.rend();
if (!do_reset) {
const auto & cur = **it;
// restore the context checkpoint
if (!cur.load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) ||
!cur.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY)) {
if (cur.id_task != -1) {
GGML_ABORT("failed to restore context checkpoint\n");
}
// restored from a slot file, not guaranteed to load - fall back to full prompt re-processing
SLT_WRN(slot, "%s", "failed to load context checkpoint restored from a slot file\n");
do_reset = true;
}
}
if (!do_reset) {
const auto & cur = **it;
// restored from a slot file, not guaranteed to load - fall back to full prompt re-processing
SLT_WRN(slot, "%s", "failed to load context checkpoint restored from a slot file\n");
reuse = {};
} else {
// restore the draft's speculative state
common_speculative_set_state(spec.get(), slot.id, cur.data_spec);
pos_next = std::min(pos_next, std::max(cur.pos_min + 1, cur.pos_max));
n_past = std::min(slot.prompt.tokens.size_up_to_pos(pos_next), (size_t) cur.n_tokens);
SLT_TRC(slot, "restored context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_past = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, n_past, (float) cur.size() / 1024 / 1024);
}
if (do_reset) {
SLT_TRC(slot, "forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see %s)\n",
"https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055");
pos_next = 0;
n_past = 0;
SLT_TRC(slot, "restored context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_past = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, reuse.n_past, (float) cur.size() / 1024 / 1024);
}
}
if (reuse.n_past == 0 && !reuse.ckpt) {
SLT_TRC(slot, "forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see %s)\n",
"https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055");
}
pos_next = reuse.pos_next;
n_past = reuse.n_past;
}
{