server: refactor, add server_slot_sched, prompt running hash (#32154)
* server: refactor, add server_slot_sched, prompt running hash * use sha256 hash
This commit is contained in:
1 parent
58a4a6b49c
commit
429460dc0e
11 files changed
+790
-474
No files matched your search
+58
-174
@@ -10,6 +10,7 @@
|
||||
#include "server-queue.h"
|
||||
#include "server-realtime.h"
|
||||
#include "server-schema.h"
|
||||
#include "server-slot-sched.h"
|
||||
#include "server-stream.h"
|
||||
|
||||
#include "build-info.h"
|
||||
@@ -1124,8 +1125,7 @@ private:
|
||||
|
||||
json json_ui_settings = json::object();
|
||||
|
||||
// Necessary similarity of prompt for slot selection
|
||||
float slot_prompt_similarity = 0.0f;
|
||||
server_slot_sched slot_sched;
|
||||
|
||||
std::string model_name; // name of the loaded model, to be used by API
|
||||
std::set<std::string> model_aliases; // additional names for the model
|
||||
@@ -1441,7 +1441,7 @@ private:
|
||||
n_swa = params_base.swa_full ? 0 : llama_model_n_swa(model_tgt);
|
||||
|
||||
// Necessary similarity of prompt for slot selection
|
||||
slot_prompt_similarity = params_base.slot_prompt_similarity;
|
||||
slot_sched.slot_prompt_similarity = params_base.slot_prompt_similarity;
|
||||
|
||||
const int n_ctx_train = llama_model_n_ctx_train(model_tgt);
|
||||
|
||||
@@ -1800,113 +1800,29 @@ private:
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
server_slot * get_available_slot(const server_task & task) {
|
||||
server_slot * ret = nullptr;
|
||||
std::vector<server_slot_info> get_slot_infos() const {
|
||||
std::vector<server_slot_info> res;
|
||||
for (const server_slot & slot : slots) {
|
||||
res.push_back({ slot.id, slot.task.get(), slot.is_processing(), slot.t_last_used, &slot.prompt });
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
bool update_cache = false;
|
||||
// call only when the task is sure to launch on the slot
|
||||
void update_prompt_cache(server_slot & slot, const server_task & task) {
|
||||
SRV_TRC("%s", "updating prompt cache\n");
|
||||
|
||||
// if a specific slot is requested, use it (still goes through cache update logic below)
|
||||
if (task.id_slot != -1) {
|
||||
ret = get_slot_by_id(task.id_slot);
|
||||
if (ret) {
|
||||
// a busy slot is returned untouched, the caller defers the task
|
||||
if (ret->is_processing()) {
|
||||
return ret;
|
||||
}
|
||||
const int64_t t_start = ggml_time_us();
|
||||
|
||||
SLT_INF(*ret, "selected slot by id (%d)\n", task.id_slot);
|
||||
}
|
||||
slot.prompt_save(*prompt_cache);
|
||||
|
||||
if (!slot.prompt_load(*prompt_cache, task.tokens)) {
|
||||
slot.prompt_clear();
|
||||
}
|
||||
|
||||
// find the slot that has at least n% prompt similarity
|
||||
if (slot_prompt_similarity != 0.0f) {
|
||||
float f_sim_best = 0;
|
||||
prompt_cache->update();
|
||||
|
||||
for (server_slot & slot : slots) {
|
||||
if (task.id_slot != -1 && slot.id != task.id_slot) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// skip the slot if it is not available
|
||||
if (slot.is_processing()) {
|
||||
SLT_TRC(slot, " - skipping, is_processing = %d\n", slot.is_processing());
|
||||
continue;
|
||||
}
|
||||
|
||||
const auto & tokens = slot.prompt.tokens;
|
||||
|
||||
// skip the slot if it does not contains cached tokens
|
||||
if (tokens.empty()) {
|
||||
SLT_TRC(slot, "%s", " - skipping, slot is empty\n");
|
||||
continue;
|
||||
}
|
||||
|
||||
// fraction of the Longest Common Prefix length with respect to the input prompt length
|
||||
const size_t lcp_len = tokens.get_common_prefix(task.tokens);
|
||||
const float f_sim_cur = float(lcp_len) / task.tokens.size();
|
||||
|
||||
SLT_TRC(slot, " - checking sim = %.3f (%zu/%zu) > %.3f\n", f_sim_cur, lcp_len, task.tokens.size(), slot_prompt_similarity);
|
||||
|
||||
// select the current slot if the criteria match
|
||||
if (f_sim_cur > f_sim_best && f_sim_cur > slot_prompt_similarity) {
|
||||
f_sim_best = f_sim_cur;
|
||||
|
||||
ret = &slot;
|
||||
}
|
||||
}
|
||||
|
||||
if (ret != nullptr) {
|
||||
const float f_keep = (f_sim_best*task.tokens.size()) / ret->prompt.tokens.size();
|
||||
|
||||
if (task.id_slot == -1) {
|
||||
SLT_INF(*ret, "selected slot by LCP similarity, f_sim_best = %.3f (> %.3f thold), f_keep = %.3f\n",
|
||||
f_sim_best, slot_prompt_similarity, f_keep);
|
||||
}
|
||||
|
||||
// if we are about to lose a large portion of the existing context - save it in the prompt cache
|
||||
if (f_keep < 0.5f) {
|
||||
update_cache = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// find an empty slot, or the one that has been least recently used
|
||||
if (ret == nullptr) {
|
||||
const auto free_slots = get_free_slots(1, -1);
|
||||
|
||||
if (!free_slots.empty()) {
|
||||
ret = free_slots[0];
|
||||
|
||||
SLT_INF(*ret, "selected slot by LRU, n_tokens = %d, t_last = %" PRId64 "\n", ret->prompt.n_tokens(), ret->t_last_used);
|
||||
|
||||
update_cache = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (ret) {
|
||||
update_cache = update_cache && prompt_cache;
|
||||
|
||||
// cache prompts only for the tasks that reuse them
|
||||
update_cache = update_cache && !task.is_stateless();
|
||||
|
||||
if (update_cache) {
|
||||
SRV_TRC("%s", "updating prompt cache\n");
|
||||
|
||||
const int64_t t_start = ggml_time_us();
|
||||
|
||||
ret->prompt_save(*prompt_cache);
|
||||
|
||||
if (!ret->prompt_load(*prompt_cache, task.tokens)) {
|
||||
ret->prompt_clear();
|
||||
}
|
||||
|
||||
prompt_cache->update();
|
||||
|
||||
SRV_TRC("prompt cache update took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0);
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
SRV_TRC("prompt cache update took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0);
|
||||
}
|
||||
|
||||
// return true if at least one slot has been cleared
|
||||
@@ -2651,24 +2567,11 @@ private:
|
||||
return true;
|
||||
}
|
||||
|
||||
// idle slots that lose the least by being reused: empty slots first, then the least recently used ones
|
||||
std::vector<server_slot *> get_free_slots(size_t n_slots_needed, int exclude_id_slot) {
|
||||
std::vector<server_slot *> free_slots;
|
||||
for (auto & slot : slots) {
|
||||
if (!slot.is_processing() && slot.id != exclude_id_slot) {
|
||||
free_slots.push_back(&slot);
|
||||
}
|
||||
for (int id : slot_sched.get_free_slots(get_slot_infos(), n_slots_needed, exclude_id_slot)) {
|
||||
free_slots.push_back(get_slot_by_id(id));
|
||||
}
|
||||
|
||||
std::stable_sort(free_slots.begin(), free_slots.end(), [](const server_slot * a, const server_slot * b) {
|
||||
if ((a->prompt.n_tokens() == 0) != (b->prompt.n_tokens() == 0)) {
|
||||
return a->prompt.n_tokens() == 0;
|
||||
}
|
||||
return a->t_last_used < b->t_last_used;
|
||||
});
|
||||
|
||||
free_slots.resize(std::min(free_slots.size(), n_slots_needed));
|
||||
|
||||
return free_slots;
|
||||
}
|
||||
|
||||
@@ -3040,7 +2943,9 @@ private:
|
||||
|
||||
const int id_task = task.id;
|
||||
|
||||
server_slot * slot = get_available_slot(task);
|
||||
const auto plan = slot_sched.select(get_slot_infos(), task);
|
||||
|
||||
server_slot * slot = plan.id_slot == -1 ? nullptr : get_slot_by_id(plan.id_slot);
|
||||
|
||||
//
|
||||
// slot scheduling logic
|
||||
@@ -3060,15 +2965,24 @@ private:
|
||||
break;
|
||||
}
|
||||
|
||||
std::vector<server_slot *> child_slots;
|
||||
if (task.is_parent()) {
|
||||
// try getting free slots for all child tasks
|
||||
size_t n_child_tasks = task.child_tasks.size();
|
||||
std::vector<server_slot *> child_slots = get_free_slots(n_child_tasks, slot->id);
|
||||
child_slots = get_free_slots(n_child_tasks, slot->id);
|
||||
if (child_slots.size() < n_child_tasks) {
|
||||
SRV_DBG("not enough free slots for child tasks, n_free = %zu, n_children = %zu, defer task, id_task = %d\n", child_slots.size(), n_child_tasks, id_task);
|
||||
queue_tasks.defer(std::move(task));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// the task is not deferred past this point
|
||||
if (plan.update_cache && prompt_cache) {
|
||||
update_prompt_cache(*slot, task);
|
||||
}
|
||||
|
||||
if (task.is_parent()) {
|
||||
if (!launch_slots_with_parent_task(*slot, child_slots, std::move(task))) {
|
||||
SRV_ERR("failed to launch slot with parent task, id_task = %d\n", id_task);
|
||||
break; // drop the task
|
||||
@@ -4209,13 +4123,6 @@ private:
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/24110
|
||||
const bool has_new_tokens = (n_past < slot.task->n_tokens());
|
||||
|
||||
// the first evaluated token attends to the positions > pos_eval - n_swa, so a SWA state
|
||||
// starting at pos_eval - n_swa + 1 is enough, while a state without SWA must end before pos_eval
|
||||
const llama_pos pos_eval = pos_next - (has_new_tokens ? 0 : 1);
|
||||
|
||||
// a state or a checkpoint is useful when its pos_min is below this threshold
|
||||
const auto pos_min_thold = std::max(0, n_swa > 0 ? pos_eval - n_swa + 2 : pos_eval);
|
||||
|
||||
if (n_past > 0 && n_past <= slot.prompt.n_tokens()) {
|
||||
const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id);
|
||||
if (pos_min == -1) {
|
||||
@@ -4266,58 +4173,35 @@ private:
|
||||
SLT_WRN(slot, "%s\n", st1.str().c_str());
|
||||
}
|
||||
|
||||
if (pos_min >= pos_min_thold) {
|
||||
// search for a context checkpoint
|
||||
const auto it = std::find_if(
|
||||
slot.prompt.checkpoints.rbegin(),
|
||||
slot.prompt.checkpoints.rend(),
|
||||
[&](const auto & ckpt) {
|
||||
const auto & cur = *ckpt;
|
||||
// guarantee that a checkpoint will result in at least one token being processed [TAG_PROMPT_LOGITS]
|
||||
SLT_TRC(slot, "checking checkpoint with [%d, %d] against %d...\n", cur.pos_min, cur.pos_max, pos_min_thold);
|
||||
// workaround for [TAG_CHECKPOINTS_FIX_POS_MIN]
|
||||
if (cur.pos_max > pos_next) {
|
||||
return false;
|
||||
}
|
||||
return cur.pos_min < pos_min_thold || cur.pos_min == 0;
|
||||
auto reuse = slot.prompt.get_reuse(n_past, has_new_tokens, n_swa, pos_min);
|
||||
|
||||
if (reuse.ckpt) {
|
||||
const auto & cur = *reuse.ckpt;
|
||||
|
||||
// restore the context checkpoint
|
||||
if (!cur.load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) ||
|
||||
!cur.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY)) {
|
||||
if (cur.id_task != -1) {
|
||||
GGML_ABORT("failed to restore context checkpoint\n");
|
||||
}
|
||||
);
|
||||
|
||||
bool do_reset = it == slot.prompt.checkpoints.rend();
|
||||
|
||||
if (!do_reset) {
|
||||
const auto & cur = **it;
|
||||
|
||||
// restore the context checkpoint
|
||||
if (!cur.load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) ||
|
||||
!cur.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY)) {
|
||||
if (cur.id_task != -1) {
|
||||
GGML_ABORT("failed to restore context checkpoint\n");
|
||||
}
|
||||
// restored from a slot file, not guaranteed to load - fall back to full prompt re-processing
|
||||
SLT_WRN(slot, "%s", "failed to load context checkpoint restored from a slot file\n");
|
||||
do_reset = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!do_reset) {
|
||||
const auto & cur = **it;
|
||||
|
||||
// restored from a slot file, not guaranteed to load - fall back to full prompt re-processing
|
||||
SLT_WRN(slot, "%s", "failed to load context checkpoint restored from a slot file\n");
|
||||
reuse = {};
|
||||
} else {
|
||||
// restore the draft's speculative state
|
||||
common_speculative_set_state(spec.get(), slot.id, cur.data_spec);
|
||||
|
||||
pos_next = std::min(pos_next, std::max(cur.pos_min + 1, cur.pos_max));
|
||||
n_past = std::min(slot.prompt.tokens.size_up_to_pos(pos_next), (size_t) cur.n_tokens);
|
||||
SLT_TRC(slot, "restored context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_past = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, n_past, (float) cur.size() / 1024 / 1024);
|
||||
}
|
||||
|
||||
if (do_reset) {
|
||||
SLT_TRC(slot, "forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see %s)\n",
|
||||
"https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055");
|
||||
pos_next = 0;
|
||||
n_past = 0;
|
||||
SLT_TRC(slot, "restored context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_past = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, reuse.n_past, (float) cur.size() / 1024 / 1024);
|
||||
}
|
||||
}
|
||||
|
||||
if (reuse.n_past == 0 && !reuse.ckpt) {
|
||||
SLT_TRC(slot, "forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see %s)\n",
|
||||
"https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055");
|
||||
}
|
||||
|
||||
pos_next = reuse.pos_next;
|
||||
n_past = reuse.n_past;
|
||||
}
|
||||
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user