server: visit parent slots before child slots (#30779)
* server: visit parent slots before child slots A parent slot wakes its waiting child slots once it finishes the shared prompt prefix, while slots are visited in id order. When a child task lands on a lower slot id than its parent, the child is visited before the parent wakes it and waits an extra batch to process its suffix. Non-child slots are now visited before active child slots, so every child woken by its parent joins the same batch. * server: check systemone test events through test tags The systemone tests matched free-form debug messages, including the formatted f_keep value of the prompt cache. The server now emits __TEST_TAG_WAIT_PARENT_SLOT__ when a child slot waits for its parent, __TEST_TAG_PROMPT_CACHE_RESTORE__ on every prompt cache restore and __TEST_TAG_PROMPT_CACHE_FULL_RESTORE__ when the whole cached prompt is a prefix of the new one, compared on token counts. --------- Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
1 parent
cf659f240b
commit
0956684529
3 files changed
+42
-13
No files matched your search
@@ -1836,7 +1836,7 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro
|
||||
}
|
||||
|
||||
bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) {
|
||||
const int lcp_best = prompt.tokens.get_common_prefix(tokens_new);
|
||||
int lcp_best = prompt.tokens.get_common_prefix(tokens_new);
|
||||
|
||||
float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins
|
||||
float f_sim_best = float(lcp_best) / tokens_new.size();
|
||||
@@ -1862,6 +1862,7 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
|
||||
if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) {
|
||||
f_keep_best = f_keep_cur;
|
||||
f_sim_best = f_sim_cur;
|
||||
lcp_best = lcp_cur;
|
||||
|
||||
it_best = it;
|
||||
}
|
||||
@@ -1904,6 +1905,13 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
|
||||
}
|
||||
}
|
||||
|
||||
SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_RESTORE__\n");
|
||||
|
||||
// the whole cached prompt is a prefix of the new one
|
||||
if (lcp_best == (int) it_best->prompt.tokens.size()) {
|
||||
SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_FULL_RESTORE__\n");
|
||||
}
|
||||
|
||||
prompt = std::move(it_best->prompt);
|
||||
|
||||
states.erase(it_best);
|
||||
|
||||
Reference in new issue
Block a user