diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 4596e788b..78c8ef149 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3424,13 +3424,19 @@ private: } void iterate(std::vector & slots, std::function callback) { - for (auto & slot : slots) { - try { - callback(slot); - } catch (const std::exception & e) { - SLT_ERR(slot, "got exception: %s\n", e.what()); - send_error(slot, std::string("got exception: ") + e.what(), ERROR_TYPE_SERVER); - slot.release(); + for (int pass = 0; pass < 2; ++pass) { + for (auto & slot : slots) { + const bool is_child = slot.is_processing() && slot.task->is_child(); + if (is_child != (pass == 1)) { + continue; + } + try { + callback(slot); + } catch (const std::exception & e) { + SLT_ERR(slot, "got exception: %s\n", e.what()); + send_error(slot, std::string("got exception: ") + e.what(), ERROR_TYPE_SERVER); + slot.release(); + } } } } @@ -4007,6 +4013,7 @@ private: // check if this is a child slot if (slot.state == SLOT_STATE_WAIT_OTHER) { SLT_DBG(slot, "%s", "waiting for parent slot to complete\n"); + SLT_DBG(slot, "%s", "__TEST_TAG_WAIT_PARENT_SLOT__\n"); return; } diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 1592330a7..4b2a8b2f7 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -1836,7 +1836,7 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro } bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) { - const int lcp_best = prompt.tokens.get_common_prefix(tokens_new); + int lcp_best = prompt.tokens.get_common_prefix(tokens_new); float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins float f_sim_best = float(lcp_best) / tokens_new.size(); @@ -1862,6 +1862,7 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) { f_keep_best = f_keep_cur; f_sim_best = f_sim_cur; + lcp_best = lcp_cur; it_best = it; } @@ -1904,6 +1905,13 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok } } + SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_RESTORE__\n"); + + // the whole cached prompt is a prefix of the new one + if (lcp_best == (int) it_best->prompt.tokens.size()) { + SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_FULL_RESTORE__\n"); + } + prompt = std::move(it_best->prompt); states.erase(it_best); diff --git a/tools/server/tests/unit/test_systemone.py b/tools/server/tests/unit/test_systemone.py index 67e128e11..0c94a06d1 100644 --- a/tools/server/tests/unit/test_systemone.py +++ b/tools/server/tests/unit/test_systemone.py @@ -122,10 +122,13 @@ def test_systemone_invalid_request(data: dict): assert "error" in res.body -def test_systemone_shared_prompt(): +def test_systemone_shared_prompt(tmp_path): global server server = ServerPreset.tinyopenjev() + server.n_slots = len(TEST_QUESTIONS) server.server_metrics = True + server.debug = True + server.log_path = str(tmp_path / "server.log") server.start() res = server.make_request("POST", "/v1/systemone", data={ "state": TEST_STATE, @@ -139,8 +142,19 @@ def test_systemone_shared_prompt(): assert n_cached % 2 == 0 assert n_processed + n_cached == res.body["usage"]["input_tokens"] - # with one slot the prompt cannot be shared, the answers must be the same + # the next request takes an empty slot as parent, so child slot 0 has a lower id than the parent + res_next = server.make_request("POST", "/v1/systemone", data={ + "state": "Unrelated follow-up case: " * 10, + "questions": TEST_QUESTIONS, + }) + assert res_next.status_code == 200 + + # in each request, each child waits for the parent during 1 batch only server.stop() + with open(server.log_path) as f: + assert f.read().count("__TEST_TAG_WAIT_PARENT_SLOT__") == 2 * (len(TEST_QUESTIONS) - 1) + + # with one slot the prompt cannot be shared, the answers must be the same server = ServerPreset.tinyopenjev() server.n_slots = 1 server.start() @@ -181,7 +195,7 @@ def test_systemone_prompt_cache(cache_ram, tmp_path): log = f.read() # the prompt cache is opt-in for decision models, and restores their states once enabled assert ("__TEST_TAG_CACHE_IDLE_SLOTS_ENABLED__" in log) == (cache_ram is not None) - assert ("found better prompt" in log) == (cache_ram is not None) + assert ("__TEST_TAG_PROMPT_CACHE_RESTORE__" in log) == (cache_ram is not None) def test_systemone_prompt_cache_child_slots(tmp_path): @@ -210,7 +224,7 @@ def test_systemone_prompt_cache_child_slots(tmp_path): decide(TEST_STATE, TEST_QUESTIONS) with open(server.log_path) as f: - n_restored = f.read().count("found better prompt with f_keep = 1.000") + n_restored = f.read().count("__TEST_TAG_PROMPT_CACHE_FULL_RESTORE__") # every overwritten state comes back whole from the prompt cache for state in states: @@ -219,7 +233,7 @@ def test_systemone_prompt_cache_child_slots(tmp_path): server.stop() with open(server.log_path) as f: - assert f.read().count("found better prompt with f_keep = 1.000") == n_restored + len(states) + assert f.read().count("__TEST_TAG_PROMPT_CACHE_FULL_RESTORE__") == n_restored + len(states) def test_systemone_images():