diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index d5b4a7029..c1683d9ef 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1763,25 +1763,14 @@ private: } } - // find the slot that has been least recently used + // find an empty slot, or the one that has been least recently used if (ret == nullptr) { - int64_t t_last = -1; + const auto free_slots = get_free_slots(1, -1); - for (server_slot & slot : slots) { - // skip the slot if it is not available - if (slot.is_processing()) { - continue; - } + if (!free_slots.empty()) { + ret = free_slots[0]; - // select the current slot if the criteria match - if (!ret || slot.t_last_used <= t_last) { - t_last = slot.t_last_used; - ret = &slot; - } - } - - if (ret != nullptr) { - SLT_INF(*ret, "selected slot by LRU, t_last = %" PRId64 "\n", t_last); + SLT_INF(*ret, "selected slot by LRU, n_tokens = %d, t_last = %" PRId64 "\n", ret->prompt.n_tokens(), ret->t_last_used); update_cache = true; } @@ -2480,16 +2469,24 @@ private: return true; } + // idle slots that lose the least by being reused: empty slots first, then the least recently used ones std::vector get_free_slots(size_t n_slots_needed, int exclude_id_slot) { std::vector free_slots; for (auto & slot : slots) { if (!slot.is_processing() && slot.id != exclude_id_slot) { free_slots.push_back(&slot); } - if (free_slots.size() >= n_slots_needed) { - break; - } } + + std::stable_sort(free_slots.begin(), free_slots.end(), [](const server_slot * a, const server_slot * b) { + if ((a->prompt.n_tokens() == 0) != (b->prompt.n_tokens() == 0)) { + return a->prompt.n_tokens() == 0; + } + return a->t_last_used < b->t_last_used; + }); + + free_slots.resize(std::min(free_slots.size(), n_slots_needed)); + return free_slots; } @@ -2519,6 +2516,11 @@ private: size_t idx = 0; GGML_ASSERT(child_slots.size() == parent_task.child_tasks.size()); for (auto * slot : child_slots) { + // keep the state that the child task is about to overwrite + if (prompt_cache && slot->prompt_save(*prompt_cache)) { + prompt_cache->update(); + } + int id_child = parent_task.child_tasks[idx].id; if (!launch_slot_with_task(*slot, std::move(parent_task.child_tasks[idx]))) { SRV_ERR("failed to launch slot with child task, id_task = %d\n", id_child); diff --git a/tools/server/tests/unit/test_systemone.py b/tools/server/tests/unit/test_systemone.py index 61c2ab098..67e128e11 100644 --- a/tools/server/tests/unit/test_systemone.py +++ b/tools/server/tests/unit/test_systemone.py @@ -184,6 +184,44 @@ def test_systemone_prompt_cache(cache_ram, tmp_path): assert ("found better prompt" in log) == (cache_ram is not None) +def test_systemone_prompt_cache_child_slots(tmp_path): + global server + server = ServerPreset.tinyopenjev() + server.n_slots = len(TEST_QUESTIONS) + server.cache_ram = 100 + server.debug = True + server.log_path = str(tmp_path / "server.log") + server.start() + + states = [ + "My parcel shows delivered but it never arrived at my door.", + "The app crashes every time I open the settings page.", + "Please close my account and delete all my personal data.", + ] + angry = {"angry": TEST_QUESTIONS["angry"]} + + def decide(state, questions): + res = server.make_request("POST", "/v1/systemone", data={"state": state, "questions": questions}) + assert res.status_code == 200 + + # one state per slot, then a request whose child tasks take every slot + for state in states: + decide(state, angry) + decide(TEST_STATE, TEST_QUESTIONS) + + with open(server.log_path) as f: + n_restored = f.read().count("found better prompt with f_keep = 1.000") + + # every overwritten state comes back whole from the prompt cache + for state in states: + decide(state, angry) + + server.stop() + + with open(server.log_path) as f: + assert f.read().count("found better prompt with f_keep = 1.000") == n_restored + len(states) + + def test_systemone_images(): global server server = ServerPreset.tinyopenjev()