server: prefer empty slots and keep states overwritten by child tasks (#30567)

Slot selection takes free slots by id or by last use only, so child
tasks often land on a slot holding a parent state while empty slots
are idle, and overwrite it without saving it to the prompt cache.

Free slots are now ordered empty first, then least recently used, for
both the LRU fallback and the child slots, and a child task saves the
state it overwrites. With 4 grouped decision requests in flight and
12 states in rotation, prompt cache reads go from 5 to 36.

[no release]
This commit is contained in:
Pascal authored and GitHub committed 2026-10-15 16:24:01 +03:00
1 parent c0cb53ac4e
commit b0d239164b
2 files changed
+59 -19

No files matched your search

+21 -19
View File
@@ -1763,25 +1763,14 @@ private:
}
}
// find the slot that has been least recently used
// find an empty slot, or the one that has been least recently used
if (ret == nullptr) {
int64_t t_last = -1;
const auto free_slots = get_free_slots(1, -1);
for (server_slot & slot : slots) {
// skip the slot if it is not available
if (slot.is_processing()) {
continue;
}
if (!free_slots.empty()) {
ret = free_slots[0];
// select the current slot if the criteria match
if (!ret || slot.t_last_used <= t_last) {
t_last = slot.t_last_used;
ret = &slot;
}
}
if (ret != nullptr) {
SLT_INF(*ret, "selected slot by LRU, t_last = %" PRId64 "\n", t_last);
SLT_INF(*ret, "selected slot by LRU, n_tokens = %d, t_last = %" PRId64 "\n", ret->prompt.n_tokens(), ret->t_last_used);
update_cache = true;
}
@@ -2480,16 +2469,24 @@ private:
return true;
}
// idle slots that lose the least by being reused: empty slots first, then the least recently used ones
std::vector<server_slot *> get_free_slots(size_t n_slots_needed, int exclude_id_slot) {
std::vector<server_slot *> free_slots;
for (auto & slot : slots) {
if (!slot.is_processing() && slot.id != exclude_id_slot) {
free_slots.push_back(&slot);
}
if (free_slots.size() >= n_slots_needed) {
break;
}
}
std::stable_sort(free_slots.begin(), free_slots.end(), [](const server_slot * a, const server_slot * b) {
if ((a->prompt.n_tokens() == 0) != (b->prompt.n_tokens() == 0)) {
return a->prompt.n_tokens() == 0;
}
return a->t_last_used < b->t_last_used;
});
free_slots.resize(std::min(free_slots.size(), n_slots_needed));
return free_slots;
}
@@ -2519,6 +2516,11 @@ private:
size_t idx = 0;
GGML_ASSERT(child_slots.size() == parent_task.child_tasks.size());
for (auto * slot : child_slots) {
// keep the state that the child task is about to overwrite
if (prompt_cache && slot->prompt_save(*prompt_cache)) {
prompt_cache->update();
}
int id_child = parent_task.child_tasks[idx].id;
if (!launch_slot_with_task(*slot, std::move(parent_task.child_tasks[idx]))) {
SRV_ERR("failed to launch slot with child task, id_task = %d\n", id_child);
+38
View File
@@ -184,6 +184,44 @@ def test_systemone_prompt_cache(cache_ram, tmp_path):
assert ("found better prompt" in log) == (cache_ram is not None)
def test_systemone_prompt_cache_child_slots(tmp_path):
global server
server = ServerPreset.tinyopenjev()
server.n_slots = len(TEST_QUESTIONS)
server.cache_ram = 100
server.debug = True
server.log_path = str(tmp_path / "server.log")
server.start()
states = [
"My parcel shows delivered but it never arrived at my door.",
"The app crashes every time I open the settings page.",
"Please close my account and delete all my personal data.",
]
angry = {"angry": TEST_QUESTIONS["angry"]}
def decide(state, questions):
res = server.make_request("POST", "/v1/systemone", data={"state": state, "questions": questions})
assert res.status_code == 200
# one state per slot, then a request whose child tasks take every slot
for state in states:
decide(state, angry)
decide(TEST_STATE, TEST_QUESTIONS)
with open(server.log_path) as f:
n_restored = f.read().count("found better prompt with f_keep = 1.000")
# every overwritten state comes back whole from the prompt cache
for state in states:
decide(state, angry)
server.stop()
with open(server.log_path) as f:
assert f.read().count("found better prompt with f_keep = 1.000") == n_restored + len(states)
def test_systemone_images():
global server
server = ServerPreset.tinyopenjev()