server: prefer empty slots and keep states overwritten by child tasks (#30567)
Slot selection takes free slots by id or by last use only, so child tasks often land on a slot holding a parent state while empty slots are idle, and overwrite it without saving it to the prompt cache. Free slots are now ordered empty first, then least recently used, for both the LRU fallback and the child slots, and a child task saves the state it overwrites. With 4 grouped decision requests in flight and 12 states in rotation, prompt cache reads go from 5 to 36. [no release]
This commit is contained in:
1 parent
c0cb53ac4e
commit
b0d239164b
2 files changed
+59
-19
No files matched your search
@@ -1763,25 +1763,14 @@ private:
|
||||
}
|
||||
}
|
||||
|
||||
// find the slot that has been least recently used
|
||||
// find an empty slot, or the one that has been least recently used
|
||||
if (ret == nullptr) {
|
||||
int64_t t_last = -1;
|
||||
const auto free_slots = get_free_slots(1, -1);
|
||||
|
||||
for (server_slot & slot : slots) {
|
||||
// skip the slot if it is not available
|
||||
if (slot.is_processing()) {
|
||||
continue;
|
||||
}
|
||||
if (!free_slots.empty()) {
|
||||
ret = free_slots[0];
|
||||
|
||||
// select the current slot if the criteria match
|
||||
if (!ret || slot.t_last_used <= t_last) {
|
||||
t_last = slot.t_last_used;
|
||||
ret = &slot;
|
||||
}
|
||||
}
|
||||
|
||||
if (ret != nullptr) {
|
||||
SLT_INF(*ret, "selected slot by LRU, t_last = %" PRId64 "\n", t_last);
|
||||
SLT_INF(*ret, "selected slot by LRU, n_tokens = %d, t_last = %" PRId64 "\n", ret->prompt.n_tokens(), ret->t_last_used);
|
||||
|
||||
update_cache = true;
|
||||
}
|
||||
@@ -2480,16 +2469,24 @@ private:
|
||||
return true;
|
||||
}
|
||||
|
||||
// idle slots that lose the least by being reused: empty slots first, then the least recently used ones
|
||||
std::vector<server_slot *> get_free_slots(size_t n_slots_needed, int exclude_id_slot) {
|
||||
std::vector<server_slot *> free_slots;
|
||||
for (auto & slot : slots) {
|
||||
if (!slot.is_processing() && slot.id != exclude_id_slot) {
|
||||
free_slots.push_back(&slot);
|
||||
}
|
||||
if (free_slots.size() >= n_slots_needed) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
std::stable_sort(free_slots.begin(), free_slots.end(), [](const server_slot * a, const server_slot * b) {
|
||||
if ((a->prompt.n_tokens() == 0) != (b->prompt.n_tokens() == 0)) {
|
||||
return a->prompt.n_tokens() == 0;
|
||||
}
|
||||
return a->t_last_used < b->t_last_used;
|
||||
});
|
||||
|
||||
free_slots.resize(std::min(free_slots.size(), n_slots_needed));
|
||||
|
||||
return free_slots;
|
||||
}
|
||||
|
||||
@@ -2519,6 +2516,11 @@ private:
|
||||
size_t idx = 0;
|
||||
GGML_ASSERT(child_slots.size() == parent_task.child_tasks.size());
|
||||
for (auto * slot : child_slots) {
|
||||
// keep the state that the child task is about to overwrite
|
||||
if (prompt_cache && slot->prompt_save(*prompt_cache)) {
|
||||
prompt_cache->update();
|
||||
}
|
||||
|
||||
int id_child = parent_task.child_tasks[idx].id;
|
||||
if (!launch_slot_with_task(*slot, std::move(parent_task.child_tasks[idx]))) {
|
||||
SRV_ERR("failed to launch slot with child task, id_task = %d\n", id_child);
|
||||
|
||||
@@ -184,6 +184,44 @@ def test_systemone_prompt_cache(cache_ram, tmp_path):
|
||||
assert ("found better prompt" in log) == (cache_ram is not None)
|
||||
|
||||
|
||||
def test_systemone_prompt_cache_child_slots(tmp_path):
|
||||
global server
|
||||
server = ServerPreset.tinyopenjev()
|
||||
server.n_slots = len(TEST_QUESTIONS)
|
||||
server.cache_ram = 100
|
||||
server.debug = True
|
||||
server.log_path = str(tmp_path / "server.log")
|
||||
server.start()
|
||||
|
||||
states = [
|
||||
"My parcel shows delivered but it never arrived at my door.",
|
||||
"The app crashes every time I open the settings page.",
|
||||
"Please close my account and delete all my personal data.",
|
||||
]
|
||||
angry = {"angry": TEST_QUESTIONS["angry"]}
|
||||
|
||||
def decide(state, questions):
|
||||
res = server.make_request("POST", "/v1/systemone", data={"state": state, "questions": questions})
|
||||
assert res.status_code == 200
|
||||
|
||||
# one state per slot, then a request whose child tasks take every slot
|
||||
for state in states:
|
||||
decide(state, angry)
|
||||
decide(TEST_STATE, TEST_QUESTIONS)
|
||||
|
||||
with open(server.log_path) as f:
|
||||
n_restored = f.read().count("found better prompt with f_keep = 1.000")
|
||||
|
||||
# every overwritten state comes back whole from the prompt cache
|
||||
for state in states:
|
||||
decide(state, angry)
|
||||
|
||||
server.stop()
|
||||
|
||||
with open(server.log_path) as f:
|
||||
assert f.read().count("found better prompt with f_keep = 1.000") == n_restored + len(states)
|
||||
|
||||
|
||||
def test_systemone_images():
|
||||
global server
|
||||
server = ServerPreset.tinyopenjev()
|
||||
|
||||
Reference in new issue
Block a user