server: prefer empty slots and keep states overwritten by child tasks (#30567)
Slot selection takes free slots by id or by last use only, so child tasks often land on a slot holding a parent state while empty slots are idle, and overwrite it without saving it to the prompt cache. Free slots are now ordered empty first, then least recently used, for both the LRU fallback and the child slots, and a child task saves the state it overwrites. With 4 grouped decision requests in flight and 12 states in rotation, prompt cache reads go from 5 to 36. [no release]
This commit is contained in:
1 parent
c0cb53ac4e
commit
b0d239164b
2 files changed
+59
-19
No files matched your search
@@ -1763,25 +1763,14 @@ private:
|
||||
}
|
||||
}
|
||||
|
||||
// find the slot that has been least recently used
|
||||
// find an empty slot, or the one that has been least recently used
|
||||
if (ret == nullptr) {
|
||||
int64_t t_last = -1;
|
||||
const auto free_slots = get_free_slots(1, -1);
|
||||
|
||||
for (server_slot & slot : slots) {
|
||||
// skip the slot if it is not available
|
||||
if (slot.is_processing()) {
|
||||
continue;
|
||||
}
|
||||
if (!free_slots.empty()) {
|
||||
ret = free_slots[0];
|
||||
|
||||
// select the current slot if the criteria match
|
||||
if (!ret || slot.t_last_used <= t_last) {
|
||||
t_last = slot.t_last_used;
|
||||
ret = &slot;
|
||||
}
|
||||
}
|
||||
|
||||
if (ret != nullptr) {
|
||||
SLT_INF(*ret, "selected slot by LRU, t_last = %" PRId64 "\n", t_last);
|
||||
SLT_INF(*ret, "selected slot by LRU, n_tokens = %d, t_last = %" PRId64 "\n", ret->prompt.n_tokens(), ret->t_last_used);
|
||||
|
||||
update_cache = true;
|
||||
}
|
||||
@@ -2480,16 +2469,24 @@ private:
|
||||
return true;
|
||||
}
|
||||
|
||||
// idle slots that lose the least by being reused: empty slots first, then the least recently used ones
|
||||
std::vector<server_slot *> get_free_slots(size_t n_slots_needed, int exclude_id_slot) {
|
||||
std::vector<server_slot *> free_slots;
|
||||
for (auto & slot : slots) {
|
||||
if (!slot.is_processing() && slot.id != exclude_id_slot) {
|
||||
free_slots.push_back(&slot);
|
||||
}
|
||||
if (free_slots.size() >= n_slots_needed) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
std::stable_sort(free_slots.begin(), free_slots.end(), [](const server_slot * a, const server_slot * b) {
|
||||
if ((a->prompt.n_tokens() == 0) != (b->prompt.n_tokens() == 0)) {
|
||||
return a->prompt.n_tokens() == 0;
|
||||
}
|
||||
return a->t_last_used < b->t_last_used;
|
||||
});
|
||||
|
||||
free_slots.resize(std::min(free_slots.size(), n_slots_needed));
|
||||
|
||||
return free_slots;
|
||||
}
|
||||
|
||||
@@ -2519,6 +2516,11 @@ private:
|
||||
size_t idx = 0;
|
||||
GGML_ASSERT(child_slots.size() == parent_task.child_tasks.size());
|
||||
for (auto * slot : child_slots) {
|
||||
// keep the state that the child task is about to overwrite
|
||||
if (prompt_cache && slot->prompt_save(*prompt_cache)) {
|
||||
prompt_cache->update();
|
||||
}
|
||||
|
||||
int id_child = parent_task.child_tasks[idx].id;
|
||||
if (!launch_slot_with_task(*slot, std::move(parent_task.child_tasks[idx]))) {
|
||||
SRV_ERR("failed to launch slot with child task, id_task = %d\n", id_child);
|
||||
|
||||
Reference in new issue
Block a user