Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

server: refactor sleep handling, allow access /metrics during sleep (#27376)

Browse Source 
* add cached responses

* refactor on_sleeping_state

* allow accessing metrics during sleep

* metrics task should not reset timer

* updated docs

* fix

* fix get_res_model_info

* add test

* fix a race condition

* split metrics and slots tasks / results

* should_reset_buckets
This commit is contained in:
Xuan-Son Nguyen
2026-08-19 20:48:09 +02:00
committed by GitHub
parent ee0ea03adf
commit 947fd9bb2b
9 changed files with 422 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-queue.h
+7 -12
View File
@@ -44,7 +44,7 @@ private:
// callback functions
std::function<bool(server_task &&, bool)> callback_new_task;
std::function<void(void)> callback_update_slots;
std::function<void(bool)> callback_sleeping_state;
std::vector<std::function<void(bool)>> callback_sleeping_state;
public:
~server_queue() { worker_stop(); }
@@ -86,6 +86,7 @@ public:
*
* Sleeping procedure (disabled if idle_sleep_ms < 0):
* - If there is no task after idle_sleep_ms, enter sleeping state
* note: metrics tasks are processed as usual, but do not reset the idle timer
* - Call callback_sleeping_state(true)
* - Wait until req_stop_sleeping is set to true
* - Call callback_sleeping_state(false)
@@ -127,18 +128,12 @@ public:
}
// Register callback for sleeping state change; multiple callbacks are allowed
// note: when entering sleeping state, the callback is called AFTER sleeping is set to true
// when leaving sleeping state, the callback is called BEFORE sleeping is set to false
// for example: register order cb0, cb1, cb2
// entering sleep: queue.sleeping = true --> cb0(true) --> cb1(true) --> cb2(true)
// leaving sleep: cb2(false) --> cb1(false) --> cb0(false) --> queue.sleeping = false
// note: caller will hold mutex_tasks while calling the callbacks
void on_sleeping_state(std::function<void(bool)> callback) {
if (callback_sleeping_state) {
auto prev_callback = std::move(callback_sleeping_state);
callback_sleeping_state = [prev_callback, callback](bool sleeping) {
prev_callback(sleeping);
callback(sleeping);
};
} else {
callback_sleeping_state = std::move(callback);
}
callback_sleeping_state.push_back(std::move(callback));
}
private: