Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

server: refactor sleep handling, allow access /metrics during sleep (#27376)

Browse Source 
* add cached responses

* refactor on_sleeping_state

* allow accessing metrics during sleep

* metrics task should not reset timer

* updated docs

* fix

* fix get_res_model_info

* add test

* fix a race condition

* split metrics and slots tasks / results

* should_reset_buckets
This commit is contained in:
Xuan-Son Nguyen
2026-08-19 20:48:09 +02:00
committed by GitHub
parent ee0ea03adf
commit 947fd9bb2b
9 changed files with 422 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-task.h
+15 -9
View File
@@ -22,6 +22,7 @@ enum server_task_type {
SERVER_TASK_TYPE_CONTROL,
SERVER_TASK_TYPE_NEXT_RESPONSE,
SERVER_TASK_TYPE_METRICS,
SERVER_TASK_TYPE_SLOT_GET,
SERVER_TASK_TYPE_SLOT_SAVE,
SERVER_TASK_TYPE_SLOT_RESTORE,
SERVER_TASK_TYPE_SLOT_ERASE,
@@ -489,22 +490,16 @@ struct server_task_result_error : server_task_result {
virtual json to_json() override;
};
// used by /metrics API
struct server_task_result_metrics : server_task_result {
// these are immediate stats, not accumulated (server_metrics is cumulative)
int n_idle_slots;
int n_processing_slots;
int n_tasks_deferred;
int n_processing_slots = 0;
int n_tasks_deferred = 0;
server_metrics metrics;
// while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
// therefore, we use json to temporarily store the slot.to_json() result
json slots_data = json::array();
// used by /slots API
virtual json to_json() override;
// used by /metrics API
struct metric_item {
std::string name;
std::string description;
@@ -513,6 +508,17 @@ struct server_task_result_metrics : server_task_result {
std::string to_metrics();
};
// used by /slots API
struct server_task_result_slots : server_task_result {
int n_idle_slots = 0;
// while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
// therefore, we use json to temporarily store the slot.to_json() result
json slots_data = json::array();
virtual json to_json() override;
};
struct server_task_result_slot_save_load : server_task_result {
std::string filename;
bool is_save; // true = save, false = load