Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

server: refactor + correctness fixes for metrics (#26920)

Browse Source 
* server: refactor metrics

* move most fields to server_slot_stats

* cont

* rm result_timings

* tie stats to batch

* cont

* nits: move place in code

* exclude first generated token

* more accurate batch metrics tracking

* n_predict --> n_gen

* metrics_on_prediction

* metrics_flush_idle

* metrics: seperate cache/processed prompt tokens

* refactor server_task_result_metrics

* add test

* nits

* fix flush before reset()

* cont

* rm dead code

* nits
This commit is contained in:
Xuan-Son Nguyen
2026-08-13 10:02:01 +02:00
committed by GitHub
parent e79e4bf660
commit decaf508bb
6 changed files with 813 additions and 475 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-task.h
+13 -44
View File
@@ -259,26 +259,6 @@ struct server_task {
}
};
struct result_timings {
int32_t cache_n = -1;
int32_t prompt_n = -1;
double prompt_ms = 0.0;
double prompt_per_token_ms = 0.0;
double prompt_per_second = 0.0;
int32_t predicted_n = -1;
double predicted_ms = 0.0;
double predicted_per_token_ms = 0.0;
double predicted_per_second = 0.0;
// Optional speculative metrics - only included when > 0
int32_t draft_n = 0;
int32_t draft_n_accepted = 0;
json to_json() const;
};
struct result_prompt_progress {
int32_t total = 0;
int32_t cache = 0;
@@ -343,7 +323,7 @@ struct server_task_result_cmpl_final : server_task_result {
bool stream;
bool include_usage;
result_timings timings;
server_slot_stats stats;
std::string prompt;
bool truncated;
@@ -425,7 +405,7 @@ struct server_task_result_cmpl_partial : server_task_result {
bool is_begin = false; // whether to send 200 status to HTTP client (begin of SSE stream)
// ref: https://github.com/ggml-org/llama.cpp/pull/23884
completion_token_output prob_output;
result_timings timings;
server_slot_stats stats;
result_prompt_progress progress;
// response formatting
@@ -510,38 +490,27 @@ struct server_task_result_error : server_task_result {
};
struct server_task_result_metrics : server_task_result {
// these are immediate stats, not accumulated (server_metrics is cumulative)
int n_idle_slots;
int n_processing_slots;
int n_tasks_deferred;
int64_t t_start;
// TODO: somehow reuse server_metrics in the future, instead of duplicating the fields
uint64_t n_prompt_tokens_processed_total = 0;
uint64_t t_prompt_processing_total = 0;
uint64_t n_tokens_predicted_total = 0;
uint64_t t_tokens_generation_total = 0;
uint64_t n_tokens_max = 0;
uint64_t n_prompt_tokens_processed = 0;
uint64_t t_prompt_processing = 0;
uint64_t n_tokens_predicted = 0;
uint64_t t_tokens_generation = 0;
uint64_t n_decode_total = 0;
uint64_t n_busy_slots_total = 0;
uint64_t n_draft_tokens_total = 0;
uint64_t n_draft_accepted_total = 0;
uint64_t n_draft_verif_steps_total = 0;
std::vector<uint64_t> n_accepted_per_pos_total;
server_metrics metrics;
// while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
// therefore, we use json to temporarily store the slot.to_json() result
json slots_data = json::array();
// used by /slots API
virtual json to_json() override;
// used by /metrics API
struct metric_item {
std::string name;
std::string description;
double value; // prometheus values are always float64
};
std::string to_metrics();
};
struct server_task_result_slot_save_load : server_task_result {