Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

server: Adding spec-decode counters to /metrics endpoint (#26389)

Browse Source 
* * server: add spec-decode counters to /metrics endpoint

* server: fixed review comments and now aligned param names exactly with vLLM.
This commit is contained in:
Bhavik Sharda
2026-08-05 12:36:01 +02:00
committed by GitHub
parent 020760adfc
commit a035a88878
4 changed files with 58 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
+44
View File
@@ -845,6 +845,11 @@ struct server_metrics {
uint64_t n_decode_total = 0;
uint64_t n_busy_slots_total = 0;
uint64_t n_draft_tokens_total = 0;
uint64_t n_draft_accepted_total = 0;
uint64_t n_draft_verif_steps_total = 0;
std::vector<uint64_t> n_accepted_per_pos_total;
void init() {
t_start = ggml_time_us();
}
@@ -863,6 +868,17 @@ struct server_metrics {
n_tokens_predicted += slot.n_decoded;
t_tokens_generation += slot.t_token_generation;
t_tokens_generation_total += slot.t_token_generation;
n_draft_tokens_total += slot.n_draft_total;
n_draft_accepted_total += slot.n_draft_accepted;
n_draft_verif_steps_total += slot.n_draft_verif_steps;
if (n_accepted_per_pos_total.size() < slot.n_accepted_per_pos.size()) {
n_accepted_per_pos_total.resize(slot.n_accepted_per_pos.size(), 0);
}
for (size_t i = 0; i < slot.n_accepted_per_pos.size(); i++) {
n_accepted_per_pos_total[i] += slot.n_accepted_per_pos[i];
}
}
void on_decoded(const std::vector<server_slot> & slots) {
@@ -2552,6 +2568,11 @@ private:
res->n_decode_total = metrics.n_decode_total;
res->n_busy_slots_total = metrics.n_busy_slots_total;
res->n_draft_tokens_total = metrics.n_draft_tokens_total;
res->n_draft_accepted_total = metrics.n_draft_accepted_total;
res->n_draft_verif_steps_total = metrics.n_draft_verif_steps_total;
res->n_accepted_per_pos_total = metrics.n_accepted_per_pos_total;
if (task.metrics_reset_bucket) {
metrics.reset_bucket();
}
@@ -4440,6 +4461,18 @@ void server_routes::init_routes() {
{"name", "n_tokens_max"},
{"help", "Largest observed n_tokens."},
{"value", res_task->n_tokens_max}
}, {
{"name", "spec_decode_num_draft_tokens_total"},
{"help", "Total draft tokens generated"},
{"value", res_task->n_draft_tokens_total}
}, {
{"name", "spec_decode_num_accepted_tokens_total"},
{"help", "Total draft tokens accepted by the target model"},
{"value", res_task->n_draft_accepted_total}
}, {
{"name", "spec_decode_num_drafts_total"},
{"help", "Total speculative decoding verification steps"},
{"value", res_task->n_draft_verif_steps_total}
}}},
{"gauge", {{
{"name", "prompt_tokens_seconds"},
@@ -4481,6 +4514,17 @@ void server_routes::init_routes() {
}
}
// labeled counter: one time series per draft position
if (!res_task->n_accepted_per_pos_total.empty()) {
prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos_total"
" Accepted tokens per draft position\n"
<< "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos_total counter\n";
for (size_t i = 0; i < res_task->n_accepted_per_pos_total.size(); i++) {
prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\""
<< i << "\"} " << res_task->n_accepted_per_pos_total[i] << "\n";
}
}
res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->t_start);
res->content_type = "text/plain; version=0.0.4";
res->status = 200;