server: Support logprobs with speculative decoding (#27196)

* return logprobs in spec path

* server: add regression test for logprobs with speculative decoding

Asserts accepted draft tokens report the target model's logprobs,
comparing a draft-enabled run against a no-draft baseline (#24271).
Fails on master (top_logprobs empty, logprob 0.0), passes with the
fix from #27196.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KARBWpPcijcBXzJEvttuwk

---------

Co-authored-by: Jeonghyun Lee <sig.jhl@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-10-20 19:36:14 +02:00
1 parent b684b498b6
commit ead9f32058
2 files changed
+48 -7

No files matched your search

+12 -7
View File
@@ -4522,11 +4522,14 @@ private:
GGML_ASSERT(n_draft > 0);
// batch indices of the draft tokens, used to get the token probabilities below
auto spec_i_batch = std::move(slot.spec_i_batch);
// verify and try to accept the draft
{
common_sampler_ptr smpl_save(common_sampler_clone(slot.smpl.get()));
GGML_ASSERT(slot.spec_i_batch.size() == n_draft + 1);
GGML_ASSERT(spec_i_batch.size() == n_draft + 1);
GGML_ASSERT(slot.spec_draft_q.empty() || (slot.spec_draft_q.size() == slot.spec_draft.size()));
const auto & synth_probs = common_speculative_get_synth_probs(spec.get());
@@ -4537,16 +4540,15 @@ private:
if (!synth_probs.empty()) {
// synthetic acceptance replaces verification entirely, so it comes first
accepted = server_sample_and_accept_synth(
slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft,
slot.smpl.get(), slot.ctx_tgt, spec_i_batch, slot.spec_draft,
synth_probs, slot.spec_synth_rng, slot.spec_is_replay);
} else if (slot.spec_is_replay && slot.use_spec_rejection()) {
accepted = server_accept_replay(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft);
accepted = server_accept_replay(slot.smpl.get(), slot.ctx_tgt, spec_i_batch, slot.spec_draft);
} else if (use_rejection) {
accepted = common_sampler_sample_and_accept_n_rejection(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft, slot.spec_draft_q);
accepted = common_sampler_sample_and_accept_n_rejection(slot.smpl.get(), slot.ctx_tgt, spec_i_batch, slot.spec_draft, slot.spec_draft_q);
} else {
accepted = common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft);
accepted = common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, spec_i_batch, slot.spec_draft);
}
slot.spec_i_batch.clear();
GGML_ASSERT(accepted.size() >= 1);
@@ -4633,7 +4635,10 @@ private:
result.text_to_send = common_token_to_piece(slot.ctx_tgt, result.tok, accept_special_token(slot, result.tok));
result.prob = 1.0f; // set later
// TODO: set result.probs
// post_sampling_probs is not supported with speculative decoding
if (slot.task->params.sampling.n_probs > 0 && !slot.task->params.post_sampling_probs) {
populate_token_probs(slot, result, false, params_base.special, spec_i_batch[i]);
}
slot.stats.n_gen += 1;