spec : add probabilistic sampling for simple draft and MTP (#27694)

* Make the drafter probabilistic and the target verify by rejection sampling

* Drop stale spec_draft_q before drafting

* Fallback to argmax sampling for grammar-constrained requests and adding flag for enabling probabilistic draft sampling. Default flag value is greedy.

* Support grammar-constrained requests in rejection sampling

* Fix - renormalize distribution after masking

* copy rng on sampler copy and re-accept drafted tokens on replay

* Fix draft sampler sharing the target's rng stream

* Simplify the rejection sampler's inputs and move replay to the server

* Truncate the draft candidates along with the draft

---------

Co-authored-by: praneshgo <227579474+praneshgo@users.noreply.github.com>
Co-authored-by: Pranesh Gonegandla <pgonegandla@nvidia.com>
This commit is contained in:
authored and GitHub committed 2026-10-02 17:52:54 +03:00
1 parent 134b2bb756
commit 1fb7ef3e33
7 files changed
+304 -11

No files matched your search

+57 -3
View File
@@ -53,6 +53,31 @@ static common_speculative_output_limits server_output_limits(const common_params
return result;
}
// a checkpoint restore dropped tokens the target had accepted - re-accept them rather than verify again
static std::vector<llama_token> server_accept_replay(
common_sampler * smpl,
llama_context * ctx,
const std::vector<int32_t> & idxs,
const llama_tokens & draft) {
GGML_ASSERT(idxs.size() == draft.size() + 1);
std::vector<llama_token> result;
result.reserve(idxs.size());
for (size_t i = 0; i < draft.size(); ++i) {
// the token is discarded - the call is what advances the sampler over this position
common_sampler_sample(smpl, ctx, idxs[i]);
common_sampler_accept(smpl, draft[i], true);
result.push_back(draft[i]);
}
const llama_token id = common_sampler_sample(smpl, ctx, idxs[draft.size()]);
common_sampler_accept(smpl, id, true);
result.push_back(id);
return result;
}
// synthetic draft verification for benchmarking - accept draft tokens at random instead of by match with the target
// on replay the draft was already accepted before a context checkpoint restore, so repeat the same decisions
static std::vector<llama_token> server_sample_and_accept_synth(
@@ -212,6 +237,9 @@ struct server_slot {
common_speculative * spec;
llama_tokens spec_draft;
// draft candidates per token in spec_draft; only draft-simple and draft-mtp fill it
std::vector<std::vector<llama_token_data>> spec_draft_q;
llama_tokens spec_prompt;
std::vector<int32_t> spec_i_batch;
common_prompt_checkpoint spec_ckpt;
@@ -446,6 +474,11 @@ struct server_slot {
return !!spec;
}
// at temp 0 both p and q are point masses, so rejection is the same as sample-and-match
bool use_spec_rejection() const {
return task && task->params.sampling.temp > 0.0f;
}
void add_token(const completion_token_output & token) {
if (!is_processing()) {
SLT_WRN(*this, "%s", "slot is not processing\n");
@@ -3088,6 +3121,9 @@ private:
if (n_draft_max > 0) {
GGML_ASSERT(slot.can_speculate());
// stale candidates: a replay never reads them, a new draft refills them
slot.spec_draft_q.clear();
if (!slot.spec_draft.empty()) {
// we have a previous (partial) draft to reuse
if (use_ckpt_tgt) {
@@ -3107,6 +3143,8 @@ private:
slot.spec_prompt = slot.prompt.tokens.get_text_tokens();
const bool spec_reject = slot.use_spec_rejection();
common_speculative_get_draft_params(spec.get(), slot.id) = {
/* .drafting = */ true,
/* .n_max = */ n_draft_max,
@@ -3114,6 +3152,9 @@ private:
/* .id_last = */ slot.sampled,
/* .prompt = */ &slot.spec_prompt,
/* .result = */ &slot.spec_draft,
/* .result_q = */ spec_reject ? &slot.spec_draft_q : nullptr,
/* .temp = */ slot.task->params.sampling.temp,
/* .seed = */ slot.task->params.sampling.seed,
};
drafting.push_back(&slot);
@@ -4051,12 +4092,25 @@ private:
common_sampler_ptr smpl_save(common_sampler_clone(slot.smpl.get()));
GGML_ASSERT(slot.spec_i_batch.size() == n_draft + 1);
GGML_ASSERT(slot.spec_draft_q.empty() || (slot.spec_draft_q.size() == slot.spec_draft.size()));
const auto & synth_probs = common_speculative_get_synth_probs(spec.get());
auto accepted = synth_probs.empty()
? common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft)
: server_sample_and_accept_synth(
// drafters that fill no distribution fall back here
const bool use_rejection = slot.use_spec_rejection() && !slot.spec_draft_q.empty();
std::vector<llama_token> accepted;
if (!synth_probs.empty()) {
// synthetic acceptance replaces verification entirely, so it comes first
accepted = server_sample_and_accept_synth(
slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft,
synth_probs, slot.spec_synth_rng, slot.spec_is_replay);
} else if (slot.spec_is_replay && slot.use_spec_rejection()) {
accepted = server_accept_replay(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft);
} else if (use_rejection) {
accepted = common_sampler_sample_and_accept_n_rejection(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft, slot.spec_draft_q);
} else {
accepted = common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft);
}
slot.spec_i_batch.clear();
GGML_ASSERT(accepted.size() >= 1);