spec : add probabilistic sampling for simple draft and MTP (#27694)
* Make the drafter probabilistic and the target verify by rejection sampling * Drop stale spec_draft_q before drafting * Fallback to argmax sampling for grammar-constrained requests and adding flag for enabling probabilistic draft sampling. Default flag value is greedy. * Support grammar-constrained requests in rejection sampling * Fix - renormalize distribution after masking * copy rng on sampler copy and re-accept drafted tokens on replay * Fix draft sampler sharing the target's rng stream * Simplify the rejection sampler's inputs and move replay to the server * Truncate the draft candidates along with the draft --------- Co-authored-by: praneshgo <227579474+praneshgo@users.noreply.github.com> Co-authored-by: Pranesh Gonegandla <pgonegandla@nvidia.com>
This commit is contained in:
7 files changed
+304
-11
No files matched your search
@@ -53,6 +53,31 @@ static common_speculative_output_limits server_output_limits(const common_params
|
||||
return result;
|
||||
}
|
||||
|
||||
// a checkpoint restore dropped tokens the target had accepted - re-accept them rather than verify again
|
||||
static std::vector<llama_token> server_accept_replay(
|
||||
common_sampler * smpl,
|
||||
llama_context * ctx,
|
||||
const std::vector<int32_t> & idxs,
|
||||
const llama_tokens & draft) {
|
||||
GGML_ASSERT(idxs.size() == draft.size() + 1);
|
||||
|
||||
std::vector<llama_token> result;
|
||||
result.reserve(idxs.size());
|
||||
|
||||
for (size_t i = 0; i < draft.size(); ++i) {
|
||||
// the token is discarded - the call is what advances the sampler over this position
|
||||
common_sampler_sample(smpl, ctx, idxs[i]);
|
||||
common_sampler_accept(smpl, draft[i], true);
|
||||
result.push_back(draft[i]);
|
||||
}
|
||||
|
||||
const llama_token id = common_sampler_sample(smpl, ctx, idxs[draft.size()]);
|
||||
common_sampler_accept(smpl, id, true);
|
||||
result.push_back(id);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// synthetic draft verification for benchmarking - accept draft tokens at random instead of by match with the target
|
||||
// on replay the draft was already accepted before a context checkpoint restore, so repeat the same decisions
|
||||
static std::vector<llama_token> server_sample_and_accept_synth(
|
||||
@@ -212,6 +237,9 @@ struct server_slot {
|
||||
common_speculative * spec;
|
||||
|
||||
llama_tokens spec_draft;
|
||||
|
||||
// draft candidates per token in spec_draft; only draft-simple and draft-mtp fill it
|
||||
std::vector<std::vector<llama_token_data>> spec_draft_q;
|
||||
llama_tokens spec_prompt;
|
||||
std::vector<int32_t> spec_i_batch;
|
||||
common_prompt_checkpoint spec_ckpt;
|
||||
@@ -446,6 +474,11 @@ struct server_slot {
|
||||
return !!spec;
|
||||
}
|
||||
|
||||
// at temp 0 both p and q are point masses, so rejection is the same as sample-and-match
|
||||
bool use_spec_rejection() const {
|
||||
return task && task->params.sampling.temp > 0.0f;
|
||||
}
|
||||
|
||||
void add_token(const completion_token_output & token) {
|
||||
if (!is_processing()) {
|
||||
SLT_WRN(*this, "%s", "slot is not processing\n");
|
||||
@@ -3088,6 +3121,9 @@ private:
|
||||
if (n_draft_max > 0) {
|
||||
GGML_ASSERT(slot.can_speculate());
|
||||
|
||||
// stale candidates: a replay never reads them, a new draft refills them
|
||||
slot.spec_draft_q.clear();
|
||||
|
||||
if (!slot.spec_draft.empty()) {
|
||||
// we have a previous (partial) draft to reuse
|
||||
if (use_ckpt_tgt) {
|
||||
@@ -3107,6 +3143,8 @@ private:
|
||||
|
||||
slot.spec_prompt = slot.prompt.tokens.get_text_tokens();
|
||||
|
||||
const bool spec_reject = slot.use_spec_rejection();
|
||||
|
||||
common_speculative_get_draft_params(spec.get(), slot.id) = {
|
||||
/* .drafting = */ true,
|
||||
/* .n_max = */ n_draft_max,
|
||||
@@ -3114,6 +3152,9 @@ private:
|
||||
/* .id_last = */ slot.sampled,
|
||||
/* .prompt = */ &slot.spec_prompt,
|
||||
/* .result = */ &slot.spec_draft,
|
||||
/* .result_q = */ spec_reject ? &slot.spec_draft_q : nullptr,
|
||||
/* .temp = */ slot.task->params.sampling.temp,
|
||||
/* .seed = */ slot.task->params.sampling.seed,
|
||||
};
|
||||
|
||||
drafting.push_back(&slot);
|
||||
@@ -4051,12 +4092,25 @@ private:
|
||||
common_sampler_ptr smpl_save(common_sampler_clone(slot.smpl.get()));
|
||||
|
||||
GGML_ASSERT(slot.spec_i_batch.size() == n_draft + 1);
|
||||
GGML_ASSERT(slot.spec_draft_q.empty() || (slot.spec_draft_q.size() == slot.spec_draft.size()));
|
||||
const auto & synth_probs = common_speculative_get_synth_probs(spec.get());
|
||||
auto accepted = synth_probs.empty()
|
||||
? common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft)
|
||||
: server_sample_and_accept_synth(
|
||||
|
||||
// drafters that fill no distribution fall back here
|
||||
const bool use_rejection = slot.use_spec_rejection() && !slot.spec_draft_q.empty();
|
||||
|
||||
std::vector<llama_token> accepted;
|
||||
if (!synth_probs.empty()) {
|
||||
// synthetic acceptance replaces verification entirely, so it comes first
|
||||
accepted = server_sample_and_accept_synth(
|
||||
slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft,
|
||||
synth_probs, slot.spec_synth_rng, slot.spec_is_replay);
|
||||
} else if (slot.spec_is_replay && slot.use_spec_rejection()) {
|
||||
accepted = server_accept_replay(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft);
|
||||
} else if (use_rejection) {
|
||||
accepted = common_sampler_sample_and_accept_n_rejection(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft, slot.spec_draft_q);
|
||||
} else {
|
||||
accepted = common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft);
|
||||
}
|
||||
slot.spec_i_batch.clear();
|
||||
|
||||
GGML_ASSERT(accepted.size() >= 1);
|
||||
|
||||
Reference in new issue
Block a user