dflash: copy hidden features via device copies (#31506)

This commit is contained in:
Aman Gupta authored and GitHub committed 2026-11-03 09:02:32 +08:00
1 parent 29c720eaf7
commit 2cbca389f5
14 files changed
+441 -25

No files matched your search

+39
View File
@@ -1027,6 +1027,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices
uint32_t target_layer_ids_n = 0;
// inject target features from the target's device buffer instead of host copies (env LLAMA_DFLASH_DEV=0 disables)
bool use_dev_features = false;
common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq,
common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)
: common_speculative_impl(type, n_seq, params.draft.n_max)
@@ -1141,6 +1144,23 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true);
}
// the device buffer keeps layers in ascending id order, the fc input expects target_layer_ids order
{
const char * env = getenv("LLAMA_DFLASH_DEV");
const bool ascending = std::adjacent_find(target_layer_ids, target_layer_ids + target_layer_ids_n,
[](int32_t a, int32_t b) { return a >= b; }) == target_layer_ids + target_layer_ids_n;
use_dev_features = !is_lilicorr && ascending && (env == nullptr || atoi(env) != 0);
if (use_dev_features) {
llama_set_embeddings_layer_inp_dev(ctx_tgt, true);
use_dev_features = llama_supports_embd_row(ctx_dft);
if (!use_dev_features) {
llama_set_embeddings_layer_inp_dev(ctx_tgt, false);
}
}
LOG_INF("%s: DFlash target features via %s\n", __func__, use_dev_features ? "device buffer" : "host copies");
}
// DFlash2 reads its selector lattice from h_nextn and never consumes raw logits.
llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ !is_dflash2 && !is_lilicorr);
llama_set_causal_attn(ctx_dft, causal_attn); // DFlash needs non-causal attention unless the model says otherwise
@@ -1233,6 +1253,25 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
for (int32_t offset = 0; offset < n_rows; offset += n_ubatch) {
const int32_t n_chunk = std::min(n_ubatch, n_rows - offset);
if (use_dev_features) {
// the features are the target batch rows in the target's device buffer
batch_inject.clear();
for (int32_t i = 0; i < n_chunk; ++i) {
const int32_t row = i_batch_beg[seq_id] + offset + i;
const llama_pos p = batch_in.tokens[row].pos[0];
const llama_pos pos_arr[4] = { p, p, p, 0 };
batch_inject.add_embd_row(row, pos_arr, seq_id, false);
}
const int32_t rc = llama_process(ctx_dft, LLAMA_PROCESS_TYPE_DECODE, batch_inject.get());
if (rc != 0) {
LOG_ERR("%s: llama_process(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n",
__func__, rc, (int) n_chunk, (int) offset);
return false;
}
continue;
}
// gather target features per extract layer; the fused decode encodes and
// injects them into the K/V cache at the target positions
features_buf.resize((size_t) n_chunk * n_embd_enc);