dflash: copy hidden features via device copies (#31506)
This commit is contained in:
1 parent
29c720eaf7
commit
2cbca389f5
14 files changed
+441
-25
No files matched your search
@@ -1027,6 +1027,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
||||
const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices
|
||||
uint32_t target_layer_ids_n = 0;
|
||||
|
||||
// inject target features from the target's device buffer instead of host copies (env LLAMA_DFLASH_DEV=0 disables)
|
||||
bool use_dev_features = false;
|
||||
|
||||
common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq,
|
||||
common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)
|
||||
: common_speculative_impl(type, n_seq, params.draft.n_max)
|
||||
@@ -1141,6 +1144,23 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
||||
llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true);
|
||||
}
|
||||
|
||||
// the device buffer keeps layers in ascending id order, the fc input expects target_layer_ids order
|
||||
{
|
||||
const char * env = getenv("LLAMA_DFLASH_DEV");
|
||||
const bool ascending = std::adjacent_find(target_layer_ids, target_layer_ids + target_layer_ids_n,
|
||||
[](int32_t a, int32_t b) { return a >= b; }) == target_layer_ids + target_layer_ids_n;
|
||||
|
||||
use_dev_features = !is_lilicorr && ascending && (env == nullptr || atoi(env) != 0);
|
||||
if (use_dev_features) {
|
||||
llama_set_embeddings_layer_inp_dev(ctx_tgt, true);
|
||||
use_dev_features = llama_supports_embd_row(ctx_dft);
|
||||
if (!use_dev_features) {
|
||||
llama_set_embeddings_layer_inp_dev(ctx_tgt, false);
|
||||
}
|
||||
}
|
||||
LOG_INF("%s: DFlash target features via %s\n", __func__, use_dev_features ? "device buffer" : "host copies");
|
||||
}
|
||||
|
||||
// DFlash2 reads its selector lattice from h_nextn and never consumes raw logits.
|
||||
llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ !is_dflash2 && !is_lilicorr);
|
||||
llama_set_causal_attn(ctx_dft, causal_attn); // DFlash needs non-causal attention unless the model says otherwise
|
||||
@@ -1233,6 +1253,25 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
||||
for (int32_t offset = 0; offset < n_rows; offset += n_ubatch) {
|
||||
const int32_t n_chunk = std::min(n_ubatch, n_rows - offset);
|
||||
|
||||
if (use_dev_features) {
|
||||
// the features are the target batch rows in the target's device buffer
|
||||
batch_inject.clear();
|
||||
for (int32_t i = 0; i < n_chunk; ++i) {
|
||||
const int32_t row = i_batch_beg[seq_id] + offset + i;
|
||||
const llama_pos p = batch_in.tokens[row].pos[0];
|
||||
const llama_pos pos_arr[4] = { p, p, p, 0 };
|
||||
batch_inject.add_embd_row(row, pos_arr, seq_id, false);
|
||||
}
|
||||
|
||||
const int32_t rc = llama_process(ctx_dft, LLAMA_PROCESS_TYPE_DECODE, batch_inject.get());
|
||||
if (rc != 0) {
|
||||
LOG_ERR("%s: llama_process(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n",
|
||||
__func__, rc, (int) n_chunk, (int) offset);
|
||||
return false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// gather target features per extract layer; the fused decode encodes and
|
||||
// injects them into the K/V cache at the target positions
|
||||
features_buf.resize((size_t) n_chunk * n_embd_enc);
|
||||
|
||||
Reference in new issue
Block a user