dflash: copy hidden features via device copies (#31506)

This commit is contained in:
Aman Gupta authored and GitHub committed 2026-11-03 09:02:32 +08:00
1 parent 29c720eaf7
commit 2cbca389f5
14 files changed
+441 -25

No files matched your search

+163 -2
View File
@@ -222,6 +222,7 @@ llama_context::llama_context(
// +1: id n_layer() taps the output of the last layer ("input" of the head)
cparams.embeddings_layer_inp.resize(hparams.n_layer() + 1, false);
cparams.embeddings_layer_inp_dev = false;
embd_layer_inp.resize(hparams.n_layer() + 1);
cparams.ctx_type = params.ctx_type;
@@ -257,6 +258,9 @@ llama_context::llama_context(
throw std::runtime_error(model.arch_name() + " requires ctx_other to be set (this warning is normal during memory fitting)");
}
cparams.ctx_other = params.ctx_other;
} else if (model.arch == LLM_ARCH_DFLASH) {
// used to read the target's layer input device buffer
cparams.ctx_other = params.ctx_other;
}
}
@@ -1396,6 +1400,109 @@ void llama_context::set_embeddings_layer_inp(uint32_t lid, bool enable) {
// note: without this reserve, the draft acceptance drops to zero. not sure why - this is unexpected
sched_need_reserve = true;
if (cparams.embeddings_layer_inp_dev) {
layer_inp_dev_alloc();
}
}
void llama_context::set_embeddings_layer_inp_dev(bool value) {
LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);
if (cparams.embeddings_layer_inp_dev != value) {
sched_need_reserve = true;
}
cparams.embeddings_layer_inp_dev = value;
layer_inp_dev_alloc();
}
bool llama_context::supports_embd_row() const {
// the DFlash/DSpark decoder graphs read the device buffer
if (model.arch != LLM_ARCH_DFLASH) {
return false;
}
const ggml_tensor * src = cparams.ctx_other ? cparams.ctx_other->get_layer_inp_dev() : nullptr;
if (src == nullptr) {
return false;
}
// one of our backends must be able to read the buffer, e.g. not a draft on another GPU than the target output
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(src->buffer);
for (const auto & backend : backends) {
if (ggml_backend_supports_buft(backend.get(), buft)) {
return true;
}
}
return false;
}
ggml_backend_t llama_context::backend_from_dev(ggml_backend_dev_t dev) const {
for (const auto & backend : backends) {
if (ggml_backend_get_device(backend.get()) == dev) {
return backend.get();
}
}
return nullptr;
}
void llama_context::layer_inp_dev_alloc() {
synchronize();
layer_inp_dev = nullptr;
layer_inp_dev_dev = nullptr;
layer_inp_backend = nullptr;
buf_layer_inp_dev.reset();
ctx_layer_inp_dev.reset();
if (!cparams.embeddings_layer_inp_dev) {
return;
}
int64_t n_layer_inp = 0;
for (bool enabled : cparams.embeddings_layer_inp) {
n_layer_inp += enabled ? 1 : 0;
}
if (n_layer_inp == 0) {
return;
}
ggml_init_params params = {
/*.mem_size =*/ ggml_tensor_overhead(),
/*.mem_buffer =*/ nullptr,
/*.no_alloc =*/ true,
};
ctx_layer_inp_dev.reset(ggml_init(params));
layer_inp_dev = ggml_new_tensor_2d(ctx_layer_inp_dev.get(), GGML_TYPE_F32, n_layer_inp*model.hparams.n_embd, cparams.n_batch);
ggml_set_name(layer_inp_dev, "layer_inp_dev");
layer_inp_dev_dev = model.dev_output();
auto * buft = layer_inp_dev_dev ? ggml_backend_dev_buffer_type(layer_inp_dev_dev) : ggml_backend_cpu_buffer_type();
buf_layer_inp_dev.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_layer_inp_dev.get(), buft));
if (!buf_layer_inp_dev) {
// e.g. a large n_batch that does not fit next to the model, keep the host copies
LLAMA_LOG_WARN("%s: failed to allocate the layer input device buffer, using host copies\n", __func__);
cparams.embeddings_layer_inp_dev = false;
layer_inp_dev = nullptr;
layer_inp_dev_dev = nullptr;
ctx_layer_inp_dev.reset();
return;
}
LLAMA_LOG_INFO("%s: %10s layer input buffer size = %8.2f MiB (%d layers)\n", __func__,
ggml_backend_buffer_name(buf_layer_inp_dev.get()), ggml_backend_buffer_get_size(buf_layer_inp_dev.get())/1024.0/1024.0, (int) n_layer_inp);
layer_inp_backend = layer_inp_dev_dev ? backend_from_dev(layer_inp_dev_dev) : nullptr;
if (layer_inp_backend && !ev_layer_inp_written) {
ev_layer_inp_written.reset(ggml_backend_event_new(layer_inp_dev_dev));
ev_layer_inp_read.reset(ggml_backend_event_new(layer_inp_dev_dev));
ev_layer_inp_read_recorded = false;
}
}
void llama_context::set_nextn_layer_offset(int32_t offset) {
@@ -1627,6 +1734,11 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
return -1;
}
if (!batch_inp.tokens.empty() && batch_inp.tokens[0].embd_row >= 0) {
LLAMA_LOG_ERROR("%s: embd row batches are not supported by encode\n", __func__);
return -1;
}
// eagle3/DFlash: features as encoder input, and non-draft paths fall back to model's input dim
const int64_t n_vocab = model.vocab.n_tokens();
@@ -1873,6 +1985,12 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
return -1;
}
const bool has_embd_row = !batch_inp.tokens.empty() && batch_inp.tokens[0].embd_row >= 0;
if (has_embd_row && !supports_embd_row()) {
LLAMA_LOG_ERROR("%s: this context cannot read embd rows (needs a DFlash/DSpark decoder and a layer input device buffer in ctx_other)\n", __func__);
return -1;
}
const auto & vocab = model.vocab;
const auto & hparams = model.hparams;
@@ -2016,6 +2134,24 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
int64_t n_outputs_prev = 0;
int64_t n_tokens_prev = 0;
// ordering for the layer input device buffer
llama_context * inject_src = has_embd_row ? cparams.ctx_other : nullptr;
ggml_backend_t inject_backend = nullptr;
if (inject_src) {
inject_backend = inject_src->ev_layer_inp_written ? backend_from_dev(inject_src->layer_inp_dev_dev) : nullptr;
if (inject_backend) {
// wait on the host: a device-side wait would stall our queue for the whole target batch,
// which trips the Metal command buffer timeout on long prompts
ggml_backend_event_synchronize(inject_src->ev_layer_inp_written.get());
} else {
inject_src->synchronize();
}
}
if (layer_inp_dev && ev_layer_inp_read && ev_layer_inp_read_recorded) {
// do not overwrite rows the reader has not consumed yet
ggml_backend_event_wait(layer_inp_backend, ev_layer_inp_read.get());
}
do {
const auto & ubatch = mctx->get_ubatch();
@@ -2202,6 +2338,18 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
n_tokens_prev += ubatch.n_tokens;
} while (mctx->next());
if (layer_inp_dev && ev_layer_inp_written) {
ggml_backend_event_record(ev_layer_inp_written.get(), layer_inp_backend);
}
if (inject_src) {
if (inject_backend) {
ggml_backend_event_record(inject_src->ev_layer_inp_read.get(), inject_backend);
inject_src->ev_layer_inp_read_recorded = true;
} else {
synchronize();
}
}
// set to total number of outputs in the batch, for use in llama_get_logits_ith
n_outputs = n_outputs_all;
@@ -2298,7 +2446,7 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) {
}
for (bool enabled : cparams.embeddings_layer_inp) {
if (enabled) {
if (enabled && !cparams.embeddings_layer_inp_dev) {
embd_layer_inp_float_count += (size_t) n_embd * n_batch;
}
}
@@ -2370,7 +2518,7 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) {
offset += embd_nextn.size * sizeof(float);
for (uint32_t il = 0; il < embd_layer_inp.size(); ++il) {
if (cparams.embeddings_layer_inp[il]) {
if (cparams.embeddings_layer_inp[il] && !cparams.embeddings_layer_inp_dev) {
embd_layer_inp[il] = buffer_view<float>{(float *) (base + offset), (size_t) n_embd * n_batch};
offset += embd_layer_inp[il].size * sizeof(float);
} else {
@@ -2426,6 +2574,9 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) {
bool llama_context::extract_layer_inputs(const llm_graph_result * res, size_t token_offset, size_t n_tokens) {
bool extracted = false;
if (cparams.embeddings_layer_inp_dev) {
return extracted;
}
for (uint32_t il = 0; il < cparams.embeddings_layer_inp.size(); ++il) {
if (!cparams.embeddings_layer_inp[il]) {
continue;
@@ -2742,6 +2893,8 @@ llm_graph_params llama_context::graph_params(
/*.moe_cache =*/ moe_cache.get(),
/*.prec_policy =*/ &model.prec_policy,
/*.hdmd =*/ model.hdmd.rot.empty() ? nullptr : &model.hdmd,
/*.layer_inp_dev =*/ layer_inp_dev,
/*.layer_inp_src =*/ supports_embd_row() ? cparams.ctx_other->get_layer_inp_dev() : nullptr,
/*.samplers =*/ sampling.samplers,
/*.n_outputs =*/ n_outputs,
/*.cb =*/ graph_get_cb(),
@@ -4342,6 +4495,14 @@ void llama_set_embeddings_nextn(llama_context * ctx, bool value, bool masked) {
ctx->set_embeddings_nextn(value, masked);
}
void llama_set_embeddings_layer_inp_dev(llama_context * ctx, bool value) {
ctx->set_embeddings_layer_inp_dev(value);
}
bool llama_supports_embd_row(const llama_context * ctx) {
return ctx->supports_embd_row();
}
void llama_set_embeddings_layer_inp(llama_context * ctx, uint32_t lid, bool value) {
ctx->set_embeddings_layer_inp(lid, value);
}