dflash: copy hidden features via device copies (#31506)

This commit is contained in:
Aman Gupta authored and GitHub committed 2026-11-03 09:02:32 +08:00
1 parent 29c720eaf7
commit 2cbca389f5
14 files changed
+441 -25

No files matched your search

+56 -1
View File
@@ -138,6 +138,12 @@ void llm_graph_input_embd::set_input(const llama_ubatch * ubatch) {
ggml_backend_tensor_set(tokens, ubatch->token, 0, n_tokens*ggml_element_size(tokens));
}
if (ubatch->embd_row && embd_rows) {
const int64_t n_tokens = ubatch->n_tokens;
ggml_backend_tensor_set(embd_rows, ubatch->embd_row, 0, n_tokens*ggml_element_size(embd_rows));
}
if (ubatch->embd && embd && !ubatch->is_mixed()) {
GGML_ASSERT(n_embd == embd->ne[0]);
@@ -197,6 +203,7 @@ bool llm_graph_input_embd::can_reuse(const llm_graph_params & params) {
res &= (!mixed_tokens) || mixed_tokens->ne[0] == llm_graph_n_tok_rows(params.ubatch);
res &= (!mixed_embd) || mixed_embd->ne[1] == params.ubatch.n_tokens;
res &= (!scale_rows) || scale_rows->ne[1] == params.ubatch.n_tokens;
res &= (!params.ubatch.embd_row) || (embd_rows && embd_rows->ne[0] == params.ubatch.n_tokens);
return res;
}
@@ -232,6 +239,22 @@ bool llm_graph_input_embd_h::can_reuse(const llm_graph_params & params) {
return res;
}
void llm_graph_input_batch_idxs::set_input(const llama_ubatch * ubatch) {
const int64_t n_tokens = ubatch->n_tokens;
GGML_ASSERT(ubatch->data && ubatch->data->batch_idxs.size() == (size_t) n_tokens);
GGML_ASSERT(ggml_backend_buffer_is_host(idxs->buffer));
int64_t * data = (int64_t *) idxs->data;
for (int64_t i = 0; i < n_tokens; ++i) {
data[i] = ubatch->data->batch_idxs[i];
}
}
bool llm_graph_input_batch_idxs::can_reuse(const llm_graph_params & params) {
return idxs->ne[0] == params.ubatch.n_tokens;
}
void llm_graph_input_pos::set_input(const llama_ubatch * ubatch) {
if (ubatch->pos && pos) {
const int64_t n_tokens = ubatch->n_tokens;
@@ -1436,7 +1459,7 @@ void llm_graph_result::set_outputs(const llm_graph_params & params) {
}
{
const auto & embeddings_layer_inp = params.cparams.embeddings_layer_inp;
for (size_t il = 0; il < embeddings_layer_inp.size(); ++il) {
for (size_t il = 0; il < embeddings_layer_inp.size() && !params.cparams.embeddings_layer_inp_dev; ++il) {
if (embeddings_layer_inp[il]) {
GGML_ASSERT(t_layer_inp[il] != nullptr && "layer input tensor is null");
ggml_set_output(t_layer_inp[il]);
@@ -1563,6 +1586,8 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) :
moe_cache (params.moe_cache),
prec_policy (params.prec_policy),
hdmd (params.hdmd),
layer_inp_dev (params.layer_inp_dev),
layer_inp_src (params.layer_inp_src),
samplers (params.samplers),
cb_func (params.cb),
res (params.res),
@@ -4223,6 +4248,36 @@ void llm_graph_context::build_logits() const {
}
}
void llm_graph_context::build_layer_inp_dev() const {
if (!cparams.embeddings_layer_inp_dev || layer_inp_dev == nullptr) {
return;
}
auto inp = std::make_unique<llm_graph_input_batch_idxs>();
inp->idxs = ggml_new_tensor_1d(ctx0, GGML_TYPE_I64, n_tokens);
ggml_set_input(inp->idxs);
// row layout: [layer_0 | layer_1 | ...] in ascending layer id order
int64_t col = 0;
for (size_t il = 0; il < cparams.embeddings_layer_inp.size(); ++il) {
if (!cparams.embeddings_layer_inp[il]) {
continue;
}
ggml_tensor * src = res->t_layer_inp[il];
GGML_ASSERT(src != nullptr && "layer input tensor is null");
GGML_ASSERT(src->ne[0] == n_embd);
ggml_tensor * dst = ggml_view_2d(ctx0, layer_inp_dev, n_embd, layer_inp_dev->ne[1], layer_inp_dev->nb[1], col*n_embd*ggml_element_size(layer_inp_dev));
ggml_build_forward_expand(gf, ggml_set_rows(ctx0, dst, ggml_reshape_2d(ctx0, src, n_embd, n_tokens), inp->idxs));
col++;
}
GGML_ASSERT(col*n_embd == layer_inp_dev->ne[0]);
res->add_input(std::move(inp));
}
void llm_graph_context::build_sampling() const {
if (samplers.empty() || !res->t_logits) {
return;