dflash: copy hidden features via device copies (#31506)
This commit is contained in:
1 parent
29c720eaf7
commit
2cbca389f5
14 files changed
+441
-25
No files matched your search
+163
-2
@@ -222,6 +222,7 @@ llama_context::llama_context(
|
||||
|
||||
// +1: id n_layer() taps the output of the last layer ("input" of the head)
|
||||
cparams.embeddings_layer_inp.resize(hparams.n_layer() + 1, false);
|
||||
cparams.embeddings_layer_inp_dev = false;
|
||||
embd_layer_inp.resize(hparams.n_layer() + 1);
|
||||
|
||||
cparams.ctx_type = params.ctx_type;
|
||||
@@ -257,6 +258,9 @@ llama_context::llama_context(
|
||||
throw std::runtime_error(model.arch_name() + " requires ctx_other to be set (this warning is normal during memory fitting)");
|
||||
}
|
||||
cparams.ctx_other = params.ctx_other;
|
||||
} else if (model.arch == LLM_ARCH_DFLASH) {
|
||||
// used to read the target's layer input device buffer
|
||||
cparams.ctx_other = params.ctx_other;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1396,6 +1400,109 @@ void llama_context::set_embeddings_layer_inp(uint32_t lid, bool enable) {
|
||||
|
||||
// note: without this reserve, the draft acceptance drops to zero. not sure why - this is unexpected
|
||||
sched_need_reserve = true;
|
||||
|
||||
if (cparams.embeddings_layer_inp_dev) {
|
||||
layer_inp_dev_alloc();
|
||||
}
|
||||
}
|
||||
|
||||
void llama_context::set_embeddings_layer_inp_dev(bool value) {
|
||||
LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);
|
||||
|
||||
if (cparams.embeddings_layer_inp_dev != value) {
|
||||
sched_need_reserve = true;
|
||||
}
|
||||
|
||||
cparams.embeddings_layer_inp_dev = value;
|
||||
|
||||
layer_inp_dev_alloc();
|
||||
}
|
||||
|
||||
bool llama_context::supports_embd_row() const {
|
||||
// the DFlash/DSpark decoder graphs read the device buffer
|
||||
if (model.arch != LLM_ARCH_DFLASH) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const ggml_tensor * src = cparams.ctx_other ? cparams.ctx_other->get_layer_inp_dev() : nullptr;
|
||||
if (src == nullptr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// one of our backends must be able to read the buffer, e.g. not a draft on another GPU than the target output
|
||||
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(src->buffer);
|
||||
for (const auto & backend : backends) {
|
||||
if (ggml_backend_supports_buft(backend.get(), buft)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
ggml_backend_t llama_context::backend_from_dev(ggml_backend_dev_t dev) const {
|
||||
for (const auto & backend : backends) {
|
||||
if (ggml_backend_get_device(backend.get()) == dev) {
|
||||
return backend.get();
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void llama_context::layer_inp_dev_alloc() {
|
||||
synchronize();
|
||||
|
||||
layer_inp_dev = nullptr;
|
||||
layer_inp_dev_dev = nullptr;
|
||||
layer_inp_backend = nullptr;
|
||||
buf_layer_inp_dev.reset();
|
||||
ctx_layer_inp_dev.reset();
|
||||
|
||||
if (!cparams.embeddings_layer_inp_dev) {
|
||||
return;
|
||||
}
|
||||
|
||||
int64_t n_layer_inp = 0;
|
||||
for (bool enabled : cparams.embeddings_layer_inp) {
|
||||
n_layer_inp += enabled ? 1 : 0;
|
||||
}
|
||||
if (n_layer_inp == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
ggml_init_params params = {
|
||||
/*.mem_size =*/ ggml_tensor_overhead(),
|
||||
/*.mem_buffer =*/ nullptr,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
ctx_layer_inp_dev.reset(ggml_init(params));
|
||||
|
||||
layer_inp_dev = ggml_new_tensor_2d(ctx_layer_inp_dev.get(), GGML_TYPE_F32, n_layer_inp*model.hparams.n_embd, cparams.n_batch);
|
||||
ggml_set_name(layer_inp_dev, "layer_inp_dev");
|
||||
|
||||
layer_inp_dev_dev = model.dev_output();
|
||||
auto * buft = layer_inp_dev_dev ? ggml_backend_dev_buffer_type(layer_inp_dev_dev) : ggml_backend_cpu_buffer_type();
|
||||
|
||||
buf_layer_inp_dev.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_layer_inp_dev.get(), buft));
|
||||
if (!buf_layer_inp_dev) {
|
||||
// e.g. a large n_batch that does not fit next to the model, keep the host copies
|
||||
LLAMA_LOG_WARN("%s: failed to allocate the layer input device buffer, using host copies\n", __func__);
|
||||
cparams.embeddings_layer_inp_dev = false;
|
||||
layer_inp_dev = nullptr;
|
||||
layer_inp_dev_dev = nullptr;
|
||||
ctx_layer_inp_dev.reset();
|
||||
return;
|
||||
}
|
||||
|
||||
LLAMA_LOG_INFO("%s: %10s layer input buffer size = %8.2f MiB (%d layers)\n", __func__,
|
||||
ggml_backend_buffer_name(buf_layer_inp_dev.get()), ggml_backend_buffer_get_size(buf_layer_inp_dev.get())/1024.0/1024.0, (int) n_layer_inp);
|
||||
|
||||
layer_inp_backend = layer_inp_dev_dev ? backend_from_dev(layer_inp_dev_dev) : nullptr;
|
||||
if (layer_inp_backend && !ev_layer_inp_written) {
|
||||
ev_layer_inp_written.reset(ggml_backend_event_new(layer_inp_dev_dev));
|
||||
ev_layer_inp_read.reset(ggml_backend_event_new(layer_inp_dev_dev));
|
||||
ev_layer_inp_read_recorded = false;
|
||||
}
|
||||
}
|
||||
|
||||
void llama_context::set_nextn_layer_offset(int32_t offset) {
|
||||
@@ -1627,6 +1734,11 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (!batch_inp.tokens.empty() && batch_inp.tokens[0].embd_row >= 0) {
|
||||
LLAMA_LOG_ERROR("%s: embd row batches are not supported by encode\n", __func__);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// eagle3/DFlash: features as encoder input, and non-draft paths fall back to model's input dim
|
||||
const int64_t n_vocab = model.vocab.n_tokens();
|
||||
|
||||
@@ -1873,6 +1985,12 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
const bool has_embd_row = !batch_inp.tokens.empty() && batch_inp.tokens[0].embd_row >= 0;
|
||||
if (has_embd_row && !supports_embd_row()) {
|
||||
LLAMA_LOG_ERROR("%s: this context cannot read embd rows (needs a DFlash/DSpark decoder and a layer input device buffer in ctx_other)\n", __func__);
|
||||
return -1;
|
||||
}
|
||||
|
||||
const auto & vocab = model.vocab;
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
@@ -2016,6 +2134,24 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
|
||||
int64_t n_outputs_prev = 0;
|
||||
int64_t n_tokens_prev = 0;
|
||||
|
||||
// ordering for the layer input device buffer
|
||||
llama_context * inject_src = has_embd_row ? cparams.ctx_other : nullptr;
|
||||
ggml_backend_t inject_backend = nullptr;
|
||||
if (inject_src) {
|
||||
inject_backend = inject_src->ev_layer_inp_written ? backend_from_dev(inject_src->layer_inp_dev_dev) : nullptr;
|
||||
if (inject_backend) {
|
||||
// wait on the host: a device-side wait would stall our queue for the whole target batch,
|
||||
// which trips the Metal command buffer timeout on long prompts
|
||||
ggml_backend_event_synchronize(inject_src->ev_layer_inp_written.get());
|
||||
} else {
|
||||
inject_src->synchronize();
|
||||
}
|
||||
}
|
||||
if (layer_inp_dev && ev_layer_inp_read && ev_layer_inp_read_recorded) {
|
||||
// do not overwrite rows the reader has not consumed yet
|
||||
ggml_backend_event_wait(layer_inp_backend, ev_layer_inp_read.get());
|
||||
}
|
||||
|
||||
do {
|
||||
const auto & ubatch = mctx->get_ubatch();
|
||||
|
||||
@@ -2202,6 +2338,18 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
|
||||
n_tokens_prev += ubatch.n_tokens;
|
||||
} while (mctx->next());
|
||||
|
||||
if (layer_inp_dev && ev_layer_inp_written) {
|
||||
ggml_backend_event_record(ev_layer_inp_written.get(), layer_inp_backend);
|
||||
}
|
||||
if (inject_src) {
|
||||
if (inject_backend) {
|
||||
ggml_backend_event_record(inject_src->ev_layer_inp_read.get(), inject_backend);
|
||||
inject_src->ev_layer_inp_read_recorded = true;
|
||||
} else {
|
||||
synchronize();
|
||||
}
|
||||
}
|
||||
|
||||
// set to total number of outputs in the batch, for use in llama_get_logits_ith
|
||||
n_outputs = n_outputs_all;
|
||||
|
||||
@@ -2298,7 +2446,7 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) {
|
||||
}
|
||||
|
||||
for (bool enabled : cparams.embeddings_layer_inp) {
|
||||
if (enabled) {
|
||||
if (enabled && !cparams.embeddings_layer_inp_dev) {
|
||||
embd_layer_inp_float_count += (size_t) n_embd * n_batch;
|
||||
}
|
||||
}
|
||||
@@ -2370,7 +2518,7 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) {
|
||||
offset += embd_nextn.size * sizeof(float);
|
||||
|
||||
for (uint32_t il = 0; il < embd_layer_inp.size(); ++il) {
|
||||
if (cparams.embeddings_layer_inp[il]) {
|
||||
if (cparams.embeddings_layer_inp[il] && !cparams.embeddings_layer_inp_dev) {
|
||||
embd_layer_inp[il] = buffer_view<float>{(float *) (base + offset), (size_t) n_embd * n_batch};
|
||||
offset += embd_layer_inp[il].size * sizeof(float);
|
||||
} else {
|
||||
@@ -2426,6 +2574,9 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) {
|
||||
|
||||
bool llama_context::extract_layer_inputs(const llm_graph_result * res, size_t token_offset, size_t n_tokens) {
|
||||
bool extracted = false;
|
||||
if (cparams.embeddings_layer_inp_dev) {
|
||||
return extracted;
|
||||
}
|
||||
for (uint32_t il = 0; il < cparams.embeddings_layer_inp.size(); ++il) {
|
||||
if (!cparams.embeddings_layer_inp[il]) {
|
||||
continue;
|
||||
@@ -2742,6 +2893,8 @@ llm_graph_params llama_context::graph_params(
|
||||
/*.moe_cache =*/ moe_cache.get(),
|
||||
/*.prec_policy =*/ &model.prec_policy,
|
||||
/*.hdmd =*/ model.hdmd.rot.empty() ? nullptr : &model.hdmd,
|
||||
/*.layer_inp_dev =*/ layer_inp_dev,
|
||||
/*.layer_inp_src =*/ supports_embd_row() ? cparams.ctx_other->get_layer_inp_dev() : nullptr,
|
||||
/*.samplers =*/ sampling.samplers,
|
||||
/*.n_outputs =*/ n_outputs,
|
||||
/*.cb =*/ graph_get_cb(),
|
||||
@@ -4342,6 +4495,14 @@ void llama_set_embeddings_nextn(llama_context * ctx, bool value, bool masked) {
|
||||
ctx->set_embeddings_nextn(value, masked);
|
||||
}
|
||||
|
||||
void llama_set_embeddings_layer_inp_dev(llama_context * ctx, bool value) {
|
||||
ctx->set_embeddings_layer_inp_dev(value);
|
||||
}
|
||||
|
||||
bool llama_supports_embd_row(const llama_context * ctx) {
|
||||
return ctx->supports_embd_row();
|
||||
}
|
||||
|
||||
void llama_set_embeddings_layer_inp(llama_context * ctx, uint32_t lid, bool value) {
|
||||
ctx->set_embeddings_layer_inp(lid, value);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user