llama : move no_alloc out of llama_hparams (#31141)
* llama : move no_alloc out of llama_hparams Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (2x DGX Spark (RDMA, TP=2)) * cont : clean-up * cont : clean-up Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (2x DGX Spark (RDMA, TP=2)) * tests : add a no_alloc case to test-llama-context Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (2x DGX Spark (RDMA, TP=2))
This commit is contained in:
1 parent
4b851fbee8
commit
a4699400e3
30 files changed
+266
-179
No files matched your search
@@ -489,6 +489,7 @@ llama_context::llama_context(
|
||||
/*.swa_full =*/ params.swa_full,
|
||||
/*.ctx_type =*/ cparams.ctx_type,
|
||||
/*.mem_other =*/ llama_get_memory(cparams.ctx_other),
|
||||
/*.no_alloc =*/ model.no_alloc(),
|
||||
};
|
||||
|
||||
memory.reset(model.create_memory(params_mem, cparams));
|
||||
@@ -559,7 +560,7 @@ llama_context::llama_context(
|
||||
}
|
||||
|
||||
if (cparams.moe_cache_size > 0) {
|
||||
moe_cache = std::make_unique<llama_moe_cache>(model, backend_ptrs, backend_buft, cparams.moe_cache_size);
|
||||
moe_cache = std::make_unique<llama_moe_cache>(model, model.no_alloc(), backend_ptrs, backend_buft, cparams.moe_cache_size);
|
||||
}
|
||||
|
||||
sched_reserve();
|
||||
@@ -587,7 +588,7 @@ llama_context::~llama_context() {
|
||||
synchronize();
|
||||
|
||||
// when training, ggml_opt allocates extra buffers through the scheduler, so the sizes no longer match the expectation
|
||||
if (!model.hparams.no_alloc && !opt_ctx) {
|
||||
if (!model.no_alloc() && !opt_ctx) {
|
||||
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
|
||||
ggml_backend_t backend = backend_ptrs[i];
|
||||
ggml_backend_buffer_type_t buft = backend_buft[i];
|
||||
@@ -788,7 +789,7 @@ void llama_context::sched_reserve() {
|
||||
// reserve pp (prompt processing) graph first so that buffers are only allocated once
|
||||
{
|
||||
auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(),
|
||||
model.hparams.no_alloc, model.hparams.no_alloc ? backend_buf_exp_size.data() : nullptr);
|
||||
model.no_alloc(), model.no_alloc() ? backend_buf_exp_size.data() : nullptr);
|
||||
if (!gf) {
|
||||
if (cparams.pipeline_parallel) {
|
||||
LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__);
|
||||
@@ -815,7 +816,7 @@ void llama_context::sched_reserve() {
|
||||
n_splits_tg = n_splits_pp;
|
||||
n_nodes_tg = n_nodes_pp;
|
||||
} else {
|
||||
auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.hparams.no_alloc);
|
||||
auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.no_alloc());
|
||||
if (!gf) {
|
||||
throw std::runtime_error("failed to allocate compute tg buffers");
|
||||
}
|
||||
@@ -837,10 +838,10 @@ void llama_context::sched_reserve() {
|
||||
// [TAG_RESERVE_DIAG_DECAY]
|
||||
// the `inp_diag_decay` tensor size scales with `n_seq_tokens^2` which
|
||||
// makes `n_seqs == 1` use more memory for the compute graph compared to `n_seqs > 1`
|
||||
gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.hparams.no_alloc);
|
||||
gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.no_alloc());
|
||||
break;
|
||||
default:
|
||||
gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc);
|
||||
gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.no_alloc());
|
||||
};
|
||||
|
||||
if (!gf) {
|
||||
@@ -851,7 +852,7 @@ void llama_context::sched_reserve() {
|
||||
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
|
||||
ggml_backend_t backend = backend_ptrs[i];
|
||||
ggml_backend_buffer_type_t buft = backend_buft[i];
|
||||
if (!model.hparams.no_alloc) {
|
||||
if (!model.no_alloc()) {
|
||||
backend_buf_exp_size[i] = ggml_backend_sched_get_buffer_size(sched.get(), backend);
|
||||
}
|
||||
if (backend_buf_exp_size[i] > 1) {
|
||||
@@ -3730,7 +3731,7 @@ llama_memory_breakdown llama_context::memory_breakdown() const {
|
||||
ret[buft].context += size;
|
||||
}
|
||||
}
|
||||
if (model.hparams.no_alloc) {
|
||||
if (model.no_alloc()) {
|
||||
for (size_t i = 0; i < backends.size(); ++i) {
|
||||
ggml_backend_t backend = backends[i].get();
|
||||
ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend);
|
||||
|
||||
Reference in new issue
Block a user