Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

llama : add --n-cpu-ffn option (#26622)

Browse Source 
* common : dedupe --n-cpu-moe / --spec-draft-n-cpu-moe override loops

* common : add --n-cpu-ffn to CPU-offload dense FFN weights of first N layers

* common : generalize llm_ffn_block_regex over the FFN regex, drop TODO
This commit is contained in:
Jonas J
2026-08-27 11:26:42 +02:00
committed by GitHub
parent d7a2074112
commit c5fc7e3488
3 changed files with 29 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+13 -11
View File
@@ -2750,14 +2750,20 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
if (value < 0) {
throw std::invalid_argument("invalid value");
}
for (int i = 0; i < value; ++i) {
// keep strings alive and avoid leaking memory by storing them in a static vector
static std::list<std::string> buft_overrides;
buft_overrides.push_back(llm_ffn_exps_block_regex(i));
params.tensor_buft_overrides.push_back({buft_overrides.back().c_str(), ggml_backend_cpu_buffer_type()});
}
llm_add_n_cpu_ffn_overrides(value, LLM_FFN_EXPS_REGEX, params.tensor_buft_overrides);
}
).set_env("LLAMA_ARG_N_CPU_MOE"));
add_opt(common_arg(
{"-ncffn", "--n-cpu-ffn"}, "N",
"keep the dense FFN weights of the first N layers in the CPU\n"
"(dense models; for MoE expert weights use --n-cpu-moe)",
[](common_params & params, int value) {
if (value < 0) {
throw std::invalid_argument("invalid value");
}
llm_add_n_cpu_ffn_overrides(value, LLM_FFN_DENSE_REGEX, params.tensor_buft_overrides);
}
).set_env("LLAMA_ARG_N_CPU_FFN"));
GGML_ASSERT(params.n_gpu_layers < 0); // string_format would need to be extended for a default >= 0
add_opt(common_arg(
{"-ngl", "--gpu-layers", "--n-gpu-layers"}, "N",
@@ -4084,11 +4090,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
if (value < 0) {
throw std::invalid_argument("invalid value");
}
for (int i = 0; i < value; ++i) {
static std::list<std::string> buft_overrides_draft;
buft_overrides_draft.push_back(llm_ffn_exps_block_regex(i));
params.speculative.draft.tensor_buft_overrides.push_back({buft_overrides_draft.back().c_str(), ggml_backend_cpu_buffer_type()});
}
llm_add_n_cpu_ffn_overrides(value, LLM_FFN_EXPS_REGEX, params.speculative.draft.tensor_buft_overrides);
}
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE"));