Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

llama : add --n-cpu-ffn option (#26622)

Browse Source 
* common : dedupe --n-cpu-moe / --spec-draft-n-cpu-moe override loops

* common : add --n-cpu-ffn to CPU-offload dense FFN weights of first N layers

* common : generalize llm_ffn_block_regex over the FFN regex, drop TODO
This commit is contained in:
Jonas J
2026-08-27 11:26:42 +02:00
committed by GitHub
parent d7a2074112
commit c5fc7e3488
3 changed files with 29 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/common.h
+15 -3
View File
@@ -8,6 +8,7 @@
#include "ggml.h"
#include "llama.h"
#include <list>
#include <set>
#include <sstream>
#include <string>
@@ -1108,19 +1109,30 @@ const char * const LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count";
}
//
// MoE utils
// FFN offload utils
//
const char * const LLM_FFN_EXPS_REGEX = "\\.ffn_(up|down|gate|gate_up)_(ch|)exps";
inline std::string llm_ffn_exps_block_regex(int idx) {
return string_format("blk\\.%d%s", idx, LLM_FFN_EXPS_REGEX);
const char * const LLM_FFN_DENSE_REGEX = "\\.ffn_(up|down|gate)\\.";
inline std::string llm_ffn_block_regex(int idx, const char * ffn_regex) {
return string_format("blk\\.%d%s", idx, ffn_regex);
}
inline llama_model_tensor_buft_override llm_ffn_exps_cpu_override() {
return { LLM_FFN_EXPS_REGEX, ggml_backend_cpu_buffer_type() };
}
inline void llm_add_n_cpu_ffn_overrides(int n, const char * ffn_regex, std::vector<llama_model_tensor_buft_override> & overrides) {
// keep strings alive and avoid leaking memory by storing them in a static list
static std::list<std::string> buft_override_strings;
for (int i = 0; i < n; ++i) {
buft_override_strings.push_back(llm_ffn_block_regex(i, ffn_regex));
overrides.push_back({buft_override_strings.back().c_str(), ggml_backend_cpu_buffer_type()});
}
}
//
// training utils
//