common : fractional -ncmoe for tensor-granularity expert placement
Extends --n-cpu-moe to accept a fractional layer count. The integer part offloads whole layers as before; the fractional part offloads a subset of the boundary layer expert tensors (gate, then up), keeping down_proj resident. This realizes ATSInfer tensor-granularity static placement, giving sub-layer control over expert VRAM residency. Placement-only, lossless. Assisted-by: Claude
This commit is contained in:
+7
-5
@@ -2606,15 +2606,17 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
).set_env("LLAMA_ARG_CPU_MOE"));
|
||||
add_opt(common_arg(
|
||||
{"-ncmoe", "--n-cpu-moe"}, "N",
|
||||
"keep the Mixture of Experts (MoE) weights of the first N layers in the CPU",
|
||||
[](common_params & params, int value) {
|
||||
if (value < 0) {
|
||||
"keep the Mixture of Experts (MoE) weights of the first N layers in the CPU; "
|
||||
"fractional N offloads part of the boundary layer at tensor granularity",
|
||||
[](common_params & params, const std::string & value) {
|
||||
const double n = std::stod(value);
|
||||
if (n < 0) {
|
||||
throw std::invalid_argument("invalid value");
|
||||
}
|
||||
for (int i = 0; i < value; ++i) {
|
||||
for (const std::string & re : llm_ffn_exps_cpu_block_regexes(n)) {
|
||||
// keep strings alive and avoid leaking memory by storing them in a static vector
|
||||
static std::list<std::string> buft_overrides;
|
||||
buft_overrides.push_back(llm_ffn_exps_block_regex(i));
|
||||
buft_overrides.push_back(re);
|
||||
params.tensor_buft_overrides.push_back({buft_overrides.back().c_str(), ggml_backend_cpu_buffer_type()});
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user