upstream-carry #32777: fit : prioritize looped MoE layers in device memory (open upstream PR, temporary)
Temporary carry of the open upstream PR
https://github.com/ggml-org/llama.cpp/pull/32777 at head 01945dcb0 (one
commit), carried at the owner's request: loop-aware --fit placement for
inkling2_loop (llama_model_layer_visits ranks the looped layers first, so
their experts stay in device memory before the single-pass layers). Drop
this commit when the PR is merged upstream (FORK.md F12).
Applied cleanly on upstream 5bacb7c86 plus the fork; no conflict
resolution. The F13 interaction (views break the pointer-identity loop
detection) is fixed in a separate fork commit, not here.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PZz44SLQvTXMyWGio6t9DZ
This commit is contained in:
1 parent
5f88124b38
commit
6c895a491d
3 files changed
+181
-17
No files matched your search
+145
-17
@@ -7,6 +7,7 @@
|
||||
|
||||
#include "../src/llama-ext.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cassert>
|
||||
#include <stdexcept>
|
||||
@@ -64,7 +65,8 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
|
||||
uint32_t & hp_ngl_skip,
|
||||
uint32_t & hp_n_ctx_train,
|
||||
uint32_t & hp_n_expert,
|
||||
ggml_log_level log_level) {
|
||||
ggml_log_level log_level,
|
||||
std::vector<uint32_t> * hp_layer_visits = nullptr) {
|
||||
common_fit_log_scope log_scope(log_level);
|
||||
|
||||
llama_model_params mparams_copy = *mparams;
|
||||
@@ -157,6 +159,9 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
|
||||
hp_ngl_skip = mparams->load_mtp ? 0 : llama_model_n_layer_nextn(model);
|
||||
hp_n_ctx_train = llama_model_n_ctx_train(model);
|
||||
hp_n_expert = llama_model_n_expert(model);
|
||||
if (hp_layer_visits != nullptr) {
|
||||
*hp_layer_visits = llama_model_layer_visits(model);
|
||||
}
|
||||
|
||||
common_memory_breakdown_print(ctx);
|
||||
|
||||
@@ -277,6 +282,7 @@ static void common_params_fit_impl(
|
||||
uint32_t hp_nsk = 0; // number of skipped NextN layers, counted by the model loader but not by hp_ngl
|
||||
uint32_t hp_nct = 0; // hparams.n_ctx_train
|
||||
uint32_t hp_nex = 0; // hparams.n_expert
|
||||
std::vector<uint32_t> hp_layer_visits;
|
||||
|
||||
// with non-unified kv, we need to take into account n_streams
|
||||
// for example, if memory can hold more than model's trained context size, we must extend the n_ctx to hold enough n_streams
|
||||
@@ -348,7 +354,7 @@ static void common_params_fit_impl(
|
||||
// step 1: get data for default parameters and check whether any changes are necessary in the first place
|
||||
|
||||
LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__);
|
||||
dmds_t dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
dmds_t dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level, &hp_layer_visits);
|
||||
|
||||
// saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min:
|
||||
const uint32_t n_ctx_max = (uint32_t) std::min<uint64_t>(uint64_t(hp_nct) * n_streams, UINT32_MAX);
|
||||
@@ -631,6 +637,42 @@ static void common_params_fit_impl(
|
||||
|
||||
const size_t ntbo = llama_max_tensor_buft_overrides();
|
||||
|
||||
const bool has_looped_layers = std::any_of(
|
||||
hp_layer_visits.begin(), hp_layer_visits.end(),
|
||||
[](uint32_t v) { return v != 1; });
|
||||
|
||||
// For looped architectures (e.g. inkling2_loop), logical layers share physical weights across passes
|
||||
// and extra KV slots have visits == 0. Rank physical layers by descending visit count so hot looped
|
||||
// MoE layers stay in device memory before single-pass prelude/coda layers.
|
||||
auto get_device_phys_layers = [&](const uint32_t dev_il0, const uint32_t dev_il1) -> std::vector<uint32_t> {
|
||||
std::vector<uint32_t> phys;
|
||||
const uint32_t limit = std::min(dev_il1, (uint32_t) hp_layer_visits.size());
|
||||
for (uint32_t il = dev_il0; il < limit; ++il) {
|
||||
if (hp_layer_visits[il] > 0) {
|
||||
phys.push_back(il);
|
||||
}
|
||||
}
|
||||
std::stable_sort(phys.begin(), phys.end(), [&](uint32_t a, uint32_t b) {
|
||||
return hp_layer_visits[a] > hp_layer_visits[b];
|
||||
});
|
||||
return phys;
|
||||
};
|
||||
|
||||
auto device_layer_slice = [&](const std::vector<ngl_t> & cfg, const size_t target_id) -> std::pair<uint32_t, uint32_t> {
|
||||
uint32_t total_ngl = 0;
|
||||
for (size_t id = 0; id < nd; ++id) {
|
||||
total_ngl += cfg[id].n_layer;
|
||||
}
|
||||
uint32_t cur_il0 = hp_ngl + 1 - total_ngl;
|
||||
for (size_t id = 0; id < target_id; ++id) {
|
||||
cur_il0 += cfg[id].n_layer;
|
||||
}
|
||||
const uint32_t dev_n_tf = (target_id == nd - 1 && cfg[target_id].n_layer > 0)
|
||||
? cfg[target_id].n_layer - 1
|
||||
: cfg[target_id].n_layer;
|
||||
return { cur_il0, cur_il0 + dev_n_tf };
|
||||
};
|
||||
|
||||
// utility function to set n_gpu_layers and tensor_split
|
||||
auto set_ngl_tensor_split_tbo = [&](
|
||||
const std::vector<ngl_t> & ngl_per_device,
|
||||
@@ -663,22 +705,45 @@ static void common_params_fit_impl(
|
||||
mparams.tensor_split = tensor_split;
|
||||
|
||||
size_t itbo = 0;
|
||||
for (size_t id = 0; id < nd; id++) {
|
||||
il0 += ngl_per_device[id].n_full();
|
||||
for (uint32_t il = il0; il < il0 + ngl_per_device[id].n_part; il++) {
|
||||
if (itbo + 1 >= ntbo) {
|
||||
tensor_buft_overrides[itbo].pattern = nullptr;
|
||||
tensor_buft_overrides[itbo].buft = nullptr;
|
||||
itbo++;
|
||||
mparams.tensor_buft_overrides = tensor_buft_overrides;
|
||||
throw common_params_fit_exception("llama_max_tensor_buft_overrides() == "
|
||||
+ std::to_string(ntbo) + " is insufficient for model");
|
||||
}
|
||||
tensor_buft_overrides[itbo].pattern = get_overflow_pattern(il, il == il0 ? ngl_per_device[id].overflow_type : LAYER_FRACTION_MOE);
|
||||
tensor_buft_overrides[itbo].buft = il == il0 ? overflow_bufts[id] : ggml_backend_cpu_buffer_type();
|
||||
auto push_tbo = [&](const char * pattern, ggml_backend_buffer_type_t buft) {
|
||||
if (itbo + 1 >= ntbo) {
|
||||
tensor_buft_overrides[itbo].pattern = nullptr;
|
||||
tensor_buft_overrides[itbo].buft = nullptr;
|
||||
itbo++;
|
||||
mparams.tensor_buft_overrides = tensor_buft_overrides;
|
||||
throw common_params_fit_exception("llama_max_tensor_buft_overrides() == "
|
||||
+ std::to_string(ntbo) + " is insufficient for model");
|
||||
}
|
||||
il0 += ngl_per_device[id].n_part;
|
||||
tensor_buft_overrides[itbo].pattern = pattern;
|
||||
tensor_buft_overrides[itbo].buft = buft;
|
||||
itbo++;
|
||||
};
|
||||
|
||||
for (size_t id = 0; id < nd; id++) {
|
||||
if (!has_looped_layers) {
|
||||
il0 += ngl_per_device[id].n_full();
|
||||
for (uint32_t il = il0; il < il0 + ngl_per_device[id].n_part; il++) {
|
||||
push_tbo(
|
||||
get_overflow_pattern(il, il == il0 ? ngl_per_device[id].overflow_type : LAYER_FRACTION_MOE),
|
||||
il == il0 ? overflow_bufts[id] : ggml_backend_cpu_buffer_type());
|
||||
}
|
||||
il0 += ngl_per_device[id].n_part;
|
||||
continue;
|
||||
}
|
||||
|
||||
const uint32_t dev_n_tf = (id == nd - 1 && ngl_per_device[id].n_layer > 0)
|
||||
? ngl_per_device[id].n_layer - 1
|
||||
: ngl_per_device[id].n_layer;
|
||||
const std::vector<uint32_t> phys = get_device_phys_layers(il0, il0 + dev_n_tf);
|
||||
const uint32_t n_part_tf = std::min(dev_n_tf, ngl_per_device[id].n_part);
|
||||
const uint32_t n_full_tf = dev_n_tf - n_part_tf;
|
||||
for (size_t i = n_full_tf; i < phys.size(); ++i) {
|
||||
const bool is_first = i == n_full_tf;
|
||||
push_tbo(
|
||||
get_overflow_pattern(phys[i], is_first ? ngl_per_device[id].overflow_type : LAYER_FRACTION_MOE),
|
||||
is_first ? overflow_bufts[id] : ggml_backend_cpu_buffer_type());
|
||||
}
|
||||
il0 += ngl_per_device[id].n_layer;
|
||||
}
|
||||
tensor_buft_overrides[itbo].pattern = nullptr;
|
||||
tensor_buft_overrides[itbo].buft = nullptr;
|
||||
@@ -715,6 +780,7 @@ static void common_params_fit_impl(
|
||||
};
|
||||
|
||||
int64_t global_surplus_cpu_moe = 0;
|
||||
bool keep_looped_moe_in_step3 = false;
|
||||
if (hp_nex > 0) {
|
||||
const static std::string pattern_moe_all = "blk\\.\\d+\\.ffn_(up|down|gate_up|gate)_(ch|)exps"; // matches all MoE tensors
|
||||
ggml_backend_buffer_type_t cpu_buft = ggml_backend_cpu_buffer_type();
|
||||
@@ -740,6 +806,43 @@ static void common_params_fit_impl(
|
||||
__func__, -global_surplus_cpu_moe/MiB);
|
||||
}
|
||||
|
||||
// For looped MoE models (e.g. inkling2_loop), CPU-offloading repeated core layers can inflate the
|
||||
// compute buffer enough that keeping looped experts on GPU uses less total VRAM than offloading all MoEs.
|
||||
if (has_looped_layers) {
|
||||
size_t itbo = 0;
|
||||
bool overrides_fit = true;
|
||||
for (uint32_t il = 0; il < hp_layer_visits.size(); ++il) {
|
||||
if (hp_layer_visits[il] == 1) {
|
||||
if (itbo + 1 >= ntbo) {
|
||||
overrides_fit = false;
|
||||
break;
|
||||
}
|
||||
tensor_buft_overrides[itbo++] = {get_overflow_pattern(il, LAYER_FRACTION_MOE), cpu_buft};
|
||||
}
|
||||
}
|
||||
if (overrides_fit && itbo > 0) {
|
||||
tensor_buft_overrides[itbo] = {nullptr, nullptr};
|
||||
mparams->tensor_buft_overrides = tensor_buft_overrides;
|
||||
|
||||
LOG_TRC("%s: getting device memory data with only non-looped MoE tensors moved to system memory:\n", __func__);
|
||||
dmds_t dmds_looped_gpu = common_get_device_memory_data_impl(
|
||||
paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
add_extra_memory(dmds_looped_gpu);
|
||||
|
||||
int64_t surplus_looped_gpu = 0;
|
||||
for (size_t id = 0; id < nd; id++) {
|
||||
surplus_looped_gpu += dmds_looped_gpu[id].free;
|
||||
surplus_looped_gpu -= int64_t(dmds_looped_gpu[id].mb.total()) + margins[id];
|
||||
}
|
||||
if (surplus_looped_gpu > 0 && surplus_looped_gpu >= global_surplus_cpu_moe) {
|
||||
LOG_TRC("%s: keeping looped MoE weights in device memory yields a larger surplus of %" PRId64 " MiB\n",
|
||||
__func__, surplus_looped_gpu/MiB);
|
||||
global_surplus_cpu_moe = surplus_looped_gpu;
|
||||
keep_looped_moe_in_step3 = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reset
|
||||
tensor_buft_overrides[0] = {nullptr, nullptr};
|
||||
mparams->tensor_buft_overrides = tensor_buft_overrides;
|
||||
@@ -784,6 +887,17 @@ static void common_params_fit_impl(
|
||||
ngl_per_device_high[id].n_layer = n_unassigned;
|
||||
if (hp_nex > 0) {
|
||||
ngl_per_device_high[id].n_part = size_t(id) < nd - 1 ? ngl_per_device_high[id].n_layer : ngl_per_device_high[id].n_layer - 1;
|
||||
if (keep_looped_moe_in_step3) {
|
||||
const auto [dev_il0, dev_il1] = device_layer_slice(ngl_per_device_high, id);
|
||||
uint32_t n_hot = 0;
|
||||
const uint32_t limit = std::min(dev_il1, (uint32_t) hp_layer_visits.size());
|
||||
for (uint32_t il = dev_il0; il < limit; ++il) {
|
||||
if (hp_layer_visits[il] > 1) {
|
||||
n_hot++;
|
||||
}
|
||||
}
|
||||
ngl_per_device_high[id].n_part -= std::min(ngl_per_device_high[id].n_part, n_hot);
|
||||
}
|
||||
}
|
||||
if (ngl_per_device_high[id].n_layer > 0) {
|
||||
std::vector<int64_t> mem_high = get_memory_for_layers(__func__, ngl_per_device_high, overflow_bufts);
|
||||
@@ -858,6 +972,12 @@ static void common_params_fit_impl(
|
||||
ngl_per_device_high[jd].n_part = 0;
|
||||
}
|
||||
size_t id_dense_start_high = nd - 1;
|
||||
if (has_looped_layers && id == id_dense_start && ngl_per_device_high[id].n_layer == ngl_per_device[id].n_layer) {
|
||||
const auto [dev_il0, dev_il1] = device_layer_slice(ngl_per_device_high, id);
|
||||
const uint32_t n_phys_dev = (uint32_t) get_device_phys_layers(dev_il0, dev_il1).size();
|
||||
const uint32_t dev_n_tf = dev_il1 - dev_il0;
|
||||
ngl_per_device_high[id].n_part = dev_n_tf - std::min(dev_n_tf, n_phys_dev);
|
||||
}
|
||||
std::vector<int64_t> mem_high = get_memory_for_layers(__func__, ngl_per_device_high, overflow_bufts);
|
||||
|
||||
if (mem_high[id] > targets[id]) {
|
||||
@@ -909,7 +1029,15 @@ static void common_params_fit_impl(
|
||||
}
|
||||
|
||||
// try to fit at least part of one more layer
|
||||
if (ngl_per_device[id_dense_start].n_layer > (id < nd - 1 ? 0 : 1)) {
|
||||
bool can_fit_partial = ngl_per_device[id_dense_start].n_layer > (id < nd - 1 ? 0 : 1);
|
||||
if (can_fit_partial && has_looped_layers && id == id_dense_start) {
|
||||
const auto [dev_il0, dev_il1] = device_layer_slice(ngl_per_device, id);
|
||||
const uint32_t n_phys_dev = (uint32_t) get_device_phys_layers(dev_il0, dev_il1).size();
|
||||
const uint32_t dev_n_tf = dev_il1 - dev_il0;
|
||||
const uint32_t n_full_tf = dev_n_tf - std::min(dev_n_tf, ngl_per_device[id].n_part);
|
||||
can_fit_partial = n_full_tf < n_phys_dev;
|
||||
}
|
||||
if (can_fit_partial) {
|
||||
std::vector<ngl_t> ngl_per_device_test = ngl_per_device;
|
||||
size_t id_dense_start_test = id_dense_start;
|
||||
ngl_per_device_test[id_dense_start_test].n_layer--;
|
||||
|
||||
@@ -112,6 +112,9 @@ using llama_memory_breakdown = std::map<ggml_backend_buffer_type_t, llama_memory
|
||||
LLAMA_API int32_t llama_model_n_expert (const struct llama_model * model);
|
||||
LLAMA_API int32_t llama_model_n_devices(const struct llama_model * model);
|
||||
|
||||
// number of logical passes that execute each layer's weights (0 for weight-sharing alias slots)
|
||||
LLAMA_API std::vector<uint32_t> llama_model_layer_visits(const struct llama_model * model);
|
||||
|
||||
LLAMA_API ggml_backend_dev_t llama_model_get_device(const struct llama_model * model, int i);
|
||||
|
||||
LLAMA_API llama_memory_breakdown llama_get_memory_breakdown(const struct llama_context * ctx);
|
||||
|
||||
@@ -38,6 +38,7 @@
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params & params) {
|
||||
@@ -4281,6 +4282,38 @@ int32_t llama_model_n_expert(const struct llama_model * model) {
|
||||
return model->hparams.n_expert;
|
||||
}
|
||||
|
||||
std::vector<uint32_t> llama_model_layer_visits(const struct llama_model * model) {
|
||||
const size_t n_layer = model->layers.size();
|
||||
std::vector<uint32_t> visits(n_layer, 0);
|
||||
std::unordered_map<const ggml_tensor *, size_t> owner;
|
||||
owner.reserve(n_layer);
|
||||
|
||||
for (size_t il = 0; il < n_layer; ++il) {
|
||||
const auto & l = model->layers[il];
|
||||
const ggml_tensor * key =
|
||||
l.ffn_up_exps ? l.ffn_up_exps :
|
||||
l.ffn_gate_up_exps ? l.ffn_gate_up_exps :
|
||||
l.ffn_gate_exps ? l.ffn_gate_exps :
|
||||
l.ffn_down_exps ? l.ffn_down_exps :
|
||||
l.attn_norm ? l.attn_norm :
|
||||
l.ffn_norm ? l.ffn_norm :
|
||||
l.wq ? l.wq :
|
||||
l.wqkv ? l.wqkv :
|
||||
l.ffn_up ? l.ffn_up :
|
||||
l.ffn_gate ? l.ffn_gate :
|
||||
l.ffn_down ? l.ffn_down :
|
||||
nullptr;
|
||||
if (!key) {
|
||||
visits[il] = 1;
|
||||
continue;
|
||||
}
|
||||
auto [it, inserted] = owner.emplace(key, il);
|
||||
visits[it->second]++;
|
||||
}
|
||||
|
||||
return visits;
|
||||
}
|
||||
|
||||
int32_t llama_model_n_devices(const struct llama_model * model) {
|
||||
return (int32_t)model->devices.size();
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user