fit : account for skipped NextN layers when setting n_gpu_layers (#31569)
* fit : account for skipped NextN layers when setting n_gpu_layers
Without MTP, fit counts n_layer + 1 layers while the model loader counts
n_layer_all + 1 for n_gpu_layers, including the NextN layers which hold no
weights. A fit result that offloads all layers then leaves layer 0 on the CPU
("offloaded 65/66 layers" for Qwen3.8-27B on two GPUs).
Add the skipped NextN layers to n_gpu_layers and to the tensor split of the
device that holds the output layer so the loader assigns the layers that
fit measured.
Fixes #31531
* fit : pass hp_ngl_skip by reference like the other hparams
* fit : add hp_ngl_skip to common_get_device_memory_data
This commit is contained in:
1 parent
553d2c2e7e
commit
2c192444c9
2 files changed
+30
-8
No files matched your search
+29
-8
@@ -61,6 +61,7 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
|
||||
const llama_context_params * cparams,
|
||||
std::vector<ggml_backend_dev_t> & devs,
|
||||
uint32_t & hp_ngl,
|
||||
uint32_t & hp_ngl_skip,
|
||||
uint32_t & hp_n_ctx_train,
|
||||
uint32_t & hp_n_expert,
|
||||
ggml_log_level log_level) {
|
||||
@@ -152,6 +153,8 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
|
||||
if (mparams->load_mtp) {
|
||||
hp_ngl += llama_model_n_layer_nextn(model);
|
||||
}
|
||||
// without MTP the NextN layers hold no weights but the model loader still counts them for n_gpu_layers
|
||||
hp_ngl_skip = mparams->load_mtp ? 0 : llama_model_n_layer_nextn(model);
|
||||
hp_n_ctx_train = llama_model_n_ctx_train(model);
|
||||
hp_n_expert = llama_model_n_expert(model);
|
||||
|
||||
@@ -240,11 +243,12 @@ common_device_memory_data_vec common_get_device_memory_data(
|
||||
const llama_context_params * cparams,
|
||||
std::vector<ggml_backend_dev_t> & devs,
|
||||
uint32_t & hp_ngl,
|
||||
uint32_t & hp_ngl_skip,
|
||||
uint32_t & hp_n_ctx_train,
|
||||
uint32_t & hp_n_expert,
|
||||
ggml_log_level log_level) {
|
||||
std::vector<llama_device_memory_data> impl = common_get_device_memory_data_impl(
|
||||
path_model, mparams, cparams, devs, hp_ngl, hp_n_ctx_train, hp_n_expert, log_level);
|
||||
path_model, mparams, cparams, devs, hp_ngl, hp_ngl_skip, hp_n_ctx_train, hp_n_expert, log_level);
|
||||
|
||||
common_device_memory_data_vec ret(impl.size());
|
||||
for (size_t i = 0; i < impl.size(); i++) {
|
||||
@@ -270,6 +274,7 @@ static void common_params_fit_impl(
|
||||
|
||||
std::vector<ggml_backend_dev_t> devs;
|
||||
uint32_t hp_ngl = 0; // hparams.n_gpu_layers
|
||||
uint32_t hp_nsk = 0; // number of skipped NextN layers, counted by the model loader but not by hp_ngl
|
||||
uint32_t hp_nct = 0; // hparams.n_ctx_train
|
||||
uint32_t hp_nex = 0; // hparams.n_expert
|
||||
|
||||
@@ -291,6 +296,7 @@ static void common_params_fit_impl(
|
||||
if (dmds_extra.empty() || n_ctx_extra != cparams->n_ctx) {
|
||||
std::vector<ggml_backend_dev_t> devs_extra;
|
||||
uint32_t ngl_extra = 0;
|
||||
uint32_t nsk_extra = 0;
|
||||
uint32_t nct_extra = 0;
|
||||
uint32_t nex_extra = 0;
|
||||
|
||||
@@ -302,7 +308,7 @@ static void common_params_fit_impl(
|
||||
dmds_t measured;
|
||||
try {
|
||||
measured = common_get_device_memory_data_impl(
|
||||
extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nct_extra, nex_extra, log_level);
|
||||
extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nsk_extra, nct_extra, nex_extra, log_level);
|
||||
} catch (const std::runtime_error & e) {
|
||||
// the extra model is optional, fit the main model alone rather than giving up
|
||||
LOG_WRN("%s: failed to measure the memory of the extra model, fitting without it: %s\n", __func__, e.what());
|
||||
@@ -342,7 +348,7 @@ static void common_params_fit_impl(
|
||||
// step 1: get data for default parameters and check whether any changes are necessary in the first place
|
||||
|
||||
LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__);
|
||||
dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
|
||||
// saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min:
|
||||
const uint32_t n_ctx_max = (uint32_t) std::min<uint64_t>(uint64_t(hp_nct) * n_streams, UINT32_MAX);
|
||||
@@ -354,7 +360,7 @@ static void common_params_fit_impl(
|
||||
if (n_streams > 1) {
|
||||
LOG_TRC("%s: context size unset and KV cache not unified -> using %" PRIu32 " for %" PRIu32 " sequences:\n",
|
||||
__func__, n_ctx_max, n_streams);
|
||||
dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
}
|
||||
}
|
||||
add_extra_memory(dmds_full);
|
||||
@@ -493,7 +499,7 @@ static void common_params_fit_impl(
|
||||
|
||||
int64_t sum_projected_used_min_ctx = 0;
|
||||
cparams->n_ctx = n_ctx_min_total;
|
||||
dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
add_extra_memory(dmds_min_ctx);
|
||||
if (nd == 0) {
|
||||
sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total();
|
||||
@@ -640,6 +646,20 @@ static void common_params_fit_impl(
|
||||
assert(uint32_t(mparams.n_gpu_layers) <= hp_ngl + 1);
|
||||
uint32_t il0 = hp_ngl + 1 - mparams.n_gpu_layers; // start index for tensor buft overrides
|
||||
|
||||
// the skipped NextN layers come right before the output layer, put them on the same device as the output layer
|
||||
// so that the layers assigned by the model loader match the layers above
|
||||
if (hp_nsk > 0 && mparams.n_gpu_layers > 0) {
|
||||
mparams.n_gpu_layers += hp_nsk;
|
||||
if (nd > 1) {
|
||||
// the last devices have no layers if not even the output layer fits
|
||||
size_t id_last = nd - 1;
|
||||
while (ngl_per_device[id_last].n_layer == 0) {
|
||||
id_last--;
|
||||
}
|
||||
tensor_split[id_last] += hp_nsk;
|
||||
}
|
||||
}
|
||||
|
||||
mparams.tensor_split = tensor_split;
|
||||
|
||||
size_t itbo = 0;
|
||||
@@ -675,7 +695,7 @@ static void common_params_fit_impl(
|
||||
set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy);
|
||||
|
||||
dmds_t dmd_nl = common_get_device_memory_data_impl(
|
||||
path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
add_extra_memory(dmd_nl);
|
||||
|
||||
LOG_TRC("%s: memory for test allocation by device:\n", func_name);
|
||||
@@ -704,7 +724,7 @@ static void common_params_fit_impl(
|
||||
|
||||
LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__);
|
||||
dmds_t dmds_cpu_moe = common_get_device_memory_data_impl(
|
||||
path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
add_extra_memory(dmds_cpu_moe);
|
||||
|
||||
for (size_t id = 0; id < nd; id++) {
|
||||
@@ -1185,10 +1205,11 @@ void common_fit_print(
|
||||
llama_context_params * cparams) {
|
||||
std::vector<ggml_backend_dev_t> devs;
|
||||
uint32_t hp_ngl = 0; // hparams.n_gpu_layers
|
||||
uint32_t hp_nsk = 0; // number of skipped NextN layers, counted by the model loader but not by hp_ngl
|
||||
uint32_t hp_nct = 0; // hparams.n_ctx_train
|
||||
uint32_t hp_nex = 0; // hparams.n_expert
|
||||
|
||||
auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
|
||||
auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
|
||||
GGML_ASSERT(dmd.size() == devs.size() + 1);
|
||||
|
||||
for (size_t id = 0; id < devs.size(); id++) {
|
||||
|
||||
Reference in new issue
Block a user