diff --git a/common/fit.cpp b/common/fit.cpp index 55b457e10..10d57f956 100644 --- a/common/fit.cpp +++ b/common/fit.cpp @@ -61,6 +61,7 @@ static std::vector common_get_device_memory_data_impl( const llama_context_params * cparams, std::vector & devs, uint32_t & hp_ngl, + uint32_t & hp_ngl_skip, uint32_t & hp_n_ctx_train, uint32_t & hp_n_expert, ggml_log_level log_level) { @@ -152,6 +153,8 @@ static std::vector common_get_device_memory_data_impl( if (mparams->load_mtp) { hp_ngl += llama_model_n_layer_nextn(model); } + // without MTP the NextN layers hold no weights but the model loader still counts them for n_gpu_layers + hp_ngl_skip = mparams->load_mtp ? 0 : llama_model_n_layer_nextn(model); hp_n_ctx_train = llama_model_n_ctx_train(model); hp_n_expert = llama_model_n_expert(model); @@ -240,11 +243,12 @@ common_device_memory_data_vec common_get_device_memory_data( const llama_context_params * cparams, std::vector & devs, uint32_t & hp_ngl, + uint32_t & hp_ngl_skip, uint32_t & hp_n_ctx_train, uint32_t & hp_n_expert, ggml_log_level log_level) { std::vector impl = common_get_device_memory_data_impl( - path_model, mparams, cparams, devs, hp_ngl, hp_n_ctx_train, hp_n_expert, log_level); + path_model, mparams, cparams, devs, hp_ngl, hp_ngl_skip, hp_n_ctx_train, hp_n_expert, log_level); common_device_memory_data_vec ret(impl.size()); for (size_t i = 0; i < impl.size(); i++) { @@ -270,6 +274,7 @@ static void common_params_fit_impl( std::vector devs; uint32_t hp_ngl = 0; // hparams.n_gpu_layers + uint32_t hp_nsk = 0; // number of skipped NextN layers, counted by the model loader but not by hp_ngl uint32_t hp_nct = 0; // hparams.n_ctx_train uint32_t hp_nex = 0; // hparams.n_expert @@ -291,6 +296,7 @@ static void common_params_fit_impl( if (dmds_extra.empty() || n_ctx_extra != cparams->n_ctx) { std::vector devs_extra; uint32_t ngl_extra = 0; + uint32_t nsk_extra = 0; uint32_t nct_extra = 0; uint32_t nex_extra = 0; @@ -302,7 +308,7 @@ static void common_params_fit_impl( dmds_t measured; try { measured = common_get_device_memory_data_impl( - extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nct_extra, nex_extra, log_level); + extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nsk_extra, nct_extra, nex_extra, log_level); } catch (const std::runtime_error & e) { // the extra model is optional, fit the main model alone rather than giving up LOG_WRN("%s: failed to measure the memory of the extra model, fitting without it: %s\n", __func__, e.what()); @@ -342,7 +348,7 @@ static void common_params_fit_impl( // step 1: get data for default parameters and check whether any changes are necessary in the first place LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__); - dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level); // saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min: const uint32_t n_ctx_max = (uint32_t) std::min(uint64_t(hp_nct) * n_streams, UINT32_MAX); @@ -354,7 +360,7 @@ static void common_params_fit_impl( if (n_streams > 1) { LOG_TRC("%s: context size unset and KV cache not unified -> using %" PRIu32 " for %" PRIu32 " sequences:\n", __func__, n_ctx_max, n_streams); - dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level); } } add_extra_memory(dmds_full); @@ -493,7 +499,7 @@ static void common_params_fit_impl( int64_t sum_projected_used_min_ctx = 0; cparams->n_ctx = n_ctx_min_total; - dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level); add_extra_memory(dmds_min_ctx); if (nd == 0) { sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total(); @@ -640,6 +646,20 @@ static void common_params_fit_impl( assert(uint32_t(mparams.n_gpu_layers) <= hp_ngl + 1); uint32_t il0 = hp_ngl + 1 - mparams.n_gpu_layers; // start index for tensor buft overrides + // the skipped NextN layers come right before the output layer, put them on the same device as the output layer + // so that the layers assigned by the model loader match the layers above + if (hp_nsk > 0 && mparams.n_gpu_layers > 0) { + mparams.n_gpu_layers += hp_nsk; + if (nd > 1) { + // the last devices have no layers if not even the output layer fits + size_t id_last = nd - 1; + while (ngl_per_device[id_last].n_layer == 0) { + id_last--; + } + tensor_split[id_last] += hp_nsk; + } + } + mparams.tensor_split = tensor_split; size_t itbo = 0; @@ -675,7 +695,7 @@ static void common_params_fit_impl( set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy); dmds_t dmd_nl = common_get_device_memory_data_impl( - path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level); add_extra_memory(dmd_nl); LOG_TRC("%s: memory for test allocation by device:\n", func_name); @@ -704,7 +724,7 @@ static void common_params_fit_impl( LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__); dmds_t dmds_cpu_moe = common_get_device_memory_data_impl( - path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level); add_extra_memory(dmds_cpu_moe); for (size_t id = 0; id < nd; id++) { @@ -1185,10 +1205,11 @@ void common_fit_print( llama_context_params * cparams) { std::vector devs; uint32_t hp_ngl = 0; // hparams.n_gpu_layers + uint32_t hp_nsk = 0; // number of skipped NextN layers, counted by the model loader but not by hp_ngl uint32_t hp_nct = 0; // hparams.n_ctx_train uint32_t hp_nex = 0; // hparams.n_expert - auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR); + auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR); GGML_ASSERT(dmd.size() == devs.size() + 1); for (size_t id = 0; id < devs.size(); id++) { diff --git a/common/fit.h b/common/fit.h index 47478bba1..b2be5e4aa 100644 --- a/common/fit.h +++ b/common/fit.h @@ -104,6 +104,7 @@ common_device_memory_data_vec common_get_device_memory_data( const llama_context_params * cparams, std::vector & devs, uint32_t & hp_ngl, + uint32_t & hp_ngl_skip, uint32_t & hp_n_ctx_train, uint32_t & hp_n_expert, ggml_log_level log_level);