server : fix draft model fit vs load inconsistency (#25056)

* fix: draft model fit vs load inconsistency

* refactor(server): unify draft/mtp parameter initialization, model, and context load
- moves speculative init to speculative.cpp
- changes server_context_impl model_dft and ctx_dft to use raw pointers

- fix: don't throttle progress callback when loading draft model
- refactor: rename draft model/ctx load method

* fix: valign
This commit is contained in:
Alex
2026-07-07 17:20:42 +03:00
committed by GitHub
parent 5eca4e3cab
commit f5525f7e7a
3 changed files with 176 additions and 107 deletions
+107
View File
@@ -2221,6 +2221,113 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) {
return n_max;
}
common_params common_base_params_to_speculative(const common_params & params) {
const bool has_draft = params.speculative.has_dft();
const auto & params_spec = params.speculative.draft;
common_params result = params;
if (has_draft) {
result.devices = params_spec.devices;
result.model = params_spec.mparams;
result.n_gpu_layers = params_spec.n_gpu_layers;
result.tensor_buft_overrides = params_spec.tensor_buft_overrides;
if (params_spec.cpuparams.n_threads > 0) {
result.cpuparams.n_threads = params_spec.cpuparams.n_threads;
result.cpuparams_batch.n_threads = params_spec.cpuparams_batch.n_threads;
}
}
result.cache_type_k = params_spec.cache_type_k;
result.cache_type_v = params_spec.cache_type_v;
result.n_outputs_max = params.n_parallel;
return result;
}
struct common_init_speculative_result::impl {
impl() = default;
~impl() = default;
// note: the order in which model, context, etc. are declared matters because their destructors will be called bottom-to-top
llama_model_ptr model;
llama_context_ptr context;
};
common_init_speculative_result::common_init_speculative_result(
common_params & params,
llama_model * model_tgt,
llama_context * ctx_tgt) :
pimpl(new impl{}) {
const bool has_draft = params.speculative.has_dft();
const bool spec_mtp = std::find(params.speculative.types.begin(),
params.speculative.types.end(),
COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
GGML_ASSERT(has_draft || spec_mtp);
auto mparams = common_model_params_to_llama(params);
auto cparams = common_context_params_to_llama(params);
if (spec_mtp) {
cparams.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
}
// note: for small models maybe we can set this to the maximum possible draft from all speculative types
// the extra memory for small models is likely negligible?
cparams.n_rs_seq = 0;
cparams.ctx_other = ctx_tgt;
std::string model_path;
if (has_draft) {
model_path = params.speculative.draft.mparams.path;
LOG_TRC("%s: loading draft model '%s'\n", __func__, model_path.c_str());
llama_model * model_dft = llama_model_load_from_file(params.model.path.c_str(), mparams);
if (model_dft == NULL) {
LOG_ERR("%s: failed to load draft model, '%s'\n", __func__, model_path.c_str());
return;
}
pimpl->model.reset(model_dft);
llama_context * ctx_dft = llama_init_from_model(model_dft, cparams);
if (ctx_dft == nullptr) {
LOG_ERR("%s: failed to create MTP context\n", __func__);
return;
}
pimpl->context.reset(ctx_dft);
} else if (spec_mtp) {
model_path = params.model.path;
LOG_TRC("%s: creating MTP draft context against the target model '%s'\n",
__func__, model_path.c_str());
llama_context * ctx_dft = llama_init_from_model(model_tgt, cparams);
if (ctx_dft == nullptr) {
LOG_ERR("%s: failed to create MTP context\n", __func__);
return;
}
pimpl->context.reset(ctx_dft);
}
}
common_init_speculative_result::~common_init_speculative_result() = default;
llama_model * common_init_speculative_result::model() {
return pimpl->model.get();
}
llama_context * common_init_speculative_result::context() {
return pimpl->context.get();
}
common_init_speculative_result_ptr common_init_speculative_from_params(common_params & params, llama_model * model_tgt, llama_context * ctx_tgt) {
return std::make_unique<common_init_speculative_result>(params, model_tgt, ctx_tgt);
}
// initialization of the speculative decoding system
//
common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq) {