server: add router device memory margin parameter for dynamic unloading (#21231)
* server: add --models-memory-max parameter to allow dynamically unloading models when they exceed a memory size threshold estimate with to-be-loaded model size included use no_alloc to get memory requirements for model load only set model memory_mb if not previously calculated use memory margin instead of total size limit, apply to each device separately add server memory debug logging move llama_context_device_memory function to llama-ext.h fix model count exceeded check improve memory_per_device map naming improve variable naming, fix style also strip models memory margin from child processes cont : clean-up replace device memory map with buft memory map. Use llama_get_memory_breakdown extract duplicated check into helper function move model memory estimation to subprocess precompute name->buft map, map GPU host types to CPU buft cleanup unused variable remove duplicated init calls * adapt to upstream changes * wip * common_memory_reports internface * SERVER_ROLE_MEASURE * server-models-sched * fix build * server : fix memory accounting of measure children and claim spin A measure child runs no_alloc, so the free memory it reports does not include its estimate: subtracting self there erased the memory held by other processes, down to zero with --fit on, and the next load could overshoot the margin. Subtract self only for instance reports. A queued load only checked --models-max before claiming, so when the memory margin blocked it behind a busy model, load() threw and the waiter retried in a tight loop. Refuse the claim while the limits are exceeded and no idle model can be unloaded, so the waiter sleeps on the condition variable until one goes idle. --------- Co-authored-by: Xuan Son Nguyen <son@huggingface.co> Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
15 files changed
+1323
-357
No files matched your search
+46
-1
@@ -424,6 +424,45 @@ static int server_run_download(common_params & params, server_role & role) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//
|
||||
// measure worker: a child of the router, reports the memory a model needs without loading it
|
||||
//
|
||||
|
||||
static int server_run_measure(common_params & params, server_role & role, common_models_handler & models_handler) {
|
||||
// the router sets --offline: only resolve the local path, never download
|
||||
try {
|
||||
common_models_handler_apply(models_handler, params);
|
||||
} catch (const std::exception & e) {
|
||||
SRV_ERR("failed to resolve model: %s\n", e.what());
|
||||
return 1;
|
||||
}
|
||||
|
||||
const std::string model_name = params.model.get_name();
|
||||
|
||||
// TODO: mmproj is not in llama_get_memory_breakdown, a multimodal model is underestimated
|
||||
json payload;
|
||||
try {
|
||||
common_memory_reports m = common_measure_memory(params);
|
||||
payload = {
|
||||
{"result", "measured"},
|
||||
{"n_ctx_train", m.n_ctx_train},
|
||||
{"min", server_model_memory(m.n_ctx_min, m.info_min).to_json()},
|
||||
{"max", server_model_memory(m.n_ctx_max, m.info_max).to_json()},
|
||||
};
|
||||
SRV_INF("memory measured for model name=%s at n_ctx=%u (min) and n_ctx=%u (max)\n",
|
||||
model_name.c_str(), m.n_ctx_min, m.n_ctx_max);
|
||||
} catch (const std::exception & e) {
|
||||
SRV_ERR("failed to measure memory for model name=%s: %s\n", model_name.c_str(), e.what());
|
||||
payload = {
|
||||
{"result", "failed"},
|
||||
{"error", e.what()},
|
||||
};
|
||||
}
|
||||
|
||||
role.notify_to_router(server_state_to_str(SERVER_STATE_MEASURING), payload);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//
|
||||
// router
|
||||
//
|
||||
@@ -664,7 +703,12 @@ static int server_run_instance(common_params & params, server_role & role, commo
|
||||
std::thread monitor_thread;
|
||||
if (role.is_managed_by_router()) {
|
||||
monitor_thread = role.setup(shutdown_handler);
|
||||
role.notify_to_router(server_state_to_str(SERVER_STATE_READY), routes.get_model_info());
|
||||
json info = routes.get_model_info();
|
||||
// the router tracks device memory from these reports
|
||||
if (const auto * ll_ctx = ctx_server.get_llama_context()) {
|
||||
info["memory"] = server_model_memory(llama_n_ctx(ll_ctx), common_memory_breakdown_rows(ll_ctx)).to_json();
|
||||
}
|
||||
role.notify_to_router(server_state_to_str(SERVER_STATE_READY), info);
|
||||
}
|
||||
|
||||
// this call blocks the main thread until queue_tasks.terminate() is called
|
||||
@@ -717,6 +761,7 @@ static int server_run(common_params & params, int argc, char ** argv, server_rol
|
||||
|
||||
switch (role_type) {
|
||||
case SERVER_ROLE_DOWNLOAD: return server_run_download(params, role);
|
||||
case SERVER_ROLE_MEASURE: return server_run_measure(params, role, models_handler);
|
||||
case SERVER_ROLE_ROUTER: return server_run_router(params, argc, argv, opts);
|
||||
case SERVER_ROLE_INSTANCE: return server_run_instance(params, role, models_handler, opts);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user