server: (router) lazy-load startup_models after main setup (#27424)
* server: (router) lazy-load startup_models after main setup * only allow is_first_load to populate it * nits * nits 2
This commit is contained in:
@@ -136,6 +136,10 @@ private:
|
||||
// if true, the next get_meta() will trigger a reload of model list
|
||||
bool need_reload = false;
|
||||
|
||||
// models marked with load-on-startup, unset once load_startup_models() drains it
|
||||
// no value means the startup phase is over, so a reload must not queue anything
|
||||
std::optional<std::vector<std::string>> startup_models{std::in_place};
|
||||
|
||||
// conv_id -> model name that currently serves its stream session, lets the resumable stream
|
||||
// routes go straight to the owning child instead of polling every one. populated when
|
||||
// proxy_request forwards a POST carrying an X-Conversation-Id. best effort: a stale entry just
|
||||
@@ -231,6 +235,9 @@ public:
|
||||
// - if a model is not running, it will be added or updated according to the source
|
||||
void load_models();
|
||||
|
||||
// lazy-load startup_models, to be called after main() setup phase
|
||||
void load_startup_models();
|
||||
|
||||
// check if a model instance exists (thread-safe)
|
||||
bool has_model(const std::string & name);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user