server: add --connect, allow p2p access to server outside LAN (#28277)
* init * wip * wire it up * make it clear that it's experimental * add props.connect_code * nits * fix windows build * server: do not advertise a connect code without the tunnel --connect-code on its own had no effect other than being served in /props, so the Web UI showed a remote access code that no tunnel was serving. Clear it when --connect is off, and say so once at startup. The teardown comment claimed that terminate() closes the child stdout. It does not: the log thread leaves fgets() only once every writer of that pipe is gone, so a descendant of llama-connect would block the join forever. Describe the real contract instead. Also restore the --reasoning-preserve default in the generated docs. * Update tools/connect/CMakeLists.txt Co-authored-by: Adrien Gallouët <angt@huggingface.co> * server: dial llama-connect on the bound port and a reachable host Pass ctx_http.port so --port 0 and the GCP override work, pick the dial host from --host (wildcards map to loopback, IPv6 is bracketed) and refuse --connect when only unix sockets are bound. --------- Co-authored-by: Pascal <admin@serveurperso.com> Co-authored-by: Adrien Gallouët <angt@huggingface.co>
This commit is contained in:
18 files changed
+644
-53
No files matched your search
@@ -1,3 +1,4 @@
|
||||
#include "server-connect.h"
|
||||
#include "server-context.h"
|
||||
#include "server-http.h"
|
||||
#include "server-models.h"
|
||||
@@ -191,6 +192,20 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
|
||||
params.model_alias.insert(model_name);
|
||||
}
|
||||
|
||||
// check early, so a missing llama-connect fails-fast
|
||||
if (params.server_connect) {
|
||||
const std::string reason = server_connect::unavailable_reason(params);
|
||||
if (!reason.empty()) {
|
||||
SRV_ERR("--connect is not available: %s\n", reason.c_str());
|
||||
return 1;
|
||||
}
|
||||
params.server_connect_code = server_connect::resolve_code(params.server_connect_code);
|
||||
} else if (!params.server_connect_code.empty()) {
|
||||
// the code is only meaningful while the tunnel runs, and /props keys off it being set
|
||||
SRV_WRN("%s", "--connect-code is ignored without --connect\n");
|
||||
params.server_connect_code.clear();
|
||||
}
|
||||
|
||||
// note: this is guaranteed to out-live ctx_http and tools
|
||||
server_mcp mcp_mgr;
|
||||
|
||||
@@ -347,6 +362,10 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
|
||||
warn_names.push_back("router mode");
|
||||
}
|
||||
|
||||
if (params.server_connect) {
|
||||
warn_names.push_back("peer-to-peer tunnel (llama-connect, experimental)");
|
||||
}
|
||||
|
||||
if (params.ui_mcp_proxy) {
|
||||
ctx_http.get ("/cors-proxy", ex_wrapper(proxy_handler_get));
|
||||
ctx_http.post("/cors-proxy", ex_wrapper(proxy_handler_post));
|
||||
@@ -525,6 +544,16 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
|
||||
SRV_INF("listening on %s\n", address.c_str());
|
||||
}
|
||||
|
||||
// spawn only once listening, so the port is bound and the child startup health check reaches us. the destructor stops it
|
||||
server_connect connect_proc;
|
||||
if (params.server_connect && !connect_proc.start(params, ctx_http.port)) {
|
||||
SRV_ERR("%s", "exiting due to llama-connect error\n");
|
||||
ctx_http.stop();
|
||||
ctx_http.join();
|
||||
clean_up();
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (is_router_server) {
|
||||
if (!params.models_preset_hf.empty()) {
|
||||
SRV_WRN( "NOTE: using preset.ini from HF repo '%s'\n", params.models_preset_hf.c_str());
|
||||
|
||||
Reference in new issue
Block a user