server: add --connect, allow p2p access to server outside LAN (#28277)

* init

* wip

* wire it up

* make it clear that it's experimental

* add props.connect_code

* nits

* fix windows build

* server: do not advertise a connect code without the tunnel

--connect-code on its own had no effect other than being served in
/props, so the Web UI showed a remote access code that no tunnel was
serving. Clear it when --connect is off, and say so once at startup.

The teardown comment claimed that terminate() closes the child stdout.
It does not: the log thread leaves fgets() only once every writer of
that pipe is gone, so a descendant of llama-connect would block the
join forever. Describe the real contract instead.

Also restore the --reasoning-preserve default in the generated docs.

* Update tools/connect/CMakeLists.txt

Co-authored-by: Adrien Gallouët <angt@huggingface.co>

* server: dial llama-connect on the bound port and a reachable host

Pass ctx_http.port so --port 0 and the GCP override work, pick the
dial host from --host (wildcards map to loopback, IPv6 is bracketed)
and refuse --connect when only unix sockets are bound.

---------

Co-authored-by: Pascal <admin@serveurperso.com>
Co-authored-by: Adrien Gallouët <angt@huggingface.co>
This commit is contained in:
authored and GitHub committed 2026-10-20 00:23:14 +02:00
1 parent ab7ed0d4b5
commit 7adf160c41
18 files changed
+644 -53

No files matched your search

+8 -49
View File
@@ -35,12 +35,6 @@
extern char **environ;
#endif
#if defined(__APPLE__) && defined(__MACH__)
// macOS: use _NSGetExecutablePath to get the executable path
#include <mach-o/dyld.h>
#include <limits.h>
#endif
#define DEFAULT_STOP_TIMEOUT 10 // seconds
#define CMD_ROUTER_TO_CHILD_EXIT "cmd_router_to_child:exit"
@@ -436,47 +430,6 @@ struct server_lru_sched {
// delete). distinct from params.timeout_read/write which only applies to the generation proxy
static constexpr int STREAM_LOOKUP_TIMEOUT_MS = 250;
static std::filesystem::path get_server_exec_path() {
#if defined(_WIN32)
wchar_t buf[32768] = { 0 }; // Large buffer to handle long paths
DWORD len = GetModuleFileNameW(nullptr, buf, _countof(buf));
if (len == 0 || len >= _countof(buf)) {
throw std::runtime_error("GetModuleFileNameW failed or path too long");
}
return std::filesystem::path(buf);
#elif defined(__APPLE__) && defined(__MACH__)
char small_path[PATH_MAX];
uint32_t size = sizeof(small_path);
if (_NSGetExecutablePath(small_path, &size) == 0) {
// resolve any symlinks to get absolute path
try {
return std::filesystem::canonical(std::filesystem::path(small_path));
} catch (...) {
return std::filesystem::path(small_path);
}
} else {
// buffer was too small, allocate required size and call again
std::vector<char> buf(size);
if (_NSGetExecutablePath(buf.data(), &size) == 0) {
try {
return std::filesystem::canonical(std::filesystem::path(buf.data()));
} catch (...) {
return std::filesystem::path(buf.data());
}
}
throw std::runtime_error("_NSGetExecutablePath failed after buffer resize");
}
#else
char path[FILENAME_MAX];
ssize_t count = readlink("/proc/self/exe", path, FILENAME_MAX);
if (count <= 0) {
throw std::runtime_error("failed to resolve /proc/self/exe");
}
return std::filesystem::path(std::string(path, count));
#endif
}
static void unset_reserved_args(common_preset & preset, bool unset_model_args) {
preset.unset_option("LLAMA_ARG_SSL_KEY_FILE");
preset.unset_option("LLAMA_ARG_SSL_CERT_FILE");
@@ -486,6 +439,8 @@ static void unset_reserved_args(common_preset & preset, bool unset_model_args) {
preset.unset_option("LLAMA_ARG_MODELS_MAX");
preset.unset_option("LLAMA_ARG_MODELS_PRESET");
preset.unset_option("LLAMA_ARG_MODELS_AUTOLOAD");
preset.unset_option("LLAMA_ARG_CONNECT");
preset.unset_option("LLAMA_ARG_CONNECT_CODE");
if (unset_model_args) {
preset.unset_option("LLAMA_ARG_MODEL");
preset.unset_option("LLAMA_ARG_MMPROJ");
@@ -2282,7 +2237,7 @@ void server_models_routes::init_routes() {
if (name.empty()) {
// main instance
auto res = std::make_unique<server_http_res>();
res_ok(res, {
json props = {
// TODO: add support for this on web UI
{"role", "router"},
{"max_instances", params.models_max},
@@ -2298,7 +2253,11 @@ void server_models_routes::init_routes() {
{"ui_settings", ui_settings},
{"build_info", std::string(llama_build_info())},
{"cors_proxy_enabled", params.ui_mcp_proxy},
});
};
if (!params.server_connect_code.empty()) {
props["connect_code"] = params.server_connect_code;
}
res_ok(res, props);
return res;
}
return proxy_get(req);