server: add --connect, allow p2p access to server outside LAN (#28277)
* init * wip * wire it up * make it clear that it's experimental * add props.connect_code * nits * fix windows build * server: do not advertise a connect code without the tunnel --connect-code on its own had no effect other than being served in /props, so the Web UI showed a remote access code that no tunnel was serving. Clear it when --connect is off, and say so once at startup. The teardown comment claimed that terminate() closes the child stdout. It does not: the log thread leaves fgets() only once every writer of that pipe is gone, so a descendant of llama-connect would block the join forever. Describe the real contract instead. Also restore the --reasoning-preserve default in the generated docs. * Update tools/connect/CMakeLists.txt Co-authored-by: Adrien Gallouët <angt@huggingface.co> * server: dial llama-connect on the bound port and a reachable host Pass ctx_http.port so --port 0 and the GCP override work, pick the dial host from --host (wildcards map to loopback, IPv6 is bracketed) and refuse --connect when only unix sockets are bound. --------- Co-authored-by: Pascal <admin@serveurperso.com> Co-authored-by: Adrien Gallouët <angt@huggingface.co>
This commit is contained in:
18 files changed
+644
-53
No files matched your search
@@ -35,12 +35,6 @@
|
||||
extern char **environ;
|
||||
#endif
|
||||
|
||||
#if defined(__APPLE__) && defined(__MACH__)
|
||||
// macOS: use _NSGetExecutablePath to get the executable path
|
||||
#include <mach-o/dyld.h>
|
||||
#include <limits.h>
|
||||
#endif
|
||||
|
||||
#define DEFAULT_STOP_TIMEOUT 10 // seconds
|
||||
|
||||
#define CMD_ROUTER_TO_CHILD_EXIT "cmd_router_to_child:exit"
|
||||
@@ -436,47 +430,6 @@ struct server_lru_sched {
|
||||
// delete). distinct from params.timeout_read/write which only applies to the generation proxy
|
||||
static constexpr int STREAM_LOOKUP_TIMEOUT_MS = 250;
|
||||
|
||||
static std::filesystem::path get_server_exec_path() {
|
||||
#if defined(_WIN32)
|
||||
wchar_t buf[32768] = { 0 }; // Large buffer to handle long paths
|
||||
DWORD len = GetModuleFileNameW(nullptr, buf, _countof(buf));
|
||||
if (len == 0 || len >= _countof(buf)) {
|
||||
throw std::runtime_error("GetModuleFileNameW failed or path too long");
|
||||
}
|
||||
return std::filesystem::path(buf);
|
||||
#elif defined(__APPLE__) && defined(__MACH__)
|
||||
char small_path[PATH_MAX];
|
||||
uint32_t size = sizeof(small_path);
|
||||
|
||||
if (_NSGetExecutablePath(small_path, &size) == 0) {
|
||||
// resolve any symlinks to get absolute path
|
||||
try {
|
||||
return std::filesystem::canonical(std::filesystem::path(small_path));
|
||||
} catch (...) {
|
||||
return std::filesystem::path(small_path);
|
||||
}
|
||||
} else {
|
||||
// buffer was too small, allocate required size and call again
|
||||
std::vector<char> buf(size);
|
||||
if (_NSGetExecutablePath(buf.data(), &size) == 0) {
|
||||
try {
|
||||
return std::filesystem::canonical(std::filesystem::path(buf.data()));
|
||||
} catch (...) {
|
||||
return std::filesystem::path(buf.data());
|
||||
}
|
||||
}
|
||||
throw std::runtime_error("_NSGetExecutablePath failed after buffer resize");
|
||||
}
|
||||
#else
|
||||
char path[FILENAME_MAX];
|
||||
ssize_t count = readlink("/proc/self/exe", path, FILENAME_MAX);
|
||||
if (count <= 0) {
|
||||
throw std::runtime_error("failed to resolve /proc/self/exe");
|
||||
}
|
||||
return std::filesystem::path(std::string(path, count));
|
||||
#endif
|
||||
}
|
||||
|
||||
static void unset_reserved_args(common_preset & preset, bool unset_model_args) {
|
||||
preset.unset_option("LLAMA_ARG_SSL_KEY_FILE");
|
||||
preset.unset_option("LLAMA_ARG_SSL_CERT_FILE");
|
||||
@@ -486,6 +439,8 @@ static void unset_reserved_args(common_preset & preset, bool unset_model_args) {
|
||||
preset.unset_option("LLAMA_ARG_MODELS_MAX");
|
||||
preset.unset_option("LLAMA_ARG_MODELS_PRESET");
|
||||
preset.unset_option("LLAMA_ARG_MODELS_AUTOLOAD");
|
||||
preset.unset_option("LLAMA_ARG_CONNECT");
|
||||
preset.unset_option("LLAMA_ARG_CONNECT_CODE");
|
||||
if (unset_model_args) {
|
||||
preset.unset_option("LLAMA_ARG_MODEL");
|
||||
preset.unset_option("LLAMA_ARG_MMPROJ");
|
||||
@@ -2282,7 +2237,7 @@ void server_models_routes::init_routes() {
|
||||
if (name.empty()) {
|
||||
// main instance
|
||||
auto res = std::make_unique<server_http_res>();
|
||||
res_ok(res, {
|
||||
json props = {
|
||||
// TODO: add support for this on web UI
|
||||
{"role", "router"},
|
||||
{"max_instances", params.models_max},
|
||||
@@ -2298,7 +2253,11 @@ void server_models_routes::init_routes() {
|
||||
{"ui_settings", ui_settings},
|
||||
{"build_info", std::string(llama_build_info())},
|
||||
{"cors_proxy_enabled", params.ui_mcp_proxy},
|
||||
});
|
||||
};
|
||||
if (!params.server_connect_code.empty()) {
|
||||
props["connect_code"] = params.server_connect_code;
|
||||
}
|
||||
res_ok(res, props);
|
||||
return res;
|
||||
}
|
||||
return proxy_get(req);
|
||||
|
||||
Reference in new issue
Block a user