diff --git a/common/arg.cpp b/common/arg.cpp index 4b716398e..8527c8a82 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -943,8 +943,8 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context } const bool mcp_enabled = !params.mcp_servers_config.empty() || !params.mcp_servers_json.empty(); - if ((!params.server_tools.empty() || mcp_enabled) && !params.cors_origins_explicit) { - LOG_WRN("server tools or MCP servers are enabled, using localhost as default CORS origin (change via --cors-origins)\n"); + if ((!params.server_tools.empty() || mcp_enabled || params.ui_mcp_proxy) && !params.cors_origins_explicit) { + LOG_WRN("server tools, MCP servers or MCP proxy are enabled, using localhost as default CORS origin (change via --cors-origins)\n"); params.cors_origins = "localhost"; } @@ -3361,7 +3361,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--cors-origins"}, "ORIGINS", string_format( "comma-separated list of allowed origins for CORS (default: %s)\n" - "if set to special value 'localhost', reflect the Origin header only if it is localhost", + "if set to special value 'localhost', only allow same-origin requests (e.g. the built-in UI)", params.cors_origins.c_str()), [](common_params & params, const std::string & value) { params.cors_origins = value; @@ -3417,7 +3417,8 @@ common_params_context common_params_parser_init(common_params & params, llama_ex add_opt(common_arg( {"--ui-mcp-proxy", "--webui-mcp-proxy"}, {"--no-ui-mcp-proxy", "--no-webui-mcp-proxy"}, - "experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)", + "experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)\n" + "note: for security reasons, this will limit --cors-origins to localhost by default", [](common_params & params, bool value) { params.ui_mcp_proxy = value; } @@ -3443,6 +3444,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.server_tools_runtime = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS_RUNTIME")); + add_opt(common_arg( + {"--allow-unauthenticated-tools"}, + "allow server tools, MCP servers or MCP proxy on a non-loopback address without an API key - only for trusted networks (default: disabled)", + [](common_params & params) { + params.server_tools_allow_unauth = true; + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_ALLOW_UNAUTHENTICATED_TOOLS")); add_opt(common_arg( {"--mcp-servers-config"}, "PATH", "experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)\n" diff --git a/common/common.h b/common/common.h index cc6eef833..2b51a36b3 100644 --- a/common/common.h +++ b/common/common.h @@ -705,6 +705,7 @@ struct common_params { // enable built-in tools std::vector server_tools; std::string server_tools_runtime; + bool server_tools_allow_unauth = false; // allow tools/MCP on a non-loopback address without API key // MCP server configs (Cursor-compatible JSON) std::string mcp_servers_config; // path to JSON file with MCP server definitions diff --git a/tools/cli/README.md b/tools/cli/README.md index 8e9f057f8..98fdef318 100644 --- a/tools/cli/README.md +++ b/tools/cli/README.md @@ -50,8 +50,8 @@ | `-kvo, --kv-offload, -nkvo, --no-kv-offload` | whether to enable KV cache offloading (default: enabled)
(env: LLAMA_ARG_KV_OFFLOAD) | | `--repack, -nr, --no-repack` | whether to enable weight repacking (default: enabled)
(env: LLAMA_ARG_REPACK) | | `--no-host` | bypass host buffer allowing extra buffers to be used
(env: LLAMA_ARG_NO_HOST) | -| `-ctk, --cache-type-k TYPE` | KV cache data type for K
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_K) | -| `-ctv, --cache-type-v TYPE` | KV cache data type for V
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_V) | +| `-ctk, --cache-type-k TYPE` | KV cache data type for K
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_K) | +| `-ctv, --cache-type-v TYPE` | KV cache data type for V
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_V) | | `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)
(env: LLAMA_ARG_DEFRAG_THOLD) | | `-np, --parallel N` | number of parallel sequences to decode (default: 1)
(env: LLAMA_ARG_N_PARALLEL) | | `--rpc SERVERS` | comma-separated list of RPC servers (host:port)
(env: LLAMA_ARG_RPC) | @@ -82,6 +82,7 @@ | `--control-vector-scaled FNAME:SCALE,...` | add a control vector with user defined scaling SCALE
note: use comma-separated values (format: FNAME:SCALE,...) | | `--control-vector-layer-range START END` | layer range to apply the control vector(s) to, start and end inclusive | | `-m, --model FNAME` | model path to load
(env: LLAMA_ARG_MODEL) | +| `--ple, --shard FNAME,...` | extra GGUF shards for the model, comma separated; may be given more than once
paths may live anywhere, including on different drives; --ple is an alias for --shard
(env: LLAMA_ARG_SHARD) | | `-mu, --model-url MODEL_URL` | model download url (default: unused)
(env: LLAMA_ARG_MODEL_URL) | | `-dr, --docker-repo [/][:quant]` | Docker Hub model repository. repo is optional, default to ai/. quant is optional, default to :latest.
example: gemma3
(default: unused)
(env: LLAMA_ARG_DOCKER_REPO) | | `-hf, -hfr, --hf-repo /[:quant]` | Hugging Face model repository; quant is optional, case-insensitive, default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.
mmproj is also downloaded automatically if available. to disable, add --no-mmproj
example: ggml-org/GLM-4.7-Flash-GGUF:Q4_K_M
(default: unused)
(env: LLAMA_ARG_HF_REPO) | @@ -96,8 +97,8 @@ | `-lv, --verbosity, --log-verbosity N` | Set the verbosity threshold. Messages with a higher verbosity will be ignored. Values:
- 0: generic output
- 1: error
- 2: warning
- 3: info
- 4: trace (more info)
- 5: debug
(default: 3)

(env: LLAMA_ARG_LOG_VERBOSITY) | | `--log-prefix, --no-log-prefix` | Enable prefix in log messages
(env: LLAMA_ARG_LOG_PREFIX) | | `--log-timestamps, --no-log-timestamps` | Enable timestamps in log messages
(env: LLAMA_ARG_LOG_TIMESTAMPS) | -| `--spec-draft-type-k, -ctkd, --cache-type-k-draft TYPE` | KV cache data type for K for the draft model
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K) | -| `--spec-draft-type-v, -ctvd, --cache-type-v-draft TYPE` | KV cache data type for V for the draft model
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V) | +| `--spec-draft-type-k, -ctkd, --cache-type-k-draft TYPE` | KV cache data type for K for the draft model
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K) | +| `--spec-draft-type-v, -ctvd, --cache-type-v-draft TYPE` | KV cache data type for V for the draft model
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V) | ### Sampling params @@ -178,7 +179,7 @@ | `--reasoning-effort LEVEL` | reasoning effort level given to the chat template: 'default' to keep the template default,
or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)
(env: LLAMA_ARG_REASONING_EFFORT) | | `--reasoning-budget N` | token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)
(env: LLAMA_ARG_THINK_BUDGET) | | `--reasoning-budget-message MESSAGE` | message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)
(env: LLAMA_ARG_THINK_BUDGET_MESSAGE) | -| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: template default)
compatible with certain templates having 'supports_preserve_reasoning' capability
example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking
(env: LLAMA_ARG_REASONING_PRESERVE) | +| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: enabled)
compatible with certain templates having 'supports_preserve_reasoning' capability
example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking
(env: LLAMA_ARG_REASONING_PRESERVE) | | `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)
if suffix/prefix are specified, template will be disabled
only commonly used templates are accepted (unless --jinja is set before this flag):
list of built-in templates:
bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr
(env: LLAMA_ARG_CHAT_TEMPLATE) | | `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)
if suffix/prefix are specified, template will be disabled
only commonly used templates are accepted (unless --jinja is set before this flag):
list of built-in templates:
bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr
(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) | | `--skip-chat-parsing, --no-skip-chat-parsing` | force a pure content parser, even if a Jinja template is specified; model will output everything in the content section, including any reasoning and/or tool calls (default: disabled)
(env: LLAMA_ARG_SKIP_CHAT_PARSING) | diff --git a/tools/completion/README.md b/tools/completion/README.md index 9b824f968..c50d14bae 100644 --- a/tools/completion/README.md +++ b/tools/completion/README.md @@ -133,8 +133,8 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `-kvo, --kv-offload, -nkvo, --no-kv-offload` | whether to enable KV cache offloading (default: enabled)
(env: LLAMA_ARG_KV_OFFLOAD) | | `--repack, -nr, --no-repack` | whether to enable weight repacking (default: enabled)
(env: LLAMA_ARG_REPACK) | | `--no-host` | bypass host buffer allowing extra buffers to be used
(env: LLAMA_ARG_NO_HOST) | -| `-ctk, --cache-type-k TYPE` | KV cache data type for K
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_K) | -| `-ctv, --cache-type-v TYPE` | KV cache data type for V
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_V) | +| `-ctk, --cache-type-k TYPE` | KV cache data type for K
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_K) | +| `-ctv, --cache-type-v TYPE` | KV cache data type for V
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_V) | | `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)
(env: LLAMA_ARG_DEFRAG_THOLD) | | `-np, --parallel N` | number of parallel sequences to decode (default: 1)
(env: LLAMA_ARG_N_PARALLEL) | | `--rpc SERVERS` | comma-separated list of RPC servers (host:port)
(env: LLAMA_ARG_RPC) | @@ -165,6 +165,7 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `--control-vector-scaled FNAME:SCALE,...` | add a control vector with user defined scaling SCALE
note: use comma-separated values (format: FNAME:SCALE,...) | | `--control-vector-layer-range START END` | layer range to apply the control vector(s) to, start and end inclusive | | `-m, --model FNAME` | model path to load
(env: LLAMA_ARG_MODEL) | +| `--ple, --shard FNAME,...` | extra GGUF shards for the model, comma separated; may be given more than once
paths may live anywhere, including on different drives; --ple is an alias for --shard
(env: LLAMA_ARG_SHARD) | | `-mu, --model-url MODEL_URL` | model download url (default: unused)
(env: LLAMA_ARG_MODEL_URL) | | `-dr, --docker-repo [/][:quant]` | Docker Hub model repository. repo is optional, default to ai/. quant is optional, default to :latest.
example: gemma3
(default: unused)
(env: LLAMA_ARG_DOCKER_REPO) | | `-hf, -hfr, --hf-repo /[:quant]` | Hugging Face model repository; quant is optional, case-insensitive, default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.
mmproj is also downloaded automatically if available. to disable, add --no-mmproj
example: ggml-org/GLM-4.7-Flash-GGUF:Q4_K_M
(default: unused)
(env: LLAMA_ARG_HF_REPO) | @@ -179,8 +180,8 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `-lv, --verbosity, --log-verbosity N` | Set the verbosity threshold. Messages with a higher verbosity will be ignored. Values:
- 0: generic output
- 1: error
- 2: warning
- 3: info
- 4: trace (more info)
- 5: debug
(default: 3)

(env: LLAMA_ARG_LOG_VERBOSITY) | | `--log-prefix, --no-log-prefix` | Enable prefix in log messages
(env: LLAMA_ARG_LOG_PREFIX) | | `--log-timestamps, --no-log-timestamps` | Enable timestamps in log messages
(env: LLAMA_ARG_LOG_TIMESTAMPS) | -| `--spec-draft-type-k, -ctkd, --cache-type-k-draft TYPE` | KV cache data type for K for the draft model
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K) | -| `--spec-draft-type-v, -ctvd, --cache-type-v-draft TYPE` | KV cache data type for V for the draft model
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V) | +| `--spec-draft-type-k, -ctkd, --cache-type-k-draft TYPE` | KV cache data type for K for the draft model
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K) | +| `--spec-draft-type-v, -ctvd, --cache-type-v-draft TYPE` | KV cache data type for V for the draft model
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V) | ### Sampling params @@ -256,7 +257,7 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `--reasoning-effort LEVEL` | reasoning effort level given to the chat template: 'default' to keep the template default,
or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)
(env: LLAMA_ARG_REASONING_EFFORT) | | `--reasoning-budget N` | token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)
(env: LLAMA_ARG_THINK_BUDGET) | | `--reasoning-budget-message MESSAGE` | message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)
(env: LLAMA_ARG_THINK_BUDGET_MESSAGE) | -| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: template default)
compatible with certain templates having 'supports_preserve_reasoning' capability
example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking
(env: LLAMA_ARG_REASONING_PRESERVE) | +| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: enabled)
compatible with certain templates having 'supports_preserve_reasoning' capability
example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking
(env: LLAMA_ARG_REASONING_PRESERVE) | | `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)
if suffix/prefix are specified, template will be disabled
only commonly used templates are accepted (unless --jinja is set before this flag):
list of built-in templates:
bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr
(env: LLAMA_ARG_CHAT_TEMPLATE) | | `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)
if suffix/prefix are specified, template will be disabled
only commonly used templates are accepted (unless --jinja is set before this flag):
list of built-in templates:
bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr
(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) | | `--skip-chat-parsing, --no-skip-chat-parsing` | force a pure content parser, even if a Jinja template is specified; model will output everything in the content section, including any reasoning and/or tool calls (default: disabled)
(env: LLAMA_ARG_SKIP_CHAT_PARSING) | diff --git a/tools/server/README.md b/tools/server/README.md index 3ca5d238e..a160342fe 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -68,8 +68,8 @@ For the full list of features, please refer to [server's changelog](https://gith | `-kvo, --kv-offload, -nkvo, --no-kv-offload` | whether to enable KV cache offloading (default: enabled)
(env: LLAMA_ARG_KV_OFFLOAD) | | `--repack, -nr, --no-repack` | whether to enable weight repacking (default: enabled)
(env: LLAMA_ARG_REPACK) | | `--no-host` | bypass host buffer allowing extra buffers to be used
(env: LLAMA_ARG_NO_HOST) | -| `-ctk, --cache-type-k TYPE` | KV cache data type for K
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_K) | -| `-ctv, --cache-type-v TYPE` | KV cache data type for V
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_V) | +| `-ctk, --cache-type-k TYPE` | KV cache data type for K
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_K) | +| `-ctv, --cache-type-v TYPE` | KV cache data type for V
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_V) | | `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)
(env: LLAMA_ARG_DEFRAG_THOLD) | | `--rpc SERVERS` | comma-separated list of RPC servers (host:port)
(env: LLAMA_ARG_RPC) | | `-lm, --load-mode MODE` | model loading mode (default: auto)
- auto: mmap, unless a device does not support it
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available

(env: LLAMA_ARG_LOAD_MODE) | @@ -99,6 +99,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `--control-vector-scaled FNAME:SCALE,...` | add a control vector with user defined scaling SCALE
note: use comma-separated values (format: FNAME:SCALE,...) | | `--control-vector-layer-range START END` | layer range to apply the control vector(s) to, start and end inclusive | | `-m, --model FNAME` | model path to load
(env: LLAMA_ARG_MODEL) | +| `--ple, --shard FNAME,...` | extra GGUF shards for the model, comma separated; may be given more than once
paths may live anywhere, including on different drives; --ple is an alias for --shard
(env: LLAMA_ARG_SHARD) | | `-mu, --model-url MODEL_URL` | model download url (default: unused)
(env: LLAMA_ARG_MODEL_URL) | | `-dr, --docker-repo [/][:quant]` | Docker Hub model repository. repo is optional, default to ai/. quant is optional, default to :latest.
example: gemma3
(default: unused)
(env: LLAMA_ARG_DOCKER_REPO) | | `-hf, -hfr, --hf-repo /[:quant]` | Hugging Face model repository; quant is optional, case-insensitive, default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.
mmproj is also downloaded automatically if available. to disable, add --no-mmproj
example: ggml-org/GLM-4.7-Flash-GGUF:Q4_K_M
(default: unused)
(env: LLAMA_ARG_HF_REPO) | @@ -113,8 +114,8 @@ For the full list of features, please refer to [server's changelog](https://gith | `-lv, --verbosity, --log-verbosity N` | Set the verbosity threshold. Messages with a higher verbosity will be ignored. Values:
- 0: generic output
- 1: error
- 2: warning
- 3: info
- 4: trace (more info)
- 5: debug
(default: 3)

(env: LLAMA_ARG_LOG_VERBOSITY) | | `--log-prefix, --no-log-prefix` | Enable prefix in log messages
(env: LLAMA_ARG_LOG_PREFIX) | | `--log-timestamps, --no-log-timestamps` | Enable timestamps in log messages
(env: LLAMA_ARG_LOG_TIMESTAMPS) | -| `--spec-draft-type-k, -ctkd, --cache-type-k-draft TYPE` | KV cache data type for K for the draft model
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K) | -| `--spec-draft-type-v, -ctvd, --cache-type-v-draft TYPE` | KV cache data type for V for the draft model
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V) | +| `--spec-draft-type-k, -ctkd, --cache-type-k-draft TYPE` | KV cache data type for K for the draft model
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K) | +| `--spec-draft-type-v, -ctvd, --cache-type-v-draft TYPE` | KV cache data type for V for the draft model
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V) | ### Sampling params @@ -195,16 +196,17 @@ For the full list of features, please refer to [server's changelog](https://gith | `--port PORT` | port to listen (default: 9931)
(env: LLAMA_ARG_PORT) | | `--reuse-port` | allow multiple sockets to bind to the same port (default: disabled)
(env: LLAMA_ARG_REUSE_PORT) | | `--path PATH` | path to serve static files from (default: )
(env: LLAMA_ARG_STATIC_PATH) | -| `--cors-origins ORIGINS` | comma-separated list of allowed origins for CORS (default: *)
if set to special value 'localhost', reflect the Origin header only if it is localhost
(env: LLAMA_ARG_CORS_ORIGINS) | +| `--cors-origins ORIGINS` | comma-separated list of allowed origins for CORS (default: *)
if set to special value 'localhost', only allow same-origin requests (e.g. the built-in UI)
(env: LLAMA_ARG_CORS_ORIGINS) | | `--cors-methods METHODS` | comma-separated list of allowed methods for CORS (default: GET, POST, DELETE, OPTIONS)
(env: LLAMA_ARG_CORS_METHODS) | | `--cors-headers HEADERS` | comma-separated list of allowed headers for CORS (default: *)
(env: LLAMA_ARG_CORS_HEADERS) | | `--cors-credentials, --no-cors-credentials` | whether to allow credentials for CORS (default: enabled)
note: if this is enabled and --cors-origins is set to * (default), the Origin header will be echoed back, and credentials will always be allowed
(env: LLAMA_ARG_CORS_CREDENTIALS) | | `--api-prefix PREFIX` | prefix path the server serves from, without the trailing slash (default: )
(env: LLAMA_ARG_API_PREFIX) | | `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG) | | `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG_FILE) | -| `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)
(env: LLAMA_ARG_UI_MCP_PROXY) | +| `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_UI_MCP_PROXY) | | `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)
specify "all" to enable all tools
available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_TOOLS) | | `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)
available options:
'docker:', 'podman:': spin up a new container and reuse it for all invocations, clean up on server exit
'docker-container:', 'podman-container:': use an existing container by ID, won't stop on server exit
'ssh:': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required

(env: LLAMA_ARG_TOOLS_RUNTIME) | +| `--allow-unauthenticated-tools` | allow server tools, MCP servers or MCP proxy on a non-loopback address without an API key - only for trusted networks (default: disabled)
(env: LLAMA_ARG_ALLOW_UNAUTHENTICATED_TOOLS) | | `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_CONFIG) | | `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_JSON) | | `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_AGENT) | @@ -239,7 +241,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `--reasoning-effort LEVEL` | reasoning effort level given to the chat template: 'default' to keep the template default,
or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)
(env: LLAMA_ARG_REASONING_EFFORT) | | `--reasoning-budget N` | token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)
(env: LLAMA_ARG_THINK_BUDGET) | | `--reasoning-budget-message MESSAGE` | message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)
(env: LLAMA_ARG_THINK_BUDGET_MESSAGE) | -| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: template default)
compatible with certain templates having 'supports_preserve_reasoning' capability
example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking
(env: LLAMA_ARG_REASONING_PRESERVE) | +| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: enabled)
compatible with certain templates having 'supports_preserve_reasoning' capability
example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking
(env: LLAMA_ARG_REASONING_PRESERVE) | | `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)
if suffix/prefix are specified, template will be disabled
only commonly used templates are accepted (unless --jinja is set before this flag):
list of built-in templates:
bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr
(env: LLAMA_ARG_CHAT_TEMPLATE) | | `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)
if suffix/prefix are specified, template will be disabled
only commonly used templates are accepted (unless --jinja is set before this flag):
list of built-in templates:
bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr
(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) | | `--skip-chat-parsing, --no-skip-chat-parsing` | force a pure content parser, even if a Jinja template is specified; model will output everything in the content section, including any reasoning and/or tool calls (default: disabled)
(env: LLAMA_ARG_SKIP_CHAT_PARSING) | @@ -293,6 +295,8 @@ For the full list of features, please refer to [server's changelog](https://gith | `--spec-ngram-size-n N` | the argument has been removed. use the respective --spec-ngram-*-size-n or --spec-ngram-mod-n-match | | `--spec-ngram-size-m N` | the argument has been removed. use the respective --spec-ngram-*-size-m | | `--spec-ngram-min-hits N` | the argument has been removed. use the respective --spec-ngram-*-min-hits | +| `--tts-lang FNAME` | language (ISO 639-1) for audio generation
see tts/README.md for per-model usage notes | +| `--tts-voice VOICE` | voice for audio generation (model-dependent)
see tts/README.md for per-model usage notes | | `--embd-gemma-default` | use default EmbeddingGemma model (note: can download weights from the internet) | | `--fim-qwen-1.5b-default` | use default Qwen 2.5 Coder 1.5B (note: can download weights from the internet) | | `--fim-qwen-3b-default` | use default Qwen 2.5 Coder 3B (note: can download weights from the internet) | diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index 1a344e53c..5bdece55b 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -74,16 +74,6 @@ static void log_server_request(const httplib::Request & req, const httplib::Resp SRV_DBG("response: %s\n", res.body.c_str()); } -// returns true if the Origin header value's host is localhost / 127.0.0.1 / ::1 (any port) -static bool origin_is_localhost(const std::string & origin) { - try { - const std::string host = common_http_parse_url(origin).host; - return host == "localhost" || host == "127.0.0.1" || host == "::1"; - } catch (const std::exception &) { - return false; - } -} - static std::string ascii_lower(std::string s) { std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); }); return s; @@ -159,7 +149,7 @@ static bool request_origin_is_allowed(const std::string & cors_origins, const ht return true; } if (cors_origins == "localhost") { - return origin_is_localhost(origin); + return false; // same-origin only, other localhost ports can be untrusted apps } for (const auto & allowed : string_split(cors_origins, ',')) { if (string_strip(allowed) == origin) { @@ -448,13 +438,7 @@ bool server_http_context::init_listener(const common_params & params) { // special case: echo back the Origin header to allow any origin to access the server with credentials res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); } else if (params.cors_origins == "localhost") { - // special case: only reflect the Origin header if it is a localhost origin - std::string origin = req.get_header_value("Origin"); - if (!origin.empty() && origin_is_localhost(origin)) { - res.set_header("Access-Control-Allow-Origin", origin); - } else if (!origin.empty()) { - SRV_WRN("(CORS) skip non-localhost origin: %s\n", origin.c_str()); - } + // special case: same-origin only, no CORS header needed } else { res.set_header("Access-Control-Allow-Origin", params.cors_origins); } diff --git a/tools/server/server.cpp b/tools/server/server.cpp index a0640f12b..ca3eb2dbb 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -13,6 +13,7 @@ #include "llama.h" #include "log.h" +#include #include #include #include @@ -258,6 +259,17 @@ static bool server_setup_features( SRV_WRN("%s", "security: no API key is set and CORS allows all origins (see https://github.com/ggml-org/llama.cpp/pull/25655)\n"); } + // tools can run shell commands: do not expose them to the network without auth + const bool tools_enabled = !params.server_tools.empty() || !params.mcp_servers_config.empty() || !params.mcp_servers_json.empty() || params.ui_mcp_proxy; + const bool loopback_only = std::all_of(params.hostnames.begin(), params.hostnames.end(), [](const std::string & h) { + return string_ends_with(h, ".sock") || h == "localhost" || h == "::1" || h == "[::1]" || string_starts_with(h, "127."); + }); + if (tools_enabled && !loopback_only && params.api_keys.empty() && !params.server_tools_allow_unauth) { + SRV_ERR("%s", "security: server tools, MCP servers or MCP proxy are enabled on a non-loopback address without an API key\n"); + SRV_ERR("%s", "set --api-key, bind to 127.0.0.1, or pass --allow-unauthenticated-tools on a trusted network\n"); + return false; + } + std::vector warn_names; if (role_type == SERVER_ROLE_ROUTER) { warn_names.push_back("router mode"); diff --git a/tools/server/tests/unit/test_security.py b/tools/server/tests/unit/test_security.py index 4113c1254..eca78ea9d 100644 --- a/tools/server/tests/unit/test_security.py +++ b/tools/server/tests/unit/test_security.py @@ -110,32 +110,12 @@ def test_cors_options(origin: str, cors_header: str, cors_header_value: str): @pytest.mark.parametrize("origin", [ "http://localhost", "http://localhost:8080", - "http://127.0.0.1", "http://127.0.0.1:3000", - "http://[::1]", "http://[::1]:3000", -]) -def test_cors_origins_localhost_reflects(origin: str): - global server - server = ServerPreset.router() - server.cors_origins = "localhost" - server.start() - res = server.make_request("OPTIONS", "/completions", headers={ - "Origin": origin, - "Access-Control-Request-Method": "POST", - "Access-Control-Request-Headers": "Authorization", - }) - assert res.status_code == 200 - assert res.headers["Access-Control-Allow-Origin"] == origin - - -@pytest.mark.parametrize("origin", [ - "http://web.mydomain.fr", "http://evil.com", - "http://notlocalhost", "http://localhost.evil.com", ]) -def test_cors_origins_localhost_rejects(origin: str): +def test_cors_origins_localhost_no_cross_origin_header(origin: str): global server server = ServerPreset.router() server.cors_origins = "localhost" @@ -149,26 +129,19 @@ def test_cors_origins_localhost_rejects(origin: str): assert "Access-Control-Allow-Origin" not in res.headers -def test_cors_origins_defaults_to_localhost_with_tools_enabled(): +@pytest.mark.parametrize("feature", ["server_tools", "ui_mcp_proxy"]) +def test_cors_origins_defaults_to_localhost(feature: str): global server server = ServerPreset.router() - server.server_tools = "all" + if feature == "server_tools": + server.server_tools = "all" + else: + server.ui_mcp_proxy = True server.start() - res = server.make_request("OPTIONS", "/completions", headers={ - "Origin": "http://localhost:8080", - "Access-Control-Request-Method": "POST", - "Access-Control-Request-Headers": "Authorization", - }) + res = server.make_request("GET", "/props", headers={"Origin": f"http://127.0.0.1:{server.server_port}"}) assert res.status_code == 200 - assert res.headers["Access-Control-Allow-Origin"] == "http://localhost:8080" - - res = server.make_request("OPTIONS", "/completions", headers={ - "Origin": "http://evil.com", - "Access-Control-Request-Method": "POST", - "Access-Control-Request-Headers": "Authorization", - }) - assert res.status_code == 200 - assert "Access-Control-Allow-Origin" not in res.headers + res = server.make_request("GET", "/props", headers={"Origin": "http://localhost:5173"}) + assert res.status_code == 403 def _raw_post(path: str, body: str, headers: dict) -> requests.Response: @@ -178,7 +151,8 @@ def _raw_post(path: str, body: str, headers: dict) -> requests.Response: @pytest.mark.parametrize("headers,expected_status", [ ({}, 200), # non-browser client - ({"Origin": "http://localhost:3000"}, 200), # allowed localhost origin + ({"Origin": "http://127.0.0.1:{port}"}, 200), # same-origin + ({"Origin": "http://localhost:3000"}, 403), # other localhost app ({"Origin": "http://evil.com"}, 403), # cross-origin simple request ({"Origin": "http://localhost.evil.com"}, 403), ({"Origin": "null"}, 403), # sandboxed iframe / file:// @@ -189,6 +163,7 @@ def test_cors_origins_localhost_blocks_simple_requests(headers: dict, expected_s server.api_key = None server.cors_origins = "localhost" server.start() + headers = {k: v.format(port=server.server_port) for k, v in headers.items()} res = _raw_post("/tokenize", '{"content": "hello"}', headers) assert res.status_code == expected_status @@ -292,10 +267,33 @@ def test_tools_drive_by_exec_is_blocked(): assert "csrf-guard-test" not in res.text res = _raw_post("/tools", body, {"Origin": "http://localhost:5173"}) + assert res.status_code == 403 + + res = _raw_post("/tools", body, {"Origin": f"http://127.0.0.1:{server.server_port}"}) assert res.status_code == 200 assert "csrf-guard-test" in res.text +@pytest.mark.parametrize("server_host,api_key,allow_unauth,should_start", [ + ("127.0.0.1", None, False, True), + ("0.0.0.0", None, False, False), + ("0.0.0.0", TEST_API_KEY, False, True), + ("0.0.0.0", None, True, True), +]) +def test_tools_refuse_unauthenticated_network_bind(server_host: str, api_key: str | None, allow_unauth: bool, should_start: bool): + global server + server = ServerPreset.router() + server.server_tools = "all" + server.server_host = server_host + server.api_key = api_key + server.allow_unauthenticated_tools = allow_unauth + if should_start: + server.start() + else: + with pytest.raises(RuntimeError): + server.start() + + def test_cors_proxy_only_forwards_explicit_proxy_headers(): class CaptureHeadersHandler(BaseHTTPRequestHandler): def do_GET(self): diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py index 5dd760baf..8009b4c1f 100644 --- a/tools/server/tests/utils.py +++ b/tools/server/tests/utils.py @@ -122,6 +122,7 @@ class ServerProcess: no_cache_idle_slots: bool = False log_path: str | None = None ui_mcp_proxy: bool = False + allow_unauthenticated_tools: bool = False backend_sampling: bool = False gcp_compat: bool = False server_tools: str | None = None @@ -302,6 +303,8 @@ class ServerProcess: server_args.append("--no-cache-idle-slots") if self.ui_mcp_proxy: server_args.append("--ui-mcp-proxy") + if self.allow_unauthenticated_tools: + server_args.append("--allow-unauthenticated-tools") if self.server_tools: server_args.extend(["--tools", self.server_tools]) if self.server_tools_runtime: @@ -388,6 +391,8 @@ class ServerProcess: def make_url(self, path: str, host: str | None = None) -> str: if host is None: host = self.server_host.split(",")[0].strip() + # a wildcard bind cannot be connected to on Windows + host = {"0.0.0.0": "127.0.0.1", "::": "[::1]"}.get(host, host) return f"http://{host}:{self.server_port}{path}" def make_request(