Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tokenize : drop --stdin mutual-exclusion check (#25672)

Browse Source 
match cli and completion, which don't enforce it
This commit is contained in:
Adrien Gallouët
2026-07-15 18:41:51 +02:00
committed by GitHub
parent c3d47e696b
commit aff6eb6e75
2 changed files with 3 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+1 -1
View File
@@ -2932,7 +2932,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
).set_examples({LLAMA_EXAMPLE_TOKENIZE})); ).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg( add_opt(common_arg(
{"--stdin"}, {"--stdin"},
string_format("read the prompt from stdin (mutually exclusive with -f/--file and -p/--prompt) (default: %s)", params.tokenize_stdin ? "true" : "false"), string_format("read the prompt from stdin (takes precedence over -f/--file and -p/--prompt) (default: %s)", params.tokenize_stdin ? "true" : "false"),
[](common_params & params) { [](common_params & params) {
params.tokenize_stdin = true; params.tokenize_stdin = true;
} }
tools/tokenize/tokenize.cpp
+2 -10
View File
@@ -103,19 +103,11 @@ int main(int argc, char ** argv) {
return 1; return 1;
} }
// which prompt source was requested? // -f and -p both land in params.prompt; -f also sets prompt_file. -f and -p
// -p/--prompt and -f/--file both end up in params.prompt (common's -f also // resolve like the other tools (no mutual exclusion), --stdin takes precedence.
// strips a single trailing newline), but -f additionally records the path
// in params.prompt_file, so we use that to tell them apart.
const bool use_stdin = params.tokenize_stdin; const bool use_stdin = params.tokenize_stdin;
const bool use_file = !params.prompt_file.empty(); const bool use_file = !params.prompt_file.empty();
// sanity check: --stdin is mutually exclusive with -f/--file and -p/--prompt
if (use_stdin && (use_file || !params.prompt.empty())) {
LOG_ERR("error: --stdin is mutually exclusive with --file and --prompt\n");
return 1;
}
// must have some prompt // must have some prompt
if (!use_stdin && !use_file && params.prompt.empty()) { if (!use_stdin && !use_file && params.prompt.empty()) {
LOG_ERR("error: must specify one of: --stdin, --file or --prompt\n"); LOG_ERR("error: must specify one of: --stdin, --file or --prompt\n");