diff --git a/tools/llama-bench/README.md b/tools/llama-bench/README.md index 82e846108..376007212 100644 --- a/tools/llama-bench/README.md +++ b/tools/llama-bench/README.md @@ -35,7 +35,7 @@ options: --progress print test progress indicators --no-warmup skip warmup runs before benchmarking -fitt, --fit-target fit model to device memory with this margin per device in MiB (default: off) - -fitc, --fit-ctx minimum ctx size for --fit-target (default: 4096) + -fitc, --fit-ctx minimum ctx size for --fit-target (default: 0) -rpc, --rpc register RPC devices (comma separated) test parameters: diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp index 8361e6ae6..96ae4870f 100644 --- a/tools/llama-bench/llama-bench.cpp +++ b/tools/llama-bench/llama-bench.cpp @@ -441,7 +441,7 @@ static void print_usage(int /* argc */, char ** argv) { printf(" --progress print test progress indicators\n"); printf(" --no-warmup skip warmup runs before benchmarking\n"); printf(" -fitt, --fit-target fit model to device memory with this margin per device in MiB (default: off)\n"); - printf(" -fitc, --fit-ctx minimum ctx size for --fit-target (default: 4096)\n"); + printf(" -fitc, --fit-ctx minimum ctx size for --fit-target (default: 0)\n"); if (llama_supports_rpc()) { printf(" -rpc, --rpc register RPC devices (comma separated)\n"); } @@ -2348,7 +2348,8 @@ int llama_bench(int argc, char ** argv) { std::vector margins(llama_max_devices(), inst.fit_target * 1024 * 1024); - uint32_t n_ctx_needed = inst.n_prompt + inst.n_gen + inst.n_depth; + // fit at least the requested minimum context size, not just the tokens the benchmark processes + uint32_t n_ctx_needed = std::max(inst.n_prompt + inst.n_gen + inst.n_depth, inst.fit_min_ctx); cparams.n_ctx = std::max(cparams.n_ctx, n_ctx_needed); common_fit_params(inst.model.c_str(), &mparams, &cparams,