diff --git a/common/arg.cpp b/common/arg.cpp index 29f6df32f..8b302d5d7 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1717,10 +1717,11 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_env("LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT").set_examples({LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( {"-cram", "--cache-ram"}, "N", - string_format("set the maximum cache size in MiB (default: %d, -1 - no limit, 0 - disable)" + string_format("set the maximum cache size in MiB (default: %d, 0 for decision models, -1 - no limit, 0 - disable)" "[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)", params.cache_ram_mib), [](common_params & params, int value) { params.cache_ram_mib = value; + params.cache_ram_explicit = true; } ).set_env("LLAMA_ARG_CACHE_RAM").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( diff --git a/common/common.h b/common/common.h index 2048d6a79..2cd6e2ec9 100644 --- a/common/common.h +++ b/common/common.h @@ -640,6 +640,7 @@ struct common_params { int32_t kv_unified_per_slot = 0; // max context per parallel slot; 0 = unset int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc. + bool cache_ram_explicit = false; // set by --cache-ram, decision models disable the cache otherwise std::string public_path = ""; // NOLINT std::string api_prefix = ""; // NOLINT diff --git a/tools/cli/README.md b/tools/cli/README.md index 163774119..be8fb09d2 100644 --- a/tools/cli/README.md +++ b/tools/cli/README.md @@ -148,7 +148,7 @@ | `--display-prompt, --no-display-prompt` | whether to print prompt at generation (default: true) | | `-co, --color [on\|off\|auto]` | Colorize output to distinguish prompt and user input from generations ('on', 'off', or 'auto', default: 'auto')
'auto' enables colors when output is to a terminal | | `-ctxcp, --ctx-checkpoints, --swa-checkpoints N` | max number of context checkpoints to create per slot (default: 32)[(more info)](https://github.com/ggml-org/llama.cpp/pull/15293)
(env: LLAMA_ARG_CTX_CHECKPOINTS) | -| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)
(env: LLAMA_ARG_CACHE_RAM) | +| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, 0 for decision models, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)
(env: LLAMA_ARG_CACHE_RAM) | | `--context-shift, --no-context-shift` | whether to use context shift on infinite text generation (default: disabled)
(env: LLAMA_ARG_CONTEXT_SHIFT) | | `-sys, --system-prompt PROMPT` | system prompt to use with model (if applicable, depending on chat template) | | `--show-timings, --no-show-timings` | whether to show timing information after each response (default: true)
(env: LLAMA_ARG_SHOW_TIMINGS) | diff --git a/tools/server/README.md b/tools/server/README.md index 37c41f1bb..4e4472ec2 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -165,7 +165,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `--kv-unified-per-slot N` | context limit per parallel slot (default: unset, behavior unchanged).
when set without -c/--ctx-size, the shared KV pool is sized to n_parallel*N
(env: LLAMA_ARG_KV_UNIFIED_PER_SLOT) | | `-ctxcp, --ctx-checkpoints, --swa-checkpoints N` | max number of context checkpoints to create per slot (default: 32)[(more info)](https://github.com/ggml-org/llama.cpp/pull/15293)
(env: LLAMA_ARG_CTX_CHECKPOINTS) | | `-cms, --checkpoint-min-step N` | minimum spacing between context checkpoints in tokens (default: 8192, 0 = no minimum)
(env: LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT) | -| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)
(env: LLAMA_ARG_CACHE_RAM) | +| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, 0 for decision models, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)
(env: LLAMA_ARG_CACHE_RAM) | | `-kvu, --kv-unified, -no-kvu, --no-kv-unified` | use single unified KV buffer shared across all sequences (default: enabled if number of slots is auto)
(env: LLAMA_ARG_KV_UNIFIED) | | `--cache-idle-slots, --no-cache-idle-slots` | save idle slots to the prompt cache on new task, and clear them when using unified KV (default: enabled, requires cache-ram)
(env: LLAMA_ARG_CACHE_IDLE_SLOTS) | | `--context-shift, --no-context-shift` | whether to use context shift on infinite text generation (default: disabled)
(env: LLAMA_ARG_CONTEXT_SHIFT) | diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 60c563345..d5b4a7029 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -594,7 +594,8 @@ struct server_slot { state = SLOT_STATE_IDLE; // do not keep context of the child slots - the parent's context is enough - if (task->is_child()) { + // nor of the stateless slots - no task ever reuses it + if (task->is_child() || task->is_stateless()) { prompt_clear(); } @@ -1489,6 +1490,13 @@ private: batch.init(ctx_tgt, std::max(n_batch, params_base.n_parallel), n_embd); } + // decision requests rarely repeat a state, the prompt cache is opt-in for decision models + if (decision.type != COMMON_DECISION_TYPE_NONE && !params_base.cache_ram_explicit) { + SRV_INF("%s", "decision model: prompt cache disabled by default, use `--cache-ram N` to enable it\n"); + params_base.cache_ram_mib = 0; + params_base.cache_idle_slots = false; + } + if (params_base.cache_ram_mib != 0) { if (params_base.cache_ram_mib < 0) { SRV_TRC("prompt cache is enabled, size limit: %s\n", "no limit"); @@ -1782,8 +1790,8 @@ private: if (ret) { update_cache = update_cache && prompt_cache; - // cache prompts only for completion tasks - update_cache = update_cache && task.type == SERVER_TASK_TYPE_COMPLETION; + // cache prompts only for the tasks that reuse them + update_cache = update_cache && !task.is_stateless(); if (update_cache) { SRV_TRC("%s", "updating prompt cache\n"); @@ -3597,9 +3605,7 @@ private: return; } - const bool is_stateless_task = slot.task->type == SERVER_TASK_TYPE_EMBEDDING || slot.task->type == SERVER_TASK_TYPE_RERANK; - - if (slot.task->params.cache_prompt && !is_stateless_task) { + if (slot.task->params.cache_prompt && !slot.task->is_stateless()) { // reuse any previously computed tokens that are common with the new prompt n_past = slot.prompt.tokens.get_common_prefix(input_tokens); diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 50926ec3c..e7bb04398 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -300,6 +300,11 @@ struct server_task { bool is_child() const { return id_parent != -1; } + + // embedding and rerank tasks always process their whole prompt, their state is never reused + bool is_stateless() const { + return type == SERVER_TASK_TYPE_EMBEDDING || type == SERVER_TASK_TYPE_RERANK; + } }; struct result_prompt_progress { diff --git a/tools/server/tests/unit/test_rerank.py b/tools/server/tests/unit/test_rerank.py index ded826710..53f15af09 100644 --- a/tools/server/tests/unit/test_rerank.py +++ b/tools/server/tests/unit/test_rerank.py @@ -41,6 +41,31 @@ def test_rerank(): assert least_relevant["index"] == 3 +@pytest.mark.parametrize("kv_unified", [False, True]) +def test_rerank_skips_prompt_cache(kv_unified, tmp_path): + global server + server.n_slots = 2 + server.cache_ram = 100 + server.kv_unified = kv_unified + server.debug = True + server.log_path = str(tmp_path / "server.log") + server.start() + + for i in range(4): + res = server.make_request("POST", "/rerank", data={ + "query": f"query {i}", + "documents": TEST_DOCUMENTS[i:], + }) + assert res.status_code == 200 + + server.stop() + + with open(server.log_path) as f: + log = f.read() + assert "__TEST_TAG_CACHE_IDLE_SLOTS_ENABLED__" in log + assert "__TEST_TAG_CACHE_IDLE_SLOT__" not in log + + def test_rerank_tei_format(): global server server.start() diff --git a/tools/server/tests/unit/test_systemone.py b/tools/server/tests/unit/test_systemone.py index d9d988410..61c2ab098 100644 --- a/tools/server/tests/unit/test_systemone.py +++ b/tools/server/tests/unit/test_systemone.py @@ -158,6 +158,32 @@ def test_systemone_shared_prompt(): assert abs(res.body["answers"]["angry"]["noul"] - res_single.body["answers"]["angry"]["noul"]) < 0.01 +@pytest.mark.parametrize("cache_ram", [None, 100]) +def test_systemone_prompt_cache(cache_ram, tmp_path): + global server + server.n_slots = 1 + server.cache_ram = cache_ram + server.debug = True + server.log_path = str(tmp_path / "server.log") + server.start() + + # the third request comes back to the state of the first one + for state in ["Ticket A: " + TEST_STATE, "Ticket B: " + TEST_STATE * 2, "Ticket A: " + TEST_STATE]: + res = server.make_request("POST", "/v1/systemone", data={ + "state": state, + "questions": TEST_QUESTIONS, + }) + assert res.status_code == 200 + + server.stop() + + with open(server.log_path) as f: + log = f.read() + # the prompt cache is opt-in for decision models, and restores their states once enabled + assert ("__TEST_TAG_CACHE_IDLE_SLOTS_ENABLED__" in log) == (cache_ram is not None) + assert ("found better prompt" in log) == (cache_ram is not None) + + def test_systemone_images(): global server server = ServerPreset.tinyopenjev()