diff --git a/common/arg.cpp b/common/arg.cpp
index 29f6df32f..8b302d5d7 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -1717,10 +1717,11 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
).set_env("LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT").set_examples({LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"-cram", "--cache-ram"}, "N",
- string_format("set the maximum cache size in MiB (default: %d, -1 - no limit, 0 - disable)"
+ string_format("set the maximum cache size in MiB (default: %d, 0 for decision models, -1 - no limit, 0 - disable)"
"[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)", params.cache_ram_mib),
[](common_params & params, int value) {
params.cache_ram_mib = value;
+ params.cache_ram_explicit = true;
}
).set_env("LLAMA_ARG_CACHE_RAM").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
add_opt(common_arg(
diff --git a/common/common.h b/common/common.h
index 2048d6a79..2cd6e2ec9 100644
--- a/common/common.h
+++ b/common/common.h
@@ -640,6 +640,7 @@ struct common_params {
int32_t kv_unified_per_slot = 0; // max context per parallel slot; 0 = unset
int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints
int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
+ bool cache_ram_explicit = false; // set by --cache-ram, decision models disable the cache otherwise
std::string public_path = ""; // NOLINT
std::string api_prefix = ""; // NOLINT
diff --git a/tools/cli/README.md b/tools/cli/README.md
index 163774119..be8fb09d2 100644
--- a/tools/cli/README.md
+++ b/tools/cli/README.md
@@ -148,7 +148,7 @@
| `--display-prompt, --no-display-prompt` | whether to print prompt at generation (default: true) |
| `-co, --color [on\|off\|auto]` | Colorize output to distinguish prompt and user input from generations ('on', 'off', or 'auto', default: 'auto')
'auto' enables colors when output is to a terminal |
| `-ctxcp, --ctx-checkpoints, --swa-checkpoints N` | max number of context checkpoints to create per slot (default: 32)[(more info)](https://github.com/ggml-org/llama.cpp/pull/15293)
(env: LLAMA_ARG_CTX_CHECKPOINTS) |
-| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)
(env: LLAMA_ARG_CACHE_RAM) |
+| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, 0 for decision models, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)
(env: LLAMA_ARG_CACHE_RAM) |
| `--context-shift, --no-context-shift` | whether to use context shift on infinite text generation (default: disabled)
(env: LLAMA_ARG_CONTEXT_SHIFT) |
| `-sys, --system-prompt PROMPT` | system prompt to use with model (if applicable, depending on chat template) |
| `--show-timings, --no-show-timings` | whether to show timing information after each response (default: true)
(env: LLAMA_ARG_SHOW_TIMINGS) |
diff --git a/tools/server/README.md b/tools/server/README.md
index 37c41f1bb..4e4472ec2 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -165,7 +165,7 @@ For the full list of features, please refer to [server's changelog](https://gith
| `--kv-unified-per-slot N` | context limit per parallel slot (default: unset, behavior unchanged).
when set without -c/--ctx-size, the shared KV pool is sized to n_parallel*N
(env: LLAMA_ARG_KV_UNIFIED_PER_SLOT) |
| `-ctxcp, --ctx-checkpoints, --swa-checkpoints N` | max number of context checkpoints to create per slot (default: 32)[(more info)](https://github.com/ggml-org/llama.cpp/pull/15293)
(env: LLAMA_ARG_CTX_CHECKPOINTS) |
| `-cms, --checkpoint-min-step N` | minimum spacing between context checkpoints in tokens (default: 8192, 0 = no minimum)
(env: LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT) |
-| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)
(env: LLAMA_ARG_CACHE_RAM) |
+| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, 0 for decision models, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)
(env: LLAMA_ARG_CACHE_RAM) |
| `-kvu, --kv-unified, -no-kvu, --no-kv-unified` | use single unified KV buffer shared across all sequences (default: enabled if number of slots is auto)
(env: LLAMA_ARG_KV_UNIFIED) |
| `--cache-idle-slots, --no-cache-idle-slots` | save idle slots to the prompt cache on new task, and clear them when using unified KV (default: enabled, requires cache-ram)
(env: LLAMA_ARG_CACHE_IDLE_SLOTS) |
| `--context-shift, --no-context-shift` | whether to use context shift on infinite text generation (default: disabled)
(env: LLAMA_ARG_CONTEXT_SHIFT) |
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 60c563345..d5b4a7029 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -594,7 +594,8 @@ struct server_slot {
state = SLOT_STATE_IDLE;
// do not keep context of the child slots - the parent's context is enough
- if (task->is_child()) {
+ // nor of the stateless slots - no task ever reuses it
+ if (task->is_child() || task->is_stateless()) {
prompt_clear();
}
@@ -1489,6 +1490,13 @@ private:
batch.init(ctx_tgt, std::max(n_batch, params_base.n_parallel), n_embd);
}
+ // decision requests rarely repeat a state, the prompt cache is opt-in for decision models
+ if (decision.type != COMMON_DECISION_TYPE_NONE && !params_base.cache_ram_explicit) {
+ SRV_INF("%s", "decision model: prompt cache disabled by default, use `--cache-ram N` to enable it\n");
+ params_base.cache_ram_mib = 0;
+ params_base.cache_idle_slots = false;
+ }
+
if (params_base.cache_ram_mib != 0) {
if (params_base.cache_ram_mib < 0) {
SRV_TRC("prompt cache is enabled, size limit: %s\n", "no limit");
@@ -1782,8 +1790,8 @@ private:
if (ret) {
update_cache = update_cache && prompt_cache;
- // cache prompts only for completion tasks
- update_cache = update_cache && task.type == SERVER_TASK_TYPE_COMPLETION;
+ // cache prompts only for the tasks that reuse them
+ update_cache = update_cache && !task.is_stateless();
if (update_cache) {
SRV_TRC("%s", "updating prompt cache\n");
@@ -3597,9 +3605,7 @@ private:
return;
}
- const bool is_stateless_task = slot.task->type == SERVER_TASK_TYPE_EMBEDDING || slot.task->type == SERVER_TASK_TYPE_RERANK;
-
- if (slot.task->params.cache_prompt && !is_stateless_task) {
+ if (slot.task->params.cache_prompt && !slot.task->is_stateless()) {
// reuse any previously computed tokens that are common with the new prompt
n_past = slot.prompt.tokens.get_common_prefix(input_tokens);
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index 50926ec3c..e7bb04398 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -300,6 +300,11 @@ struct server_task {
bool is_child() const {
return id_parent != -1;
}
+
+ // embedding and rerank tasks always process their whole prompt, their state is never reused
+ bool is_stateless() const {
+ return type == SERVER_TASK_TYPE_EMBEDDING || type == SERVER_TASK_TYPE_RERANK;
+ }
};
struct result_prompt_progress {
diff --git a/tools/server/tests/unit/test_rerank.py b/tools/server/tests/unit/test_rerank.py
index ded826710..53f15af09 100644
--- a/tools/server/tests/unit/test_rerank.py
+++ b/tools/server/tests/unit/test_rerank.py
@@ -41,6 +41,31 @@ def test_rerank():
assert least_relevant["index"] == 3
+@pytest.mark.parametrize("kv_unified", [False, True])
+def test_rerank_skips_prompt_cache(kv_unified, tmp_path):
+ global server
+ server.n_slots = 2
+ server.cache_ram = 100
+ server.kv_unified = kv_unified
+ server.debug = True
+ server.log_path = str(tmp_path / "server.log")
+ server.start()
+
+ for i in range(4):
+ res = server.make_request("POST", "/rerank", data={
+ "query": f"query {i}",
+ "documents": TEST_DOCUMENTS[i:],
+ })
+ assert res.status_code == 200
+
+ server.stop()
+
+ with open(server.log_path) as f:
+ log = f.read()
+ assert "__TEST_TAG_CACHE_IDLE_SLOTS_ENABLED__" in log
+ assert "__TEST_TAG_CACHE_IDLE_SLOT__" not in log
+
+
def test_rerank_tei_format():
global server
server.start()
diff --git a/tools/server/tests/unit/test_systemone.py b/tools/server/tests/unit/test_systemone.py
index d9d988410..61c2ab098 100644
--- a/tools/server/tests/unit/test_systemone.py
+++ b/tools/server/tests/unit/test_systemone.py
@@ -158,6 +158,32 @@ def test_systemone_shared_prompt():
assert abs(res.body["answers"]["angry"]["noul"] - res_single.body["answers"]["angry"]["noul"]) < 0.01
+@pytest.mark.parametrize("cache_ram", [None, 100])
+def test_systemone_prompt_cache(cache_ram, tmp_path):
+ global server
+ server.n_slots = 1
+ server.cache_ram = cache_ram
+ server.debug = True
+ server.log_path = str(tmp_path / "server.log")
+ server.start()
+
+ # the third request comes back to the state of the first one
+ for state in ["Ticket A: " + TEST_STATE, "Ticket B: " + TEST_STATE * 2, "Ticket A: " + TEST_STATE]:
+ res = server.make_request("POST", "/v1/systemone", data={
+ "state": state,
+ "questions": TEST_QUESTIONS,
+ })
+ assert res.status_code == 200
+
+ server.stop()
+
+ with open(server.log_path) as f:
+ log = f.read()
+ # the prompt cache is opt-in for decision models, and restores their states once enabled
+ assert ("__TEST_TAG_CACHE_IDLE_SLOTS_ENABLED__" in log) == (cache_ram is not None)
+ assert ("found better prompt" in log) == (cache_ram is not None)
+
+
def test_systemone_images():
global server
server = ServerPreset.tinyopenjev()