Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

server : add LLAMA_SERVER_SLOTS_N_DIFF (#27600)

Browse Source
This commit is contained in:
Georgi Gerganov
2026-08-23 15:55:51 +03:00
committed by GitHub
parent ba8e0eddfb
commit e8eed4525a
1 changed files with 15 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
+15 -4
View File
@@ -858,8 +858,10 @@ private:
// slots / clients // slots / clients
std::vector<server_slot> slots; std::vector<server_slot> slots;
int trace = 0; int trace = 0; // env: LLAMA_TRACE
int slots_debug = 0; int slots_debug = 0; // env: LLAMA_SERVER_SLOTS_DEBUG
int slots_n_diff = 0; // env: LLAMA_SERVER_SLOTS_N_DIFF
int n_empty_consecutive = 0; int n_empty_consecutive = 0;
std::unique_ptr<server_prompt_cache> prompt_cache; std::unique_ptr<server_prompt_cache> prompt_cache;
@@ -1247,6 +1249,15 @@ private:
} }
} }
{
const char * LLAMA_SERVER_SLOTS_N_DIFF = getenv("LLAMA_SERVER_SLOTS_N_DIFF");
slots_n_diff = LLAMA_SERVER_SLOTS_N_DIFF ? atoi(LLAMA_SERVER_SLOTS_N_DIFF) : 0;
if (slots_n_diff) {
SRV_WRN("LLAMA_SERVER_SLOTS_N_DIFF = %d\n", slots_n_diff);
}
}
// the update_slots() logic will always submit a maximum of n_batch or n_parallel tokens // the update_slots() logic will always submit a maximum of n_batch or n_parallel tokens
// note that n_batch can be > n_ctx (e.g. for non-causal attention models such as BERT where the KV cache is not used) // note that n_batch can be > n_ctx (e.g. for non-causal attention models such as BERT where the KV cache is not used)
{ {
@@ -3179,8 +3190,8 @@ private:
// when the prompt prefix does not match, print the tokens around the mismatch // when the prompt prefix does not match, print the tokens around the mismatch
// this is useful for debugging prompt caching // this is useful for debugging prompt caching
if (slots_debug) { if (slots_debug) {
const int np0 = std::max<int>(n_past - 4, 0); const int np0 = std::max<int>(n_past - slots_n_diff, 0);
const int np1 = std::min<int>(n_past + 6, std::min(slot.prompt.tokens.size(), slot.task->tokens.size())); const int np1 = std::min<int>(n_past + slots_n_diff + 2, std::min(slot.prompt.tokens.size(), slot.task->tokens.size()));
std::stringstream ss0; std::stringstream ss0;
std::stringstream ss1; std::stringstream ss1;