server: refactor, add server_slot_sched, prompt running hash (#32154)
* server: refactor, add server_slot_sched, prompt running hash * use sha256 hash
This commit is contained in:
1 parent
58a4a6b49c
commit
429460dc0e
11 files changed
+790
-474
No files matched your search
@@ -1729,226 +1729,3 @@ json server_task_result_get_lora::to_json() {
|
||||
json server_task_result_apply_lora::to_json() {
|
||||
return json {{ "success", true }};
|
||||
}
|
||||
|
||||
//
|
||||
// server_prompt_cache
|
||||
//
|
||||
size_t server_prompt_cache::size() const {
|
||||
size_t res = 0;
|
||||
|
||||
for (const auto & state : states) {
|
||||
res += state.size();
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
size_t server_prompt_cache::n_tokens() const {
|
||||
size_t res = 0;
|
||||
|
||||
for (const auto & state : states) {
|
||||
res += state.prompt.n_tokens();
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & prompt, size_t state_size_tgt, size_t state_size_dft) {
|
||||
// first check if the current state is contained fully in the cache
|
||||
for (auto it = states.begin(); it != states.end(); ++it) {
|
||||
const int cur_lcp_len = it->prompt.tokens.get_common_prefix(prompt.tokens);
|
||||
|
||||
if (cur_lcp_len == (int) prompt.tokens.size()) {
|
||||
SRV_TRC("%s", " - prompt is already in the cache, skipping\n");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
// calculate checkpoints size to see if it will fit with the prompt
|
||||
size_t checkpoints_size = 0;
|
||||
for (const auto & ckpt : prompt.checkpoints) {
|
||||
checkpoints_size += ckpt->size();
|
||||
}
|
||||
|
||||
const size_t state_size_new = state_size_tgt + state_size_dft + checkpoints_size;
|
||||
|
||||
// skip over-limit entries to avoid disturbing the cache
|
||||
if (limit_size > 0 && state_size_new > limit_size) {
|
||||
SRV_WRN(" - prompt state size %.3f MiB exceeds cache size limit %.3f MiB, skipping\n",
|
||||
state_size_new / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0));
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// remove any cached prompts that are fully contained in the current prompt
|
||||
for (auto it = states.begin(); it != states.end();) {
|
||||
const int len = it->prompt.tokens.get_common_prefix(prompt.tokens);
|
||||
|
||||
if (len == (int) it->prompt.tokens.size()) {
|
||||
SRV_TRC(" - removing obsolete cached prompt with length %d\n", len);
|
||||
|
||||
it = states.erase(it);
|
||||
} else {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
|
||||
if (limit_size > 0) {
|
||||
// make room before allocating the new vectors to avoid breaching the limit
|
||||
while (!states.empty() && size() + state_size_new > limit_size) {
|
||||
SRV_WRN(" - making room for prompt cache entry, removing oldest entry (size = %.3f MiB)\n",
|
||||
states.front().size() / (1024.0 * 1024.0));
|
||||
|
||||
states.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
common_state_data state_data_tgt;
|
||||
common_state_data state_data_dft;
|
||||
|
||||
// check if we can allocate enough memory for the new state
|
||||
try {
|
||||
state_data_tgt.resize(state_size_tgt);
|
||||
state_data_dft.resize(state_size_dft);
|
||||
} catch (const std::bad_alloc & e) {
|
||||
SRV_ERR("failed to allocate memory for prompt cache state: %s\n", e.what());
|
||||
|
||||
limit_size = std::max<size_t>(1, 0.4*size());
|
||||
|
||||
SRV_WRN(" - cache size limit reduced to %.3f MiB\n", limit_size / (1024.0 * 1024.0));
|
||||
|
||||
update();
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
states.push_back({
|
||||
/*.prompt =*/ {
|
||||
/*.tokens =*/ prompt.tokens.clone(),
|
||||
/*.checkpoints =*/ prompt.checkpoints,
|
||||
},
|
||||
/*.data =*/ {
|
||||
/*.main =*/ std::move(state_data_tgt),
|
||||
/*.drft =*/ std::move(state_data_dft),
|
||||
},
|
||||
});
|
||||
|
||||
return &states.back();
|
||||
}
|
||||
|
||||
bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) {
|
||||
int lcp_best = prompt.tokens.get_common_prefix(tokens_new);
|
||||
|
||||
float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins
|
||||
float f_sim_best = float(lcp_best) / tokens_new.size();
|
||||
|
||||
SRV_TRC(" - looking for better prompt, base f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);
|
||||
|
||||
auto it_best = states.end();
|
||||
|
||||
// find the most similar cached prompt, that would also preserve the most context
|
||||
for (auto it = states.begin(); it != states.end(); ++it) {
|
||||
const int lcp_cur = it->prompt.tokens.get_common_prefix(tokens_new);
|
||||
|
||||
const float f_keep_cur = float(lcp_cur) / it->prompt.tokens.size();
|
||||
const float f_sim_cur = float(lcp_cur) / tokens_new.size();
|
||||
|
||||
SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur);
|
||||
|
||||
// don't trash large prompts
|
||||
if (f_keep_cur < 0.25f) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) {
|
||||
f_keep_best = f_keep_cur;
|
||||
f_sim_best = f_sim_cur;
|
||||
lcp_best = lcp_cur;
|
||||
|
||||
it_best = it;
|
||||
}
|
||||
}
|
||||
|
||||
if (it_best != states.end()) {
|
||||
SRV_TRC(" - found better prompt with f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);
|
||||
|
||||
{
|
||||
auto & data = it_best->data.main;
|
||||
|
||||
const size_t size = data.size();
|
||||
const size_t n = llama_state_seq_set_data_ext(ctx_tgt, data.data(), size, id_slot, 0);
|
||||
if (n != size) {
|
||||
SRV_ERR("failed to restore state with size %zu\n", size);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
data.clear();
|
||||
data.shrink_to_fit();
|
||||
}
|
||||
|
||||
{
|
||||
auto & data = it_best->data.drft;
|
||||
|
||||
if (!data.empty()) {
|
||||
GGML_ASSERT(ctx_dft);
|
||||
|
||||
const size_t size = data.size();
|
||||
const size_t n = llama_state_seq_set_data_ext(ctx_dft, data.data(), size, id_slot, 0);
|
||||
if (n != size) {
|
||||
SRV_WRN("failed to restore state with size %zu\n", size);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
data.clear();
|
||||
data.shrink_to_fit();
|
||||
}
|
||||
}
|
||||
|
||||
SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_RESTORE__\n");
|
||||
|
||||
// the whole cached prompt is a prefix of the new one
|
||||
if (lcp_best == (int) it_best->prompt.tokens.size()) {
|
||||
SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_FULL_RESTORE__\n");
|
||||
}
|
||||
|
||||
prompt = std::move(it_best->prompt);
|
||||
|
||||
states.erase(it_best);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void server_prompt_cache::update() {
|
||||
if (limit_size > 0) {
|
||||
while (!states.empty() && size() > limit_size) {
|
||||
SRV_WRN(" - cache size limit reached, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0));
|
||||
|
||||
states.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
// average size per token
|
||||
const float size_per_token = std::max<float>(1.0f, float(size()) / (std::max<size_t>(1, n_tokens())));
|
||||
|
||||
// dynamically increase the token limit if it can fit in the memory limit
|
||||
const size_t limit_tokens_cur = limit_size > 0 ? std::max<size_t>(limit_tokens, limit_size/size_per_token) : limit_tokens;
|
||||
|
||||
if (limit_tokens > 0) {
|
||||
while (!states.empty() && n_tokens() > limit_tokens_cur) {
|
||||
SRV_WRN(" - cache token limit (%zu, est: %zu) reached, removing oldest entry (size = %.3f MiB)\n",
|
||||
limit_tokens, limit_tokens_cur, states.front().size() / (1024.0 * 1024.0));
|
||||
|
||||
states.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
SRV_TRC(" - cache state: %zu prompts, %.3f MiB (limits: %.3f MiB, %zu tokens, %zu est)\n",
|
||||
states.size(), size() / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0), limit_tokens, limit_tokens_cur);
|
||||
|
||||
for (const auto & state : states) {
|
||||
SRV_TRC(" - prompt %p: %7d tokens, checkpoints: %2zu, %9.3f MiB\n",
|
||||
(const void *)&state, state.prompt.n_tokens(), state.prompt.checkpoints.size(), state.size() / (1024.0 * 1024.0));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user