The deploy of cb48bf224 with --cache-disk-path was rolled back: the first
request to the production 35B aborted in disk_cache_load().
server_tokens::get_tokens() has GGML_ASSERT(!has_mtmd), and has_mtmd is true
whenever an mmproj is loaded, even for a text-only prompt. Production's 35B
and gemma-26B always load one (-hf pulls it). disk_cache_collect() and
server_prompt_disk_cache::save() made the same call, so the shutdown save
would have aborted too. Every test had used --no-mmproj, which hid it.
disk_cache_text() takes the list from get_text_tokens(). A prompt is
text-only when that list is as long as size(), because media positions hold
LLAMA_TOKEN_NULL. So a text-only prompt is saved and restored with an mmproj
loaded, which is the main case (OpenWebUI chats on the 35B and gemma), and a
prompt with media is never saved or restored. When the slot itself holds
media, its reuse estimate is server_tokens::get_common_prefix(). These four
calls were the only get_tokens()/mtmd-unsafe accesses in the feature.
fork-validate --diskcache with the new mmproj cases, beside production:
15 PASS, 0 FAIL, GPU legs included. With the 0.8B's mmproj loaded, a text
chat is restored: prompt_n 58 of 596, output identical to the reference.
An image request is answered "red" with no restore, the text chat after it
is restored, and a SIGTERM with media in the slot copies no state.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
886 lines
31 KiB
C++
886 lines
31 KiB
C++
#include "server-slot-sched.h"
|
|
|
|
#include "common.h"
|
|
#include "llama.h"
|
|
#include "log.h"
|
|
|
|
#include <algorithm>
|
|
#include <cerrno>
|
|
#include <chrono>
|
|
#include <cstdio>
|
|
#include <cstring>
|
|
#include <filesystem>
|
|
|
|
#if defined(__linux__)
|
|
#include <fcntl.h>
|
|
#include <unistd.h>
|
|
#endif
|
|
|
|
//
|
|
// server_prompt
|
|
//
|
|
|
|
server_prompt_reuse server_prompt::get_reuse(int32_t n_prefix, bool has_new_tokens, int32_t n_swa, llama_pos pos_min) const {
|
|
server_prompt_reuse res;
|
|
res.n_past = n_prefix;
|
|
res.pos_next = tokens.pos_next(n_prefix);
|
|
|
|
// the first evaluated token attends to the positions > pos_eval - n_swa, so a SWA state
|
|
// starting at pos_eval - n_swa + 1 is enough, while a state without SWA must end before pos_eval
|
|
const llama_pos pos_eval = res.pos_next - (has_new_tokens ? 0 : 1);
|
|
|
|
// a state or a checkpoint is useful when its pos_min is below this threshold
|
|
const auto pos_min_thold = std::max(0, n_swa > 0 ? pos_eval - n_swa + 2 : pos_eval);
|
|
|
|
if (pos_min < pos_min_thold) {
|
|
return res;
|
|
}
|
|
|
|
// search for a context checkpoint
|
|
const auto it = std::find_if(
|
|
checkpoints.rbegin(),
|
|
checkpoints.rend(),
|
|
[&](const auto & ckpt) {
|
|
const auto & cur = *ckpt;
|
|
// guarantee that a checkpoint will result in at least one token being processed [TAG_PROMPT_LOGITS]
|
|
SRV_TRC("checking checkpoint with [%d, %d] against %d...\n", cur.pos_min, cur.pos_max, pos_min_thold);
|
|
// workaround for [TAG_CHECKPOINTS_FIX_POS_MIN]
|
|
if (cur.pos_max > res.pos_next) {
|
|
return false;
|
|
}
|
|
return cur.pos_min < pos_min_thold || cur.pos_min == 0;
|
|
}
|
|
);
|
|
|
|
if (it == checkpoints.rend()) {
|
|
res.n_past = 0;
|
|
res.pos_next = 0;
|
|
return res;
|
|
}
|
|
|
|
const auto & cur = **it;
|
|
|
|
res.pos_next = std::min(res.pos_next, std::max(cur.pos_min + 1, cur.pos_max));
|
|
res.n_past = std::min(tokens.size_up_to_pos(res.pos_next), (size_t) cur.n_tokens);
|
|
res.ckpt = *it;
|
|
|
|
return res;
|
|
}
|
|
|
|
//
|
|
// server_prompt_cache
|
|
//
|
|
|
|
size_t server_prompt_cache::size() const {
|
|
size_t res = 0;
|
|
|
|
for (const auto & state : states) {
|
|
res += state.size();
|
|
}
|
|
|
|
return res;
|
|
}
|
|
|
|
size_t server_prompt_cache::n_tokens() const {
|
|
size_t res = 0;
|
|
|
|
for (const auto & state : states) {
|
|
res += state.prompt.n_tokens();
|
|
}
|
|
|
|
return res;
|
|
}
|
|
|
|
server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & prompt, size_t state_size_tgt, size_t state_size_dft) {
|
|
// first check if the current state is contained fully in the cache
|
|
for (auto it = states.begin(); it != states.end(); ++it) {
|
|
const int cur_lcp_len = it->prompt.tokens.get_common_prefix(prompt.tokens);
|
|
|
|
if (cur_lcp_len == (int) prompt.tokens.size() && are_lora_equal(it->prompt.lora, prompt.lora)) {
|
|
SRV_TRC("%s", " - prompt is already in the cache, skipping\n");
|
|
return nullptr;
|
|
}
|
|
}
|
|
|
|
// calculate checkpoints size to see if it will fit with the prompt
|
|
size_t checkpoints_size = 0;
|
|
for (const auto & ckpt : prompt.checkpoints) {
|
|
checkpoints_size += ckpt->size();
|
|
}
|
|
|
|
const size_t state_size_new = state_size_tgt + state_size_dft + checkpoints_size;
|
|
|
|
// skip over-limit entries to avoid disturbing the cache
|
|
if (limit_size > 0 && state_size_new > limit_size) {
|
|
SRV_WRN(" - prompt state size %.3f MiB exceeds cache size limit %.3f MiB, skipping\n",
|
|
state_size_new / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0));
|
|
return nullptr;
|
|
}
|
|
|
|
// remove any cached prompts that are fully contained in the current prompt
|
|
for (auto it = states.begin(); it != states.end();) {
|
|
const int len = it->prompt.tokens.get_common_prefix(prompt.tokens);
|
|
|
|
if (len == (int) it->prompt.tokens.size() && are_lora_equal(it->prompt.lora, prompt.lora)) {
|
|
SRV_TRC(" - removing obsolete cached prompt with length %d\n", len);
|
|
|
|
it = states.erase(it);
|
|
} else {
|
|
++it;
|
|
}
|
|
}
|
|
|
|
if (limit_size > 0) {
|
|
// make room before allocating the new vectors to avoid breaching the limit
|
|
while (!states.empty() && size() + state_size_new > limit_size) {
|
|
SRV_WRN(" - making room for prompt cache entry, removing oldest entry (size = %.3f MiB)\n",
|
|
states.front().size() / (1024.0 * 1024.0));
|
|
|
|
states.pop_front();
|
|
}
|
|
}
|
|
|
|
common_state_data state_data_tgt;
|
|
common_state_data state_data_dft;
|
|
|
|
// check if we can allocate enough memory for the new state
|
|
try {
|
|
state_data_tgt.resize(state_size_tgt);
|
|
state_data_dft.resize(state_size_dft);
|
|
} catch (const std::bad_alloc & e) {
|
|
SRV_ERR("failed to allocate memory for prompt cache state: %s\n", e.what());
|
|
|
|
limit_size = std::max<size_t>(1, 0.4*size());
|
|
|
|
SRV_WRN(" - cache size limit reduced to %.3f MiB\n", limit_size / (1024.0 * 1024.0));
|
|
|
|
update();
|
|
|
|
return nullptr;
|
|
}
|
|
|
|
states.push_back({
|
|
/*.prompt =*/ {
|
|
/*.tokens =*/ prompt.tokens.clone(),
|
|
/*.checkpoints =*/ prompt.checkpoints,
|
|
/*.lora =*/ prompt.lora,
|
|
},
|
|
/*.data =*/ {
|
|
/*.main =*/ std::move(state_data_tgt),
|
|
/*.drft =*/ std::move(state_data_dft),
|
|
},
|
|
});
|
|
|
|
return &states.back();
|
|
}
|
|
|
|
bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, const std::vector<common_adapter_lora_info> & lora_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) {
|
|
// a prompt made with another lora config is cleared before the new one starts, so it counts as empty
|
|
const bool is_empty = prompt.tokens.empty() || lora_should_clear_cache(prompt.lora, lora_new);
|
|
|
|
int lcp_best = is_empty ? 0 : prompt.tokens.get_common_prefix(tokens_new);
|
|
|
|
float f_keep_best = is_empty ? -1.0f : float(lcp_best) / prompt.tokens.size(); // empty slot: any cache entry wins
|
|
float f_sim_best = float(lcp_best) / tokens_new.size();
|
|
|
|
SRV_TRC(" - looking for better prompt, base f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);
|
|
|
|
auto it_best = states.end();
|
|
|
|
// find the most similar cached prompt, that would also preserve the most context
|
|
for (auto it = states.begin(); it != states.end(); ++it) {
|
|
if (lora_should_clear_cache(it->prompt.lora, lora_new)) {
|
|
SRV_TRC(" - prompt with length %7zu, skipping, other lora config\n", it->prompt.tokens.size());
|
|
continue;
|
|
}
|
|
|
|
const int lcp_cur = it->prompt.tokens.get_common_prefix(tokens_new);
|
|
|
|
const float f_keep_cur = float(lcp_cur) / it->prompt.tokens.size();
|
|
const float f_sim_cur = float(lcp_cur) / tokens_new.size();
|
|
|
|
SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur);
|
|
|
|
// don't trash large prompts
|
|
if (f_keep_cur < 0.25f) {
|
|
continue;
|
|
}
|
|
|
|
if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) {
|
|
f_keep_best = f_keep_cur;
|
|
f_sim_best = f_sim_cur;
|
|
lcp_best = lcp_cur;
|
|
|
|
it_best = it;
|
|
}
|
|
}
|
|
|
|
if (it_best != states.end()) {
|
|
SRV_TRC(" - found better prompt with f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);
|
|
|
|
const int64_t t_start = ggml_time_us();
|
|
|
|
{
|
|
auto & data = it_best->data.main;
|
|
|
|
const size_t size = data.size();
|
|
const size_t n = llama_state_seq_set_data_ext(ctx_tgt, data.data(), size, id_slot, 0);
|
|
if (n != size) {
|
|
SRV_ERR("failed to restore state with size %zu\n", size);
|
|
|
|
return false;
|
|
}
|
|
|
|
data.clear();
|
|
data.shrink_to_fit();
|
|
}
|
|
|
|
{
|
|
auto & data = it_best->data.drft;
|
|
|
|
if (!data.empty()) {
|
|
GGML_ASSERT(ctx_dft);
|
|
|
|
const size_t size = data.size();
|
|
const size_t n = llama_state_seq_set_data_ext(ctx_dft, data.data(), size, id_slot, 0);
|
|
if (n != size) {
|
|
SRV_WRN("failed to restore state with size %zu\n", size);
|
|
|
|
return false;
|
|
}
|
|
|
|
data.clear();
|
|
data.shrink_to_fit();
|
|
}
|
|
}
|
|
|
|
SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_RESTORE__\n");
|
|
|
|
// the whole cached prompt is a prefix of the new one
|
|
if (lcp_best == (int) it_best->prompt.tokens.size()) {
|
|
SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_FULL_RESTORE__\n");
|
|
}
|
|
|
|
prompt = std::move(it_best->prompt);
|
|
|
|
states.erase(it_best);
|
|
|
|
SRV_TRC("slot %d: prompt cache restore took %.2f ms\n", id_slot, (ggml_time_us() - t_start) / 1000.0);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void server_prompt_cache::update() {
|
|
if (limit_size > 0) {
|
|
while (!states.empty() && size() > limit_size) {
|
|
SRV_WRN(" - cache size limit reached, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0));
|
|
|
|
states.pop_front();
|
|
}
|
|
}
|
|
|
|
// average size per token
|
|
const float size_per_token = std::max<float>(1.0f, float(size()) / (std::max<size_t>(1, n_tokens())));
|
|
|
|
// dynamically increase the token limit if it can fit in the memory limit
|
|
const size_t limit_tokens_cur = limit_size > 0 ? std::max<size_t>(limit_tokens, limit_size/size_per_token) : limit_tokens;
|
|
|
|
if (limit_tokens > 0) {
|
|
while (!states.empty() && n_tokens() > limit_tokens_cur) {
|
|
SRV_WRN(" - cache token limit (%zu, est: %zu) reached, removing oldest entry (size = %.3f MiB)\n",
|
|
limit_tokens, limit_tokens_cur, states.front().size() / (1024.0 * 1024.0));
|
|
|
|
states.pop_front();
|
|
}
|
|
}
|
|
|
|
SRV_TRC(" - cache state: %zu prompts, %.3f MiB (limits: %.3f MiB, %zu tokens, %zu est)\n",
|
|
states.size(), size() / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0), limit_tokens, limit_tokens_cur);
|
|
|
|
for (const auto & state : states) {
|
|
SRV_TRC(" - prompt %p: %7d tokens, checkpoints: %2zu, %9.3f MiB\n",
|
|
(const void *)&state, state.prompt.n_tokens(), state.prompt.checkpoints.size(), state.size() / (1024.0 * 1024.0));
|
|
}
|
|
}
|
|
|
|
//
|
|
// server_slot_sched
|
|
//
|
|
|
|
static std::vector<const server_slot_info *> get_free_slot_infos(const std::vector<server_slot_info> & slots, size_t n_slots_needed, int exclude_id_slot) {
|
|
std::vector<const server_slot_info *> free_slots;
|
|
for (const auto & slot : slots) {
|
|
if (!slot.is_processing && slot.id != exclude_id_slot) {
|
|
free_slots.push_back(&slot);
|
|
}
|
|
}
|
|
|
|
std::stable_sort(free_slots.begin(), free_slots.end(), [](const server_slot_info * a, const server_slot_info * b) {
|
|
if ((a->prompt->n_tokens() == 0) != (b->prompt->n_tokens() == 0)) {
|
|
return a->prompt->n_tokens() == 0;
|
|
}
|
|
return a->t_last_used < b->t_last_used;
|
|
});
|
|
|
|
free_slots.resize(std::min(free_slots.size(), n_slots_needed));
|
|
|
|
return free_slots;
|
|
}
|
|
|
|
std::vector<int> server_slot_sched::get_free_slots(const std::vector<server_slot_info> & slots, size_t n_slots_needed, int exclude_id_slot) const {
|
|
std::vector<int> res;
|
|
for (const auto * slot : get_free_slot_infos(slots, n_slots_needed, exclude_id_slot)) {
|
|
res.push_back(slot->id);
|
|
}
|
|
|
|
return res;
|
|
}
|
|
|
|
server_slot_plan server_slot_sched::select(const std::vector<server_slot_info> & slots, const server_task & task, const std::vector<common_adapter_lora_info> & lora) const {
|
|
const server_slot_info * ret = nullptr;
|
|
|
|
bool update_cache = false;
|
|
|
|
// if a specific slot is requested, use it (still goes through cache update logic below)
|
|
if (task.id_slot != -1 && !slots.empty()) {
|
|
// note: allow id_slot to be out of bounds (wrap around)
|
|
const int id_slot = task.id_slot % slots.size();
|
|
|
|
for (const auto & slot : slots) {
|
|
if (slot.id == id_slot) {
|
|
ret = &slot;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (ret) {
|
|
// a busy slot is returned untouched, the caller defers the task
|
|
if (ret->is_processing) {
|
|
return { ret->id, false };
|
|
}
|
|
|
|
SLT_INF(*ret, "selected slot by id (%d)\n", task.id_slot);
|
|
}
|
|
}
|
|
|
|
// find the slot that has at least n% prompt similarity
|
|
if (slot_prompt_similarity != 0.0f) {
|
|
float f_sim_best = 0;
|
|
|
|
for (const auto & slot : slots) {
|
|
if (task.id_slot != -1 && slot.id != task.id_slot) {
|
|
continue;
|
|
}
|
|
|
|
// skip the slot if it is not available
|
|
if (slot.is_processing) {
|
|
SLT_TRC(slot, " - skipping, is_processing = %d\n", slot.is_processing);
|
|
continue;
|
|
}
|
|
|
|
const auto & tokens = slot.prompt->tokens;
|
|
|
|
// skip the slot if it does not contains cached tokens
|
|
if (tokens.empty()) {
|
|
SLT_TRC(slot, "%s", " - skipping, slot is empty\n");
|
|
continue;
|
|
}
|
|
|
|
// the task would clear this slot, its state cannot be reused with another lora config
|
|
if (lora_should_clear_cache(slot.prompt->lora, lora)) {
|
|
SLT_TRC(slot, "%s", " - skipping, other lora config\n");
|
|
continue;
|
|
}
|
|
|
|
// fraction of the Longest Common Prefix length with respect to the input prompt length
|
|
const size_t lcp_len = tokens.get_common_prefix(task.tokens);
|
|
const float f_sim_cur = float(lcp_len) / task.tokens.size();
|
|
|
|
SLT_TRC(slot, " - checking sim = %.3f (%zu/%zu) > %.3f\n", f_sim_cur, lcp_len, task.tokens.size(), slot_prompt_similarity);
|
|
|
|
// select the current slot if the criteria match
|
|
if (f_sim_cur > f_sim_best && f_sim_cur > slot_prompt_similarity) {
|
|
f_sim_best = f_sim_cur;
|
|
|
|
ret = &slot;
|
|
}
|
|
}
|
|
|
|
if (ret != nullptr) {
|
|
const float f_keep = (f_sim_best*task.tokens.size()) / ret->prompt->tokens.size();
|
|
|
|
if (task.id_slot == -1) {
|
|
SLT_INF(*ret, "selected slot by LCP similarity, f_sim_best = %.3f (> %.3f thold), f_keep = %.3f\n",
|
|
f_sim_best, slot_prompt_similarity, f_keep);
|
|
}
|
|
|
|
// if we are about to lose a large portion of the existing context - save it in the prompt cache
|
|
if (f_keep < 0.5f) {
|
|
update_cache = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// find an empty slot, or the one that has been least recently used
|
|
if (ret == nullptr) {
|
|
const auto free_slots = get_free_slot_infos(slots, 1, -1);
|
|
|
|
if (!free_slots.empty()) {
|
|
ret = free_slots[0];
|
|
|
|
SLT_INF(*ret, "selected slot by LRU, n_tokens = %d, t_last = %" PRId64 "\n", ret->prompt->n_tokens(), ret->t_last_used);
|
|
|
|
update_cache = true;
|
|
}
|
|
}
|
|
|
|
if (ret == nullptr) {
|
|
return {};
|
|
}
|
|
|
|
// cache prompts only for the tasks that reuse them
|
|
update_cache = update_cache && !task.is_stateless();
|
|
|
|
return { ret->id, update_cache };
|
|
}
|
|
|
|
//
|
|
// server_prompt_disk_cache
|
|
//
|
|
|
|
// file = header + payload; the payload is a list of sections, each padded to 8 bytes:
|
|
// tokens, checkpoint table, state size, state, the state of each checkpoint
|
|
// hash_payload is FNV-1a over the 64-bit words of the payload, so a truncated or corrupt file is never used
|
|
static constexpr uint32_t DISK_CACHE_MAGIC = 0x4443504c; // "LPCD"
|
|
static constexpr uint32_t DISK_CACHE_VERSION = 1;
|
|
static constexpr const char * DISK_CACHE_EXT = ".lpcd";
|
|
|
|
// keep this much space free on the file system
|
|
static constexpr uint64_t DISK_CACHE_MIN_FREE = 1024ull*1024ull*1024ull;
|
|
|
|
struct disk_cache_header {
|
|
uint32_t magic;
|
|
uint32_t version;
|
|
uint64_t key_hash;
|
|
uint64_t n_tokens;
|
|
uint64_t n_ckpt;
|
|
uint64_t size_payload;
|
|
uint64_t hash_payload;
|
|
};
|
|
|
|
struct disk_cache_ckpt {
|
|
int64_t n_tokens;
|
|
llama_pos pos_min;
|
|
llama_pos pos_max;
|
|
uint64_t size;
|
|
};
|
|
|
|
static_assert(sizeof(disk_cache_header) == 48 && sizeof(disk_cache_ckpt) == 24, "disk cache layout");
|
|
|
|
static uint64_t disk_cache_pad(uint64_t n) {
|
|
return (8 - n % 8) % 8;
|
|
}
|
|
|
|
static uint64_t disk_cache_hash_bytes(const void * data, size_t n) {
|
|
uint64_t h = 0xcbf29ce484222325ull;
|
|
for (size_t i = 0; i < n; i++) {
|
|
h = (h ^ ((const uint8_t *) data)[i]) * 0x100000001b3ull;
|
|
}
|
|
return h;
|
|
}
|
|
|
|
static uint64_t disk_cache_payload_size(uint64_t n_tokens, const std::vector<disk_cache_ckpt> & ckpts, uint64_t n_state) {
|
|
const auto padded = [](uint64_t n) { return n + disk_cache_pad(n); };
|
|
uint64_t res = padded(n_tokens*sizeof(llama_token)) + padded(ckpts.size()*sizeof(disk_cache_ckpt)) + sizeof(uint64_t) + padded(n_state);
|
|
for (const auto & c : ckpts) {
|
|
res += padded(c.size);
|
|
}
|
|
return res;
|
|
}
|
|
|
|
// sequential section IO with the running payload hash
|
|
struct disk_cache_io {
|
|
FILE * f = nullptr;
|
|
uint64_t hash = 0xcbf29ce484222325ull;
|
|
bool ok = true;
|
|
|
|
~disk_cache_io() {
|
|
if (f) {
|
|
fclose(f);
|
|
}
|
|
}
|
|
|
|
void add_hash(const uint8_t * p, size_t size, const uint8_t * pad) {
|
|
const size_t n_full = size - size % 8;
|
|
for (size_t i = 0; i < n_full; i += 8) {
|
|
uint64_t w;
|
|
memcpy(&w, p + i, 8);
|
|
hash = (hash ^ w) * 0x100000001b3ull;
|
|
}
|
|
if (size % 8) {
|
|
uint64_t w;
|
|
memcpy(&w, p + n_full, size % 8);
|
|
memcpy((uint8_t *) &w + size % 8, pad, disk_cache_pad(size));
|
|
hash = (hash ^ w) * 0x100000001b3ull;
|
|
}
|
|
}
|
|
|
|
void write(const void * src, size_t size) {
|
|
static const uint8_t zero[8] = {};
|
|
const size_t n_pad = disk_cache_pad(size);
|
|
ok = ok && fwrite(src, 1, size, f) == size && fwrite(zero, 1, n_pad, f) == n_pad;
|
|
add_hash((const uint8_t *) src, size, zero);
|
|
}
|
|
|
|
void read(void * dst, size_t size) {
|
|
uint8_t pad[8];
|
|
const size_t n_pad = disk_cache_pad(size);
|
|
ok = ok && fread(dst, 1, size, f) == size && fread(pad, 1, n_pad, f) == n_pad;
|
|
if (ok) {
|
|
add_hash((const uint8_t *) dst, size, pad);
|
|
}
|
|
}
|
|
|
|
// the data is in our buffers, the cached pages are waste (see llama_file::drop_cache)
|
|
void drop_cache() {
|
|
#if defined(__linux__)
|
|
posix_fadvise(fileno(f), 0, 0, POSIX_FADV_DONTNEED);
|
|
#endif
|
|
}
|
|
};
|
|
|
|
// open a file and read everything up to the state data: 1 = entry of this key, 0 = other key or version, -1 = bad file
|
|
static int disk_cache_open(const std::string & path, uint64_t key_hash, disk_cache_io & io, disk_cache_header & hdr,
|
|
llama_tokens & tokens, std::vector<disk_cache_ckpt> & ckpts, uint64_t & n_state) {
|
|
std::error_code ec;
|
|
const uint64_t size = std::filesystem::file_size(path, ec);
|
|
io.f = ec ? nullptr : fopen(path.c_str(), "rb");
|
|
if (io.f == nullptr || fread(&hdr, 1, sizeof(hdr), io.f) != sizeof(hdr) || hdr.magic != DISK_CACHE_MAGIC) {
|
|
return -1;
|
|
}
|
|
if (hdr.version != DISK_CACHE_VERSION) {
|
|
return 0;
|
|
}
|
|
if (size != sizeof(hdr) + hdr.size_payload || hdr.n_tokens > size/sizeof(llama_token) || hdr.n_ckpt > size/sizeof(disk_cache_ckpt)) {
|
|
return -1;
|
|
}
|
|
if (hdr.key_hash != key_hash) {
|
|
return 0;
|
|
}
|
|
tokens.resize(hdr.n_tokens);
|
|
ckpts.resize(hdr.n_ckpt);
|
|
io.read(tokens.data(), tokens.size()*sizeof(llama_token));
|
|
io.read(ckpts.data(), ckpts.size()*sizeof(disk_cache_ckpt));
|
|
io.read(&n_state, sizeof(n_state));
|
|
if (!io.ok || n_state > size) {
|
|
return -1;
|
|
}
|
|
for (const auto & c : ckpts) {
|
|
if (c.size > size) {
|
|
return -1;
|
|
}
|
|
}
|
|
return disk_cache_payload_size(hdr.n_tokens, ckpts, n_state) == hdr.size_payload ? 1 : -1;
|
|
}
|
|
|
|
static void disk_cache_touch(const std::string & path) {
|
|
std::error_code ec;
|
|
std::filesystem::last_write_time(path, std::filesystem::file_time_type::clock::now(), ec);
|
|
}
|
|
|
|
server_prompt_disk_cache::server_prompt_disk_cache(const std::string & dir, int32_t limit_size_mib, const std::string & key) : dir(dir) {
|
|
key_hash = disk_cache_hash_bytes(key.data(), key.size());
|
|
limit_size = 1024ull*1024ull*std::max(0, limit_size_mib);
|
|
|
|
size_t n_other = 0;
|
|
size_t n_bad = 0;
|
|
try {
|
|
std::filesystem::create_directories(dir);
|
|
const auto now = std::filesystem::file_time_type::clock::now();
|
|
for (const auto & de : std::filesystem::directory_iterator(dir)) {
|
|
const std::string path = de.path().string();
|
|
if (!de.is_regular_file()) {
|
|
continue;
|
|
}
|
|
// left by a writer that was killed
|
|
if (path.find(std::string(DISK_CACHE_EXT) + ".tmp") != std::string::npos) {
|
|
if (now - de.last_write_time() > std::chrono::hours(1)) {
|
|
std::filesystem::remove(de.path());
|
|
}
|
|
continue;
|
|
}
|
|
if (de.path().extension() != DISK_CACHE_EXT) {
|
|
continue;
|
|
}
|
|
disk_cache_io io;
|
|
disk_cache_header hdr;
|
|
server_prompt_disk_entry e;
|
|
std::vector<disk_cache_ckpt> ckpts;
|
|
uint64_t n_state = 0;
|
|
const int res = disk_cache_open(path, key_hash, io, hdr, e.tokens, ckpts, n_state);
|
|
if (res < 0) {
|
|
SRV_WRN("disk cache: '%s' is truncated or corrupt, removing it\n", path.c_str());
|
|
std::filesystem::remove(de.path());
|
|
n_bad++;
|
|
continue;
|
|
}
|
|
if (res == 0) {
|
|
n_other++;
|
|
continue;
|
|
}
|
|
e.path = path;
|
|
for (const auto & c : ckpts) {
|
|
e.ckpt_n_tokens.push_back(c.n_tokens);
|
|
}
|
|
entries.push_back(std::move(e));
|
|
}
|
|
} catch (const std::exception & err) {
|
|
SRV_WRN("disk cache: error reading '%s': %s\n", dir.c_str(), err.what());
|
|
}
|
|
|
|
SRV_INF("disk cache: '%s', %zu prompts of this model, %zu other files, %zu bad files removed, limit %.0f MiB\n",
|
|
dir.c_str(), entries.size(), n_other, n_bad, limit_size / (1024.0 * 1024.0));
|
|
}
|
|
|
|
size_t server_prompt_disk_cache::n_reuse(const llama_tokens & tokens, const std::vector<int64_t> & ckpt_n_tokens, const llama_tokens & tokens_new, bool can_trim) {
|
|
size_t lcp = 0;
|
|
while (lcp < tokens.size() && lcp < tokens_new.size() && tokens[lcp] == tokens_new[lcp]) {
|
|
lcp++;
|
|
}
|
|
if (lcp == tokens.size() || can_trim) {
|
|
return lcp;
|
|
}
|
|
size_t res = 0;
|
|
for (const int64_t n : ckpt_n_tokens) {
|
|
if (n <= (int64_t) lcp) {
|
|
res = std::max(res, (size_t) n);
|
|
}
|
|
}
|
|
return res;
|
|
}
|
|
|
|
int server_prompt_disk_cache::find(const llama_tokens & tokens_new, size_t n_have, size_t n_min, bool can_trim) const {
|
|
int best = -1;
|
|
size_t n_best = 0;
|
|
for (size_t i = 0; i < entries.size(); i++) {
|
|
const size_t n = n_reuse(entries[i].tokens, entries[i].ckpt_n_tokens, tokens_new, can_trim);
|
|
if (n >= n_have + n_min && n > n_best) {
|
|
best = (int) i;
|
|
n_best = n;
|
|
}
|
|
}
|
|
return best;
|
|
}
|
|
|
|
bool server_prompt_disk_cache::has(const llama_tokens & tokens) {
|
|
for (const auto & e : entries) {
|
|
std::error_code ec;
|
|
if (e.tokens == tokens && std::filesystem::exists(e.path, ec)) {
|
|
disk_cache_touch(e.path);
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void server_prompt_disk_cache::remove(int i) {
|
|
std::error_code ec;
|
|
std::filesystem::remove(entries[i].path, ec);
|
|
entries.erase(entries.begin() + i);
|
|
}
|
|
|
|
bool server_prompt_disk_cache::load(int i, server_prompt & prompt, common_state_data & data, bool has_mtmd) {
|
|
const std::string path = entries[i].path;
|
|
|
|
disk_cache_io io;
|
|
disk_cache_header hdr;
|
|
llama_tokens tokens;
|
|
std::vector<disk_cache_ckpt> ckpts;
|
|
uint64_t n_state = 0;
|
|
std::list<std::shared_ptr<const common_prompt_checkpoint>> checkpoints;
|
|
|
|
bool ok = disk_cache_open(path, key_hash, io, hdr, tokens, ckpts, n_state) == 1 && tokens == entries[i].tokens;
|
|
if (ok) {
|
|
try {
|
|
data.resize(n_state);
|
|
io.read(data.data(), data.size());
|
|
for (const auto & c : ckpts) {
|
|
auto cur = std::make_shared<common_prompt_checkpoint>();
|
|
cur->id_task = -1; // not created by a task: may fail to load, then the prompt is re-processed
|
|
cur->n_tokens = c.n_tokens;
|
|
cur->pos_min = c.pos_min;
|
|
cur->pos_max = c.pos_max;
|
|
cur->data_tgt.resize(c.size);
|
|
io.read(cur->data_tgt.data(), cur->data_tgt.size());
|
|
checkpoints.push_back(std::move(cur));
|
|
}
|
|
} catch (const std::bad_alloc &) {
|
|
SRV_WRN("disk cache: out of memory reading '%s'\n", path.c_str());
|
|
data.clear();
|
|
return false;
|
|
}
|
|
ok = io.ok && io.hash == hdr.hash_payload;
|
|
io.drop_cache();
|
|
}
|
|
if (!ok) {
|
|
SRV_WRN("disk cache: '%s' is missing, truncated or corrupt, removing it\n", path.c_str());
|
|
data.clear();
|
|
remove(i);
|
|
return false;
|
|
}
|
|
|
|
disk_cache_touch(path);
|
|
|
|
prompt.tokens = server_tokens(tokens, has_mtmd);
|
|
prompt.checkpoints = std::move(checkpoints);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool server_prompt_disk_cache::save(const server_prompt & prompt, const common_state_data & data) {
|
|
// text only, see disk_cache_collect
|
|
const llama_tokens tokens = prompt.tokens.get_text_tokens();
|
|
|
|
std::vector<disk_cache_ckpt> ckpts;
|
|
for (const auto & c : prompt.checkpoints) {
|
|
ckpts.push_back({ c->n_tokens, c->pos_min, c->pos_max, c->data_tgt.size() });
|
|
}
|
|
|
|
disk_cache_header hdr = {
|
|
/*.magic =*/ DISK_CACHE_MAGIC,
|
|
/*.version =*/ DISK_CACHE_VERSION,
|
|
/*.key_hash =*/ key_hash,
|
|
/*.n_tokens =*/ tokens.size(),
|
|
/*.n_ckpt =*/ ckpts.size(),
|
|
/*.size_payload =*/ disk_cache_payload_size(tokens.size(), ckpts, data.size()),
|
|
/*.hash_payload =*/ 0,
|
|
};
|
|
const uint64_t size_file = sizeof(hdr) + hdr.size_payload;
|
|
|
|
char name[64];
|
|
snprintf(name, sizeof(name), "%016llx-%016llx", (unsigned long long) key_hash,
|
|
(unsigned long long) disk_cache_hash_bytes(tokens.data(), tokens.size()*sizeof(llama_token)));
|
|
const std::string path = (std::filesystem::path(dir) / name).string() + DISK_CACHE_EXT;
|
|
const std::string path_tmp = path + ".tmp";
|
|
|
|
// entries contained in the new one are obsolete: removed once it is written, not counted for the limit
|
|
const auto is_obsolete = [&](const server_prompt_disk_entry & e) {
|
|
return e.path != path && e.tokens.size() <= tokens.size() && std::equal(e.tokens.begin(), e.tokens.end(), tokens.begin());
|
|
};
|
|
std::vector<std::string> obsolete;
|
|
for (const auto & e : entries) {
|
|
if (is_obsolete(e)) {
|
|
obsolete.push_back(e.path);
|
|
}
|
|
}
|
|
|
|
if (limit_size > 0 && size_file > limit_size) {
|
|
SRV_WRN("disk cache: prompt state of %.1f MiB exceeds the limit of %.1f MiB, not saved\n", size_file / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0));
|
|
return false;
|
|
}
|
|
|
|
try {
|
|
// evict the least recently used files of every model in dir
|
|
struct file_info {
|
|
std::filesystem::path path;
|
|
uint64_t size;
|
|
std::filesystem::file_time_type time;
|
|
};
|
|
std::vector<file_info> files;
|
|
uint64_t size_total = 0;
|
|
for (const auto & de : std::filesystem::directory_iterator(dir)) {
|
|
if (de.is_regular_file() && de.path().extension() == DISK_CACHE_EXT && de.path().string() != path &&
|
|
std::find(obsolete.begin(), obsolete.end(), de.path().string()) == obsolete.end()) {
|
|
files.push_back({ de.path(), de.file_size(), de.last_write_time() });
|
|
size_total += files.back().size;
|
|
}
|
|
}
|
|
std::sort(files.begin(), files.end(), [](const file_info & a, const file_info & b) { return a.time < b.time; });
|
|
for (size_t i = 0; limit_size > 0 && size_total + size_file > limit_size && i < files.size(); i++) {
|
|
SRV_INF("disk cache: size limit, removing '%s' (%.1f MiB)\n", files[i].path.string().c_str(), files[i].size / (1024.0 * 1024.0));
|
|
std::filesystem::remove(files[i].path);
|
|
size_total -= files[i].size;
|
|
for (size_t j = 0; j < entries.size(); j++) {
|
|
if (entries[j].path == files[i].path.string()) {
|
|
entries.erase(entries.begin() + j);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
const uint64_t avail = std::filesystem::space(dir).available;
|
|
if (avail < size_file + DISK_CACHE_MIN_FREE) {
|
|
SRV_WRN("disk cache: only %.1f MiB free in '%s', prompt state of %.1f MiB not saved\n", avail / (1024.0 * 1024.0), dir.c_str(), size_file / (1024.0 * 1024.0));
|
|
return false;
|
|
}
|
|
} catch (const std::exception & err) {
|
|
SRV_WRN("disk cache: error reading '%s': %s\n", dir.c_str(), err.what());
|
|
return false;
|
|
}
|
|
|
|
bool ok;
|
|
{
|
|
disk_cache_io io;
|
|
io.f = fopen(path_tmp.c_str(), "wb");
|
|
ok = io.f != nullptr && fwrite(&hdr, 1, sizeof(hdr), io.f) == sizeof(hdr);
|
|
if (ok) {
|
|
const uint64_t n_state = data.size();
|
|
io.write(tokens.data(), tokens.size()*sizeof(llama_token));
|
|
io.write(ckpts.data(), ckpts.size()*sizeof(disk_cache_ckpt));
|
|
io.write(&n_state, sizeof(n_state));
|
|
io.write(data.data(), data.size());
|
|
for (const auto & c : prompt.checkpoints) {
|
|
io.write(c->data_tgt.data(), c->data_tgt.size());
|
|
}
|
|
hdr.hash_payload = io.hash;
|
|
ok = io.ok && fseek(io.f, 0, SEEK_SET) == 0 && fwrite(&hdr, 1, sizeof(hdr), io.f) == sizeof(hdr) && fflush(io.f) == 0;
|
|
#if defined(__linux__)
|
|
// write back now, so the pages can be dropped and the rename never exposes missing data
|
|
ok = ok && fdatasync(fileno(io.f)) == 0;
|
|
io.drop_cache();
|
|
#endif
|
|
}
|
|
if (io.f != nullptr && fclose(io.f) != 0) {
|
|
ok = false;
|
|
}
|
|
io.f = nullptr;
|
|
if (!ok) {
|
|
SRV_WRN("disk cache: failed to write '%s': %s, not saved\n", path_tmp.c_str(), strerror(errno));
|
|
}
|
|
}
|
|
std::error_code ec;
|
|
if (ok) {
|
|
std::filesystem::rename(path_tmp, path, ec);
|
|
ok = !ec;
|
|
if (!ok) {
|
|
SRV_WRN("disk cache: failed to rename '%s': %s, not saved\n", path_tmp.c_str(), ec.message().c_str());
|
|
}
|
|
}
|
|
if (!ok) {
|
|
std::filesystem::remove(path_tmp, ec);
|
|
return false;
|
|
}
|
|
|
|
for (size_t i = 0; i < entries.size(); ) {
|
|
const auto & e = entries[i];
|
|
if (e.path == path) {
|
|
entries.erase(entries.begin() + i);
|
|
} else if (is_obsolete(e)) {
|
|
SRV_INF("disk cache: removing '%s', a prefix of the new prompt\n", e.path.c_str());
|
|
remove((int) i);
|
|
} else {
|
|
i++;
|
|
}
|
|
}
|
|
|
|
server_prompt_disk_entry e;
|
|
e.path = path;
|
|
e.tokens = tokens;
|
|
for (const auto & c : ckpts) {
|
|
e.ckpt_n_tokens.push_back(c.n_tokens);
|
|
}
|
|
entries.push_back(std::move(e));
|
|
|
|
return true;
|
|
}
|