#include "server-slot-sched.h" #include "common.h" #include "llama.h" #include "log.h" #include #include #include #include #include #include #if defined(__linux__) #include #include #endif // // server_prompt // server_prompt_reuse server_prompt::get_reuse(int32_t n_prefix, bool has_new_tokens, int32_t n_swa, llama_pos pos_min) const { server_prompt_reuse res; res.n_past = n_prefix; res.pos_next = tokens.pos_next(n_prefix); // the first evaluated token attends to the positions > pos_eval - n_swa, so a SWA state // starting at pos_eval - n_swa + 1 is enough, while a state without SWA must end before pos_eval const llama_pos pos_eval = res.pos_next - (has_new_tokens ? 0 : 1); // a state or a checkpoint is useful when its pos_min is below this threshold const auto pos_min_thold = std::max(0, n_swa > 0 ? pos_eval - n_swa + 2 : pos_eval); if (pos_min < pos_min_thold) { return res; } // search for a context checkpoint const auto it = std::find_if( checkpoints.rbegin(), checkpoints.rend(), [&](const auto & ckpt) { const auto & cur = *ckpt; // guarantee that a checkpoint will result in at least one token being processed [TAG_PROMPT_LOGITS] SRV_TRC("checking checkpoint with [%d, %d] against %d...\n", cur.pos_min, cur.pos_max, pos_min_thold); // workaround for [TAG_CHECKPOINTS_FIX_POS_MIN] if (cur.pos_max > res.pos_next) { return false; } return cur.pos_min < pos_min_thold || cur.pos_min == 0; } ); if (it == checkpoints.rend()) { res.n_past = 0; res.pos_next = 0; return res; } const auto & cur = **it; res.pos_next = std::min(res.pos_next, std::max(cur.pos_min + 1, cur.pos_max)); res.n_past = std::min(tokens.size_up_to_pos(res.pos_next), (size_t) cur.n_tokens); res.ckpt = *it; return res; } // // server_prompt_cache // size_t server_prompt_cache::size() const { size_t res = 0; for (const auto & state : states) { res += state.size(); } return res; } size_t server_prompt_cache::n_tokens() const { size_t res = 0; for (const auto & state : states) { res += state.prompt.n_tokens(); } return res; } server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & prompt, size_t state_size_tgt, size_t state_size_dft) { // first check if the current state is contained fully in the cache for (auto it = states.begin(); it != states.end(); ++it) { const int cur_lcp_len = it->prompt.tokens.get_common_prefix(prompt.tokens); if (cur_lcp_len == (int) prompt.tokens.size() && are_lora_equal(it->prompt.lora, prompt.lora)) { SRV_TRC("%s", " - prompt is already in the cache, skipping\n"); return nullptr; } } // calculate checkpoints size to see if it will fit with the prompt size_t checkpoints_size = 0; for (const auto & ckpt : prompt.checkpoints) { checkpoints_size += ckpt->size(); } const size_t state_size_new = state_size_tgt + state_size_dft + checkpoints_size; // skip over-limit entries to avoid disturbing the cache if (limit_size > 0 && state_size_new > limit_size) { SRV_WRN(" - prompt state size %.3f MiB exceeds cache size limit %.3f MiB, skipping\n", state_size_new / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0)); return nullptr; } // remove any cached prompts that are fully contained in the current prompt for (auto it = states.begin(); it != states.end();) { const int len = it->prompt.tokens.get_common_prefix(prompt.tokens); if (len == (int) it->prompt.tokens.size() && are_lora_equal(it->prompt.lora, prompt.lora)) { SRV_TRC(" - removing obsolete cached prompt with length %d\n", len); it = states.erase(it); } else { ++it; } } if (limit_size > 0) { // make room before allocating the new vectors to avoid breaching the limit while (!states.empty() && size() + state_size_new > limit_size) { SRV_WRN(" - making room for prompt cache entry, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0)); states.pop_front(); } } common_state_data state_data_tgt; common_state_data state_data_dft; // check if we can allocate enough memory for the new state try { state_data_tgt.resize(state_size_tgt); state_data_dft.resize(state_size_dft); } catch (const std::bad_alloc & e) { SRV_ERR("failed to allocate memory for prompt cache state: %s\n", e.what()); limit_size = std::max(1, 0.4*size()); SRV_WRN(" - cache size limit reduced to %.3f MiB\n", limit_size / (1024.0 * 1024.0)); update(); return nullptr; } states.push_back({ /*.prompt =*/ { /*.tokens =*/ prompt.tokens.clone(), /*.checkpoints =*/ prompt.checkpoints, /*.lora =*/ prompt.lora, }, /*.data =*/ { /*.main =*/ std::move(state_data_tgt), /*.drft =*/ std::move(state_data_dft), }, }); return &states.back(); } bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, const std::vector & lora_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) { // a prompt made with another lora config is cleared before the new one starts, so it counts as empty const bool is_empty = prompt.tokens.empty() || lora_should_clear_cache(prompt.lora, lora_new); int lcp_best = is_empty ? 0 : prompt.tokens.get_common_prefix(tokens_new); float f_keep_best = is_empty ? -1.0f : float(lcp_best) / prompt.tokens.size(); // empty slot: any cache entry wins float f_sim_best = float(lcp_best) / tokens_new.size(); SRV_TRC(" - looking for better prompt, base f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best); auto it_best = states.end(); // find the most similar cached prompt, that would also preserve the most context for (auto it = states.begin(); it != states.end(); ++it) { if (lora_should_clear_cache(it->prompt.lora, lora_new)) { SRV_TRC(" - prompt with length %7zu, skipping, other lora config\n", it->prompt.tokens.size()); continue; } const int lcp_cur = it->prompt.tokens.get_common_prefix(tokens_new); const float f_keep_cur = float(lcp_cur) / it->prompt.tokens.size(); const float f_sim_cur = float(lcp_cur) / tokens_new.size(); SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur); // don't trash large prompts if (f_keep_cur < 0.25f) { continue; } if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) { f_keep_best = f_keep_cur; f_sim_best = f_sim_cur; lcp_best = lcp_cur; it_best = it; } } if (it_best != states.end()) { SRV_TRC(" - found better prompt with f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best); const int64_t t_start = ggml_time_us(); { auto & data = it_best->data.main; const size_t size = data.size(); const size_t n = llama_state_seq_set_data_ext(ctx_tgt, data.data(), size, id_slot, 0); if (n != size) { SRV_ERR("failed to restore state with size %zu\n", size); return false; } data.clear(); data.shrink_to_fit(); } { auto & data = it_best->data.drft; if (!data.empty()) { GGML_ASSERT(ctx_dft); const size_t size = data.size(); const size_t n = llama_state_seq_set_data_ext(ctx_dft, data.data(), size, id_slot, 0); if (n != size) { SRV_WRN("failed to restore state with size %zu\n", size); return false; } data.clear(); data.shrink_to_fit(); } } SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_RESTORE__\n"); // the whole cached prompt is a prefix of the new one if (lcp_best == (int) it_best->prompt.tokens.size()) { SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_FULL_RESTORE__\n"); } prompt = std::move(it_best->prompt); states.erase(it_best); SRV_TRC("slot %d: prompt cache restore took %.2f ms\n", id_slot, (ggml_time_us() - t_start) / 1000.0); } return true; } void server_prompt_cache::update() { if (limit_size > 0) { while (!states.empty() && size() > limit_size) { SRV_WRN(" - cache size limit reached, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0)); states.pop_front(); } } // average size per token const float size_per_token = std::max(1.0f, float(size()) / (std::max(1, n_tokens()))); // dynamically increase the token limit if it can fit in the memory limit const size_t limit_tokens_cur = limit_size > 0 ? std::max(limit_tokens, limit_size/size_per_token) : limit_tokens; if (limit_tokens > 0) { while (!states.empty() && n_tokens() > limit_tokens_cur) { SRV_WRN(" - cache token limit (%zu, est: %zu) reached, removing oldest entry (size = %.3f MiB)\n", limit_tokens, limit_tokens_cur, states.front().size() / (1024.0 * 1024.0)); states.pop_front(); } } SRV_TRC(" - cache state: %zu prompts, %.3f MiB (limits: %.3f MiB, %zu tokens, %zu est)\n", states.size(), size() / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0), limit_tokens, limit_tokens_cur); for (const auto & state : states) { SRV_TRC(" - prompt %p: %7d tokens, checkpoints: %2zu, %9.3f MiB\n", (const void *)&state, state.prompt.n_tokens(), state.prompt.checkpoints.size(), state.size() / (1024.0 * 1024.0)); } } // // server_slot_sched // static std::vector get_free_slot_infos(const std::vector & slots, size_t n_slots_needed, int exclude_id_slot) { std::vector free_slots; for (const auto & slot : slots) { if (!slot.is_processing && slot.id != exclude_id_slot) { free_slots.push_back(&slot); } } std::stable_sort(free_slots.begin(), free_slots.end(), [](const server_slot_info * a, const server_slot_info * b) { if ((a->prompt->n_tokens() == 0) != (b->prompt->n_tokens() == 0)) { return a->prompt->n_tokens() == 0; } return a->t_last_used < b->t_last_used; }); free_slots.resize(std::min(free_slots.size(), n_slots_needed)); return free_slots; } std::vector server_slot_sched::get_free_slots(const std::vector & slots, size_t n_slots_needed, int exclude_id_slot) const { std::vector res; for (const auto * slot : get_free_slot_infos(slots, n_slots_needed, exclude_id_slot)) { res.push_back(slot->id); } return res; } server_slot_plan server_slot_sched::select(const std::vector & slots, const server_task & task, const std::vector & lora) const { const server_slot_info * ret = nullptr; bool update_cache = false; // if a specific slot is requested, use it (still goes through cache update logic below) if (task.id_slot != -1 && !slots.empty()) { // note: allow id_slot to be out of bounds (wrap around) const int id_slot = task.id_slot % slots.size(); for (const auto & slot : slots) { if (slot.id == id_slot) { ret = &slot; break; } } if (ret) { // a busy slot is returned untouched, the caller defers the task if (ret->is_processing) { return { ret->id, false }; } SLT_INF(*ret, "selected slot by id (%d)\n", task.id_slot); } } // find the slot that has at least n% prompt similarity if (slot_prompt_similarity != 0.0f) { float f_sim_best = 0; for (const auto & slot : slots) { if (task.id_slot != -1 && slot.id != task.id_slot) { continue; } // skip the slot if it is not available if (slot.is_processing) { SLT_TRC(slot, " - skipping, is_processing = %d\n", slot.is_processing); continue; } const auto & tokens = slot.prompt->tokens; // skip the slot if it does not contains cached tokens if (tokens.empty()) { SLT_TRC(slot, "%s", " - skipping, slot is empty\n"); continue; } // the task would clear this slot, its state cannot be reused with another lora config if (lora_should_clear_cache(slot.prompt->lora, lora)) { SLT_TRC(slot, "%s", " - skipping, other lora config\n"); continue; } // fraction of the Longest Common Prefix length with respect to the input prompt length const size_t lcp_len = tokens.get_common_prefix(task.tokens); const float f_sim_cur = float(lcp_len) / task.tokens.size(); SLT_TRC(slot, " - checking sim = %.3f (%zu/%zu) > %.3f\n", f_sim_cur, lcp_len, task.tokens.size(), slot_prompt_similarity); // select the current slot if the criteria match if (f_sim_cur > f_sim_best && f_sim_cur > slot_prompt_similarity) { f_sim_best = f_sim_cur; ret = &slot; } } if (ret != nullptr) { const float f_keep = (f_sim_best*task.tokens.size()) / ret->prompt->tokens.size(); if (task.id_slot == -1) { SLT_INF(*ret, "selected slot by LCP similarity, f_sim_best = %.3f (> %.3f thold), f_keep = %.3f\n", f_sim_best, slot_prompt_similarity, f_keep); } // if we are about to lose a large portion of the existing context - save it in the prompt cache if (f_keep < 0.5f) { update_cache = true; } } } // find an empty slot, or the one that has been least recently used if (ret == nullptr) { const auto free_slots = get_free_slot_infos(slots, 1, -1); if (!free_slots.empty()) { ret = free_slots[0]; SLT_INF(*ret, "selected slot by LRU, n_tokens = %d, t_last = %" PRId64 "\n", ret->prompt->n_tokens(), ret->t_last_used); update_cache = true; } } if (ret == nullptr) { return {}; } // cache prompts only for the tasks that reuse them update_cache = update_cache && !task.is_stateless(); return { ret->id, update_cache }; } // // server_prompt_disk_cache // // file = header + payload; the payload is a list of sections, each padded to 8 bytes: // tokens, checkpoint table, state size, state, the state of each checkpoint // hash_payload is FNV-1a over the 64-bit words of the payload, so a truncated or corrupt file is never used static constexpr uint32_t DISK_CACHE_MAGIC = 0x4443504c; // "LPCD" static constexpr uint32_t DISK_CACHE_VERSION = 1; static constexpr const char * DISK_CACHE_EXT = ".lpcd"; // keep this much space free on the file system static constexpr uint64_t DISK_CACHE_MIN_FREE = 1024ull*1024ull*1024ull; struct disk_cache_header { uint32_t magic; uint32_t version; uint64_t key_hash; uint64_t n_tokens; uint64_t n_ckpt; uint64_t size_payload; uint64_t hash_payload; }; struct disk_cache_ckpt { int64_t n_tokens; llama_pos pos_min; llama_pos pos_max; uint64_t size; }; static_assert(sizeof(disk_cache_header) == 48 && sizeof(disk_cache_ckpt) == 24, "disk cache layout"); static uint64_t disk_cache_pad(uint64_t n) { return (8 - n % 8) % 8; } static uint64_t disk_cache_hash_bytes(const void * data, size_t n) { uint64_t h = 0xcbf29ce484222325ull; for (size_t i = 0; i < n; i++) { h = (h ^ ((const uint8_t *) data)[i]) * 0x100000001b3ull; } return h; } static uint64_t disk_cache_payload_size(uint64_t n_tokens, const std::vector & ckpts, uint64_t n_state) { const auto padded = [](uint64_t n) { return n + disk_cache_pad(n); }; uint64_t res = padded(n_tokens*sizeof(llama_token)) + padded(ckpts.size()*sizeof(disk_cache_ckpt)) + sizeof(uint64_t) + padded(n_state); for (const auto & c : ckpts) { res += padded(c.size); } return res; } // sequential section IO with the running payload hash struct disk_cache_io { FILE * f = nullptr; uint64_t hash = 0xcbf29ce484222325ull; bool ok = true; ~disk_cache_io() { if (f) { fclose(f); } } void add_hash(const uint8_t * p, size_t size, const uint8_t * pad) { const size_t n_full = size - size % 8; for (size_t i = 0; i < n_full; i += 8) { uint64_t w; memcpy(&w, p + i, 8); hash = (hash ^ w) * 0x100000001b3ull; } if (size % 8) { uint64_t w; memcpy(&w, p + n_full, size % 8); memcpy((uint8_t *) &w + size % 8, pad, disk_cache_pad(size)); hash = (hash ^ w) * 0x100000001b3ull; } } void write(const void * src, size_t size) { static const uint8_t zero[8] = {}; const size_t n_pad = disk_cache_pad(size); ok = ok && fwrite(src, 1, size, f) == size && fwrite(zero, 1, n_pad, f) == n_pad; add_hash((const uint8_t *) src, size, zero); } void read(void * dst, size_t size) { uint8_t pad[8]; const size_t n_pad = disk_cache_pad(size); ok = ok && fread(dst, 1, size, f) == size && fread(pad, 1, n_pad, f) == n_pad; if (ok) { add_hash((const uint8_t *) dst, size, pad); } } // the data is in our buffers, the cached pages are waste (see llama_file::drop_cache) void drop_cache() { #if defined(__linux__) posix_fadvise(fileno(f), 0, 0, POSIX_FADV_DONTNEED); #endif } }; // open a file and read everything up to the state data: 1 = entry of this key, 0 = other key or version, -1 = bad file static int disk_cache_open(const std::string & path, uint64_t key_hash, disk_cache_io & io, disk_cache_header & hdr, llama_tokens & tokens, std::vector & ckpts, uint64_t & n_state) { std::error_code ec; const uint64_t size = std::filesystem::file_size(path, ec); io.f = ec ? nullptr : fopen(path.c_str(), "rb"); if (io.f == nullptr || fread(&hdr, 1, sizeof(hdr), io.f) != sizeof(hdr) || hdr.magic != DISK_CACHE_MAGIC) { return -1; } if (hdr.version != DISK_CACHE_VERSION) { return 0; } if (size != sizeof(hdr) + hdr.size_payload || hdr.n_tokens > size/sizeof(llama_token) || hdr.n_ckpt > size/sizeof(disk_cache_ckpt)) { return -1; } if (hdr.key_hash != key_hash) { return 0; } tokens.resize(hdr.n_tokens); ckpts.resize(hdr.n_ckpt); io.read(tokens.data(), tokens.size()*sizeof(llama_token)); io.read(ckpts.data(), ckpts.size()*sizeof(disk_cache_ckpt)); io.read(&n_state, sizeof(n_state)); if (!io.ok || n_state > size) { return -1; } for (const auto & c : ckpts) { if (c.size > size) { return -1; } } return disk_cache_payload_size(hdr.n_tokens, ckpts, n_state) == hdr.size_payload ? 1 : -1; } static void disk_cache_touch(const std::string & path) { std::error_code ec; std::filesystem::last_write_time(path, std::filesystem::file_time_type::clock::now(), ec); } server_prompt_disk_cache::server_prompt_disk_cache(const std::string & dir, int32_t limit_size_mib, const std::string & key) : dir(dir) { key_hash = disk_cache_hash_bytes(key.data(), key.size()); limit_size = 1024ull*1024ull*std::max(0, limit_size_mib); size_t n_other = 0; size_t n_bad = 0; try { std::filesystem::create_directories(dir); const auto now = std::filesystem::file_time_type::clock::now(); for (const auto & de : std::filesystem::directory_iterator(dir)) { const std::string path = de.path().string(); if (!de.is_regular_file()) { continue; } // left by a writer that was killed if (path.find(std::string(DISK_CACHE_EXT) + ".tmp") != std::string::npos) { if (now - de.last_write_time() > std::chrono::hours(1)) { std::filesystem::remove(de.path()); } continue; } if (de.path().extension() != DISK_CACHE_EXT) { continue; } disk_cache_io io; disk_cache_header hdr; server_prompt_disk_entry e; std::vector ckpts; uint64_t n_state = 0; const int res = disk_cache_open(path, key_hash, io, hdr, e.tokens, ckpts, n_state); if (res < 0) { SRV_WRN("disk cache: '%s' is truncated or corrupt, removing it\n", path.c_str()); std::filesystem::remove(de.path()); n_bad++; continue; } if (res == 0) { n_other++; continue; } e.path = path; for (const auto & c : ckpts) { e.ckpt_n_tokens.push_back(c.n_tokens); } entries.push_back(std::move(e)); } } catch (const std::exception & err) { SRV_WRN("disk cache: error reading '%s': %s\n", dir.c_str(), err.what()); } SRV_INF("disk cache: '%s', %zu prompts of this model, %zu other files, %zu bad files removed, limit %.0f MiB\n", dir.c_str(), entries.size(), n_other, n_bad, limit_size / (1024.0 * 1024.0)); } size_t server_prompt_disk_cache::n_reuse(const llama_tokens & tokens, const std::vector & ckpt_n_tokens, const llama_tokens & tokens_new, bool can_trim) { size_t lcp = 0; while (lcp < tokens.size() && lcp < tokens_new.size() && tokens[lcp] == tokens_new[lcp]) { lcp++; } if (lcp == tokens.size() || can_trim) { return lcp; } size_t res = 0; for (const int64_t n : ckpt_n_tokens) { if (n <= (int64_t) lcp) { res = std::max(res, (size_t) n); } } return res; } int server_prompt_disk_cache::find(const llama_tokens & tokens_new, size_t n_have, size_t n_min, bool can_trim) const { int best = -1; size_t n_best = 0; for (size_t i = 0; i < entries.size(); i++) { const size_t n = n_reuse(entries[i].tokens, entries[i].ckpt_n_tokens, tokens_new, can_trim); if (n >= n_have + n_min && n > n_best) { best = (int) i; n_best = n; } } return best; } bool server_prompt_disk_cache::has(const llama_tokens & tokens) { for (const auto & e : entries) { std::error_code ec; if (e.tokens == tokens && std::filesystem::exists(e.path, ec)) { disk_cache_touch(e.path); return true; } } return false; } void server_prompt_disk_cache::remove(int i) { std::error_code ec; std::filesystem::remove(entries[i].path, ec); entries.erase(entries.begin() + i); } bool server_prompt_disk_cache::load(int i, server_prompt & prompt, common_state_data & data, bool has_mtmd) { const std::string path = entries[i].path; disk_cache_io io; disk_cache_header hdr; llama_tokens tokens; std::vector ckpts; uint64_t n_state = 0; std::list> checkpoints; bool ok = disk_cache_open(path, key_hash, io, hdr, tokens, ckpts, n_state) == 1 && tokens == entries[i].tokens; if (ok) { try { data.resize(n_state); io.read(data.data(), data.size()); for (const auto & c : ckpts) { auto cur = std::make_shared(); cur->id_task = -1; // not created by a task: may fail to load, then the prompt is re-processed cur->n_tokens = c.n_tokens; cur->pos_min = c.pos_min; cur->pos_max = c.pos_max; cur->data_tgt.resize(c.size); io.read(cur->data_tgt.data(), cur->data_tgt.size()); checkpoints.push_back(std::move(cur)); } } catch (const std::bad_alloc &) { SRV_WRN("disk cache: out of memory reading '%s'\n", path.c_str()); data.clear(); return false; } ok = io.ok && io.hash == hdr.hash_payload; io.drop_cache(); } if (!ok) { SRV_WRN("disk cache: '%s' is missing, truncated or corrupt, removing it\n", path.c_str()); data.clear(); remove(i); return false; } disk_cache_touch(path); prompt.tokens = server_tokens(tokens, has_mtmd); prompt.checkpoints = std::move(checkpoints); return true; } bool server_prompt_disk_cache::save(const server_prompt & prompt, const common_state_data & data) { const llama_tokens & tokens = prompt.tokens.get_tokens(); std::vector ckpts; for (const auto & c : prompt.checkpoints) { ckpts.push_back({ c->n_tokens, c->pos_min, c->pos_max, c->data_tgt.size() }); } disk_cache_header hdr = { /*.magic =*/ DISK_CACHE_MAGIC, /*.version =*/ DISK_CACHE_VERSION, /*.key_hash =*/ key_hash, /*.n_tokens =*/ tokens.size(), /*.n_ckpt =*/ ckpts.size(), /*.size_payload =*/ disk_cache_payload_size(tokens.size(), ckpts, data.size()), /*.hash_payload =*/ 0, }; const uint64_t size_file = sizeof(hdr) + hdr.size_payload; char name[64]; snprintf(name, sizeof(name), "%016llx-%016llx", (unsigned long long) key_hash, (unsigned long long) disk_cache_hash_bytes(tokens.data(), tokens.size()*sizeof(llama_token))); const std::string path = (std::filesystem::path(dir) / name).string() + DISK_CACHE_EXT; const std::string path_tmp = path + ".tmp"; // entries contained in the new one are obsolete: removed once it is written, not counted for the limit const auto is_obsolete = [&](const server_prompt_disk_entry & e) { return e.path != path && e.tokens.size() <= tokens.size() && std::equal(e.tokens.begin(), e.tokens.end(), tokens.begin()); }; std::vector obsolete; for (const auto & e : entries) { if (is_obsolete(e)) { obsolete.push_back(e.path); } } if (limit_size > 0 && size_file > limit_size) { SRV_WRN("disk cache: prompt state of %.1f MiB exceeds the limit of %.1f MiB, not saved\n", size_file / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0)); return false; } try { // evict the least recently used files of every model in dir struct file_info { std::filesystem::path path; uint64_t size; std::filesystem::file_time_type time; }; std::vector files; uint64_t size_total = 0; for (const auto & de : std::filesystem::directory_iterator(dir)) { if (de.is_regular_file() && de.path().extension() == DISK_CACHE_EXT && de.path().string() != path && std::find(obsolete.begin(), obsolete.end(), de.path().string()) == obsolete.end()) { files.push_back({ de.path(), de.file_size(), de.last_write_time() }); size_total += files.back().size; } } std::sort(files.begin(), files.end(), [](const file_info & a, const file_info & b) { return a.time < b.time; }); for (size_t i = 0; limit_size > 0 && size_total + size_file > limit_size && i < files.size(); i++) { SRV_INF("disk cache: size limit, removing '%s' (%.1f MiB)\n", files[i].path.string().c_str(), files[i].size / (1024.0 * 1024.0)); std::filesystem::remove(files[i].path); size_total -= files[i].size; for (size_t j = 0; j < entries.size(); j++) { if (entries[j].path == files[i].path.string()) { entries.erase(entries.begin() + j); break; } } } const uint64_t avail = std::filesystem::space(dir).available; if (avail < size_file + DISK_CACHE_MIN_FREE) { SRV_WRN("disk cache: only %.1f MiB free in '%s', prompt state of %.1f MiB not saved\n", avail / (1024.0 * 1024.0), dir.c_str(), size_file / (1024.0 * 1024.0)); return false; } } catch (const std::exception & err) { SRV_WRN("disk cache: error reading '%s': %s\n", dir.c_str(), err.what()); return false; } bool ok; { disk_cache_io io; io.f = fopen(path_tmp.c_str(), "wb"); ok = io.f != nullptr && fwrite(&hdr, 1, sizeof(hdr), io.f) == sizeof(hdr); if (ok) { const uint64_t n_state = data.size(); io.write(tokens.data(), tokens.size()*sizeof(llama_token)); io.write(ckpts.data(), ckpts.size()*sizeof(disk_cache_ckpt)); io.write(&n_state, sizeof(n_state)); io.write(data.data(), data.size()); for (const auto & c : prompt.checkpoints) { io.write(c->data_tgt.data(), c->data_tgt.size()); } hdr.hash_payload = io.hash; ok = io.ok && fseek(io.f, 0, SEEK_SET) == 0 && fwrite(&hdr, 1, sizeof(hdr), io.f) == sizeof(hdr) && fflush(io.f) == 0; #if defined(__linux__) // write back now, so the pages can be dropped and the rename never exposes missing data ok = ok && fdatasync(fileno(io.f)) == 0; io.drop_cache(); #endif } if (io.f != nullptr && fclose(io.f) != 0) { ok = false; } io.f = nullptr; if (!ok) { SRV_WRN("disk cache: failed to write '%s': %s, not saved\n", path_tmp.c_str(), strerror(errno)); } } std::error_code ec; if (ok) { std::filesystem::rename(path_tmp, path, ec); ok = !ec; if (!ok) { SRV_WRN("disk cache: failed to rename '%s': %s, not saved\n", path_tmp.c_str(), ec.message().c_str()); } } if (!ok) { std::filesystem::remove(path_tmp, ec); return false; } for (size_t i = 0; i < entries.size(); ) { const auto & e = entries[i]; if (e.path == path) { entries.erase(entries.begin() + i); } else if (is_obsolete(e)) { SRV_INF("disk cache: removing '%s', a prefix of the new prompt\n", e.path.c_str()); remove((int) i); } else { i++; } } server_prompt_disk_entry e; e.path = path; e.tokens = tokens; for (const auto & c : ckpts) { e.ckpt_n_tokens.push_back(c.n_tokens); } entries.push_back(std::move(e)); return true; }