llama-server: Use reference-counted checkpoints instead of deep-copies (#30137)

* checkpoints: reduce overhead

Assisted-by: Codex

* rebase

Adapt the slot save/restore checkpoint appendix from #26004 to the
shared_ptr checkpoint list and the common_state_data buffers.

Assisted-by: Claude Opus 5.5

---------

Co-authored-by: cwriter <cwriter@localhost>
This commit is contained in:
cwriterandcwriter authored and GitHub committed 2026-11-05 11:02:36 +01:00
1 parent 1fa7f1c488
commit e152332a15
4 files changed
+72 -41

No files matched your search

+3 -3
View File
@@ -1767,7 +1767,7 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro
// calculate checkpoints size to see if it will fit with the prompt
size_t checkpoints_size = 0;
for (const auto & ckpt : prompt.checkpoints) {
checkpoints_size += ckpt.size();
checkpoints_size += ckpt->size();
}
const size_t state_size_new = state_size_tgt + state_size_dft + checkpoints_size;
@@ -1802,8 +1802,8 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro
}
}
std::vector<uint8_t> state_data_tgt;
std::vector<uint8_t> state_data_dft;
common_state_data state_data_tgt;
common_state_data state_data_dft;
// check if we can allocate enough memory for the new state
try {