llama-server: Use reference-counted checkpoints instead of deep-copies (#30137)
* checkpoints: reduce overhead Assisted-by: Codex * rebase Adapt the slot save/restore checkpoint appendix from #26004 to the shared_ptr checkpoint list and the common_state_data buffers. Assisted-by: Claude Opus 5.5 --------- Co-authored-by: cwriter <cwriter@localhost>
This commit is contained in:
1 parent
1fa7f1c488
commit
e152332a15
4 files changed
+72
-41
No files matched your search
@@ -1767,7 +1767,7 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro
|
||||
// calculate checkpoints size to see if it will fit with the prompt
|
||||
size_t checkpoints_size = 0;
|
||||
for (const auto & ckpt : prompt.checkpoints) {
|
||||
checkpoints_size += ckpt.size();
|
||||
checkpoints_size += ckpt->size();
|
||||
}
|
||||
|
||||
const size_t state_size_new = state_size_tgt + state_size_dft + checkpoints_size;
|
||||
@@ -1802,8 +1802,8 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<uint8_t> state_data_tgt;
|
||||
std::vector<uint8_t> state_data_dft;
|
||||
common_state_data state_data_tgt;
|
||||
common_state_data state_data_dft;
|
||||
|
||||
// check if we can allocate enough memory for the new state
|
||||
try {
|
||||
|
||||
Reference in new issue
Block a user