server : clear slot cache when the global lora config changes (#30367)

This commit is contained in:
Aayan Karmali authored and GitHub committed 2026-11-15 09:05:10 +01:00
1 parent 5a4afd8077
commit 58a4a6b49c
2 files changed
+66 -14

No files matched your search

+10 -14
View File
@@ -1955,21 +1955,17 @@ private:
}
bool launch_slot_with_task(server_slot & slot, server_task && task) {
// process per-request lora adapters
if (!task.params.lora.empty()) {
auto task_loras = construct_lora_list(task.params.lora);
if (!are_lora_equal(task_loras, slot.lora)) {
// if lora has changed, check to see if the cache should be cleared
if (lora_should_clear_cache(slot.lora, task_loras)) {
SLT_TRC(slot, "clearing cache for lora change. %zu loras -> %zu loras\n", slot.lora.size(), task.params.lora.size());
slot.prompt.clear();
} else {
SLT_TRC(slot, "keeping cache for alora. %zu target loras\n", task_loras.size());
}
slot.lora = task_loras;
// per-request lora if given, otherwise the global config
auto task_loras = task.params.lora.empty() ? params_base.lora_adapters : construct_lora_list(task.params.lora);
if (!are_lora_equal(task_loras, slot.lora)) {
// if lora has changed, check to see if the cache should be cleared
if (lora_should_clear_cache(slot.lora, task_loras)) {
SLT_TRC(slot, "clearing cache for lora change. %zu loras -> %zu loras\n", slot.lora.size(), task_loras.size());
slot.prompt.clear();
} else {
SLT_TRC(slot, "keeping cache for alora. %zu target loras\n", task_loras.size());
}
} else {
slot.lora = params_base.lora_adapters;
slot.lora = task_loras;
}
// if using alora, make sure it's only a single one requested and active