server : clear slot cache when the global lora config changes (#30367)
This commit is contained in:
1 parent
5a4afd8077
commit
58a4a6b49c
2 files changed
+66
-14
No files matched your search
@@ -1955,21 +1955,17 @@ private:
|
||||
}
|
||||
|
||||
bool launch_slot_with_task(server_slot & slot, server_task && task) {
|
||||
// process per-request lora adapters
|
||||
if (!task.params.lora.empty()) {
|
||||
auto task_loras = construct_lora_list(task.params.lora);
|
||||
if (!are_lora_equal(task_loras, slot.lora)) {
|
||||
// if lora has changed, check to see if the cache should be cleared
|
||||
if (lora_should_clear_cache(slot.lora, task_loras)) {
|
||||
SLT_TRC(slot, "clearing cache for lora change. %zu loras -> %zu loras\n", slot.lora.size(), task.params.lora.size());
|
||||
slot.prompt.clear();
|
||||
} else {
|
||||
SLT_TRC(slot, "keeping cache for alora. %zu target loras\n", task_loras.size());
|
||||
}
|
||||
slot.lora = task_loras;
|
||||
// per-request lora if given, otherwise the global config
|
||||
auto task_loras = task.params.lora.empty() ? params_base.lora_adapters : construct_lora_list(task.params.lora);
|
||||
if (!are_lora_equal(task_loras, slot.lora)) {
|
||||
// if lora has changed, check to see if the cache should be cleared
|
||||
if (lora_should_clear_cache(slot.lora, task_loras)) {
|
||||
SLT_TRC(slot, "clearing cache for lora change. %zu loras -> %zu loras\n", slot.lora.size(), task_loras.size());
|
||||
slot.prompt.clear();
|
||||
} else {
|
||||
SLT_TRC(slot, "keeping cache for alora. %zu target loras\n", task_loras.size());
|
||||
}
|
||||
} else {
|
||||
slot.lora = params_base.lora_adapters;
|
||||
slot.lora = task_loras;
|
||||
}
|
||||
|
||||
// if using alora, make sure it's only a single one requested and active
|
||||
|
||||
Reference in new issue
Block a user