server: slot_sched: fix lora and recurrent (#32434)
* server: fix lora and recurrent * tests: run the hybrid SWA checkpoint test with one token per ubatch With and without the cache, the prompt was not split into the same ubatches, and the generated tokens are decoded one at a time while the uncached run processes them in a batch. On CUDA this moves the top logprobs of the tiny model by up to 0.03 and swaps near ties, so the test failed while the cache behaved as expected. With one token per ubatch both runs go through the same kernels and the output is compared exactly. --------- Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
1 parent
3574df84b7
commit
52975342ff
8 files changed
+244
-24
No files matched your search
@@ -370,8 +370,8 @@ struct server_slot {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool prompt_load(server_prompt_cache & prompt_cache, const server_tokens & tokens) {
|
||||
bool res = prompt_cache.load(prompt, tokens, ctx_tgt, ctx_dft, id);
|
||||
bool prompt_load(server_prompt_cache & prompt_cache, const server_tokens & tokens, const std::vector<common_adapter_lora_info> & lora_new) {
|
||||
bool res = prompt_cache.load(prompt, tokens, lora_new, ctx_tgt, ctx_dft, id);
|
||||
if (!res) {
|
||||
SLT_WRN(*this, "%s", "failed to load prompt from cache\n");
|
||||
}
|
||||
@@ -1108,6 +1108,9 @@ private:
|
||||
// if swa_full is enabled, this is set to 0 to simulate a non-SWA model
|
||||
int32_t n_swa;
|
||||
|
||||
// n_swa for server_prompt::get_reuse(): 0 with a recurrent state, which holds one position only
|
||||
int32_t n_swa_reuse;
|
||||
|
||||
// slots / clients
|
||||
std::vector<server_slot> slots;
|
||||
|
||||
@@ -1444,6 +1447,9 @@ private:
|
||||
|
||||
n_swa = params_base.swa_full ? 0 : llama_model_n_swa(model_tgt);
|
||||
|
||||
// the min position of a hybrid memory is the position of its recurrent state, not the start of the SWA window
|
||||
n_swa_reuse = llama_model_is_recurrent(model_tgt) || llama_model_is_hybrid(model_tgt) ? 0 : n_swa;
|
||||
|
||||
// Necessary similarity of prompt for slot selection
|
||||
slot_sched.slot_prompt_similarity = params_base.slot_prompt_similarity;
|
||||
|
||||
@@ -1824,7 +1830,7 @@ private:
|
||||
|
||||
slot.prompt_save(*prompt_cache);
|
||||
|
||||
if (!slot.prompt_load(*prompt_cache, task.tokens)) {
|
||||
if (!slot.prompt_load(*prompt_cache, task.tokens, get_task_lora(task))) {
|
||||
slot.prompt_clear();
|
||||
}
|
||||
|
||||
@@ -1878,19 +1884,25 @@ private:
|
||||
return output;
|
||||
}
|
||||
|
||||
// per-request lora if given, otherwise the global config
|
||||
std::vector<common_adapter_lora_info> get_task_lora(const server_task & task) const {
|
||||
return task.params.lora.empty() ? params_base.lora_adapters : construct_lora_list(task.params.lora);
|
||||
}
|
||||
|
||||
bool launch_slot_with_task(server_slot & slot, server_task && task) {
|
||||
// per-request lora if given, otherwise the global config
|
||||
auto task_loras = task.params.lora.empty() ? params_base.lora_adapters : construct_lora_list(task.params.lora);
|
||||
if (!are_lora_equal(task_loras, slot.lora)) {
|
||||
auto task_loras = get_task_lora(task);
|
||||
|
||||
// the state can come from the prompt cache, so it is compared with its own lora config, not with the one of the last task
|
||||
if (!are_lora_equal(task_loras, slot.prompt.lora)) {
|
||||
// if lora has changed, check to see if the cache should be cleared
|
||||
if (lora_should_clear_cache(slot.lora, task_loras)) {
|
||||
SLT_TRC(slot, "clearing cache for lora change. %zu loras -> %zu loras\n", slot.lora.size(), task_loras.size());
|
||||
if (lora_should_clear_cache(slot.prompt.lora, task_loras)) {
|
||||
SLT_TRC(slot, "clearing cache for lora change. %zu loras -> %zu loras\n", slot.prompt.lora.size(), task_loras.size());
|
||||
slot.prompt.clear();
|
||||
} else {
|
||||
SLT_TRC(slot, "keeping cache for alora. %zu target loras\n", task_loras.size());
|
||||
}
|
||||
slot.lora = task_loras;
|
||||
}
|
||||
slot.lora = task_loras;
|
||||
|
||||
// if using alora, make sure it's only a single one requested and active
|
||||
size_t alora_invocation_start = task.tokens.size();
|
||||
@@ -1940,6 +1952,8 @@ private:
|
||||
}
|
||||
}
|
||||
|
||||
slot.prompt.lora = slot.lora;
|
||||
|
||||
if (!task.tokens.validate(ctx_tgt)) {
|
||||
send_error(task, "Prompt contains invalid tokens", ERROR_TYPE_INVALID_REQUEST);
|
||||
return false;
|
||||
@@ -2956,7 +2970,7 @@ private:
|
||||
|
||||
const int id_task = task.id;
|
||||
|
||||
const auto plan = slot_sched.select(get_slot_infos(), task);
|
||||
const auto plan = slot_sched.select(get_slot_infos(), task, get_task_lora(task));
|
||||
|
||||
server_slot * slot = plan.id_slot == -1 ? nullptr : get_slot_by_id(plan.id_slot);
|
||||
|
||||
@@ -4186,7 +4200,7 @@ private:
|
||||
SLT_WRN(slot, "%s\n", st1.str().c_str());
|
||||
}
|
||||
|
||||
auto reuse = slot.prompt.get_reuse(n_past, has_new_tokens, n_swa, pos_min);
|
||||
auto reuse = slot.prompt.get_reuse(n_past, has_new_tokens, n_swa_reuse, pos_min);
|
||||
|
||||
if (reuse.ckpt) {
|
||||
const auto & cur = *reuse.ckpt;
|
||||
|
||||
Reference in new issue
Block a user