server: slot_sched: fix lora and recurrent (#32434)
* server: fix lora and recurrent * tests: run the hybrid SWA checkpoint test with one token per ubatch With and without the cache, the prompt was not split into the same ubatches, and the generated tokens are decoded one at a time while the uncached run processes them in a batch. On CUDA this moves the top logprobs of the tiny model by up to 0.03 and swaps near ties, so the test failed while the cache behaved as expected. With one token per ubatch both runs go through the same kernels and the output is compared exactly. --------- Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
1 parent
3574df84b7
commit
52975342ff
8 files changed
+244
-24
No files matched your search
@@ -4,6 +4,7 @@
|
||||
#include "../tools/server/server-common.h"
|
||||
#include "../tools/server/server-decision.h"
|
||||
#include "../tools/server/server-schema.h"
|
||||
#include "../tools/server/server-slot-sched.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
@@ -330,6 +331,82 @@ static void test_json_helpers(testing & t) {
|
||||
});
|
||||
}
|
||||
|
||||
//
|
||||
// server_prompt
|
||||
//
|
||||
|
||||
static void test_server_prompt_reuse(testing & t) {
|
||||
// 110 text tokens, the position of a token is its index
|
||||
auto make_prompt = [](std::vector<std::array<llama_pos, 2>> checkpoints) {
|
||||
llama_tokens toks(110);
|
||||
for (size_t i = 0; i < toks.size(); i++) {
|
||||
toks[i] = 100 + (llama_token) i;
|
||||
}
|
||||
|
||||
server_prompt res;
|
||||
res.tokens = server_tokens(toks, false);
|
||||
for (const auto & range : checkpoints) {
|
||||
auto ckpt = std::make_shared<common_prompt_checkpoint>();
|
||||
ckpt->pos_min = range[0];
|
||||
ckpt->pos_max = range[1];
|
||||
ckpt->n_tokens = range[1] + 1;
|
||||
res.checkpoints.push_back(std::move(ckpt));
|
||||
}
|
||||
return res;
|
||||
};
|
||||
|
||||
t.test("recurrent", [&](testing & t) {
|
||||
// a recurrent state holds one position: the live state is at 109, n_swa is 0
|
||||
const auto prompt = make_prompt({{95, 95}, {99, 99}, {105, 105}});
|
||||
const llama_pos pos_min = 109;
|
||||
|
||||
auto res = prompt.get_reuse(110, true, 0, pos_min);
|
||||
t.assert_true("new tokens after the prompt, no checkpoint", res.ckpt == nullptr);
|
||||
t.assert_equal("new tokens after the prompt, n_past", 110, res.n_past);
|
||||
|
||||
res = prompt.get_reuse(110, false, 0, pos_min);
|
||||
t.assert_true("same prompt, last checkpoint", res.ckpt && res.ckpt->pos_max == 105);
|
||||
t.assert_equal("same prompt, n_past", 106, res.n_past);
|
||||
t.assert_equal("same prompt, pos_next", 106, res.pos_next);
|
||||
|
||||
// the closest checkpoint before the new tokens is used, the one after them is not
|
||||
res = prompt.get_reuse(104, true, 0, pos_min);
|
||||
t.assert_true("cut at 104, checkpoint", res.ckpt && res.ckpt->pos_max == 99);
|
||||
t.assert_equal("cut at 104, n_past", 100, res.n_past);
|
||||
t.assert_equal("cut at 104, pos_next", 100, res.pos_next);
|
||||
|
||||
// the state at 99 already holds the token at 99, it cannot be used to process it again
|
||||
res = prompt.get_reuse(99, true, 0, pos_min);
|
||||
t.assert_true("cut at 99, checkpoint", res.ckpt && res.ckpt->pos_max == 95);
|
||||
t.assert_equal("cut at 99, n_past", 96, res.n_past);
|
||||
|
||||
res = prompt.get_reuse(50, true, 0, pos_min);
|
||||
t.assert_true("cut at 50, no checkpoint", res.ckpt == nullptr);
|
||||
t.assert_equal("cut at 50, n_past", 0, res.n_past);
|
||||
t.assert_equal("cut at 50, pos_next", 0, res.pos_next);
|
||||
});
|
||||
|
||||
t.test("swa", [&](testing & t) {
|
||||
// n_swa is 8: the live state starts at 102, the checkpoint holds the positions 92 to 99
|
||||
const auto prompt = make_prompt({{92, 99}});
|
||||
const llama_pos pos_min = 102;
|
||||
|
||||
auto res = prompt.get_reuse(110, true, 8, pos_min);
|
||||
t.assert_true("new tokens after the prompt, no checkpoint", res.ckpt == nullptr);
|
||||
t.assert_equal("new tokens after the prompt, n_past", 110, res.n_past);
|
||||
|
||||
// the token at 104 needs the positions 97 to 103, the live state starts too late for a cut at 104
|
||||
res = prompt.get_reuse(104, true, 8, pos_min);
|
||||
t.assert_true("cut at 104, checkpoint", res.ckpt != nullptr);
|
||||
t.assert_equal("cut at 104, n_past", 99, res.n_past);
|
||||
t.assert_equal("cut at 104, pos_next", 99, res.pos_next);
|
||||
|
||||
res = prompt.get_reuse(95, true, 8, pos_min);
|
||||
t.assert_true("cut at 95, no checkpoint", res.ckpt == nullptr);
|
||||
t.assert_equal("cut at 95, n_past", 0, res.n_past);
|
||||
});
|
||||
}
|
||||
|
||||
static void test_lora_helpers(testing & t) {
|
||||
// only pointer identity is used, never dereferenced
|
||||
auto * p0 = reinterpret_cast<llama_adapter_lora *>(0x10);
|
||||
@@ -364,6 +441,13 @@ static void test_lora_helpers(testing & t) {
|
||||
t.assert_true("diff size", !are_lora_equal({make(p0, 1.0f)}, {}));
|
||||
});
|
||||
|
||||
t.test("lora_should_clear_cache", [&](testing & t) {
|
||||
// the adapter pointer is only read when the configs differ
|
||||
t.assert_true("both empty", !lora_should_clear_cache({}, {}));
|
||||
t.assert_true("same", !lora_should_clear_cache({make(p0, 1.0f)}, {make(p0, 1.0f)}));
|
||||
t.assert_true("to no lora", lora_should_clear_cache({make(p0, 0.0f)}, {}));
|
||||
});
|
||||
|
||||
t.test("lora_get_enabled_ids", [&](testing & t) {
|
||||
auto ids = lora_get_enabled_ids({make(p0, 0.0f), make(p1, 0.5f), make(p0, -1.0f), make(p1, 2.0f)});
|
||||
t.assert_true("ids", ids == std::vector<size_t>({1, 3}));
|
||||
@@ -869,6 +953,7 @@ int main(int argc, char ** argv) {
|
||||
t.test("sse", test_sse);
|
||||
t.test("json_helpers", test_json_helpers);
|
||||
t.test("lora_helpers", test_lora_helpers);
|
||||
t.test("server_prompt_reuse", test_server_prompt_reuse);
|
||||
t.test("server_pipe", test_server_pipe);
|
||||
t.test("decision_escape", test_decision_escape);
|
||||
t.test("rerank_template", test_rerank_template);
|
||||
|
||||
Reference in new issue
Block a user