Files
llama.cpp/tests/test-server-component.cpp
T
Xuan-Son NguyenandPascal 52975342ff server: slot_sched: fix lora and recurrent (#32434)
* server: fix lora and recurrent

* tests: run the hybrid SWA checkpoint test with one token per ubatch

With and without the cache, the prompt was not split into the same
ubatches, and the generated tokens are decoded one at a time while the
uncached run processes them in a batch. On CUDA this moves the top
logprobs of the tiny model by up to 0.03 and swaps near ties, so the
test failed while the cache behaved as expected.

With one token per ubatch both runs go through the same kernels and
the output is compared exactly.

---------

Co-authored-by: Pascal <admin@serveurperso.com>
2026-11-18 00:12:20 +01:00

971 lines
40 KiB
C++

#include "testing.h"
#include "../tools/server/server-chat.h"
#include "../tools/server/server-common.h"
#include "../tools/server/server-decision.h"
#include "../tools/server/server-schema.h"
#include "../tools/server/server-slot-sched.h"
#include <algorithm>
#include <cmath>
#include <cstdlib>
#include <cstring>
#include <regex>
#include <string>
#include <thread>
#include <utility>
#include <vector>
// unit tests for server internal components, no model required
// NOTE: end-to-end tests live in tools/server/tests
// see skills/code-review/SKILL.md for more
//
// server_tokens (text only, has_mtmd = false)
//
static void test_server_tokens_basic(testing & t) {
server_tokens st(llama_tokens{1, 2, 3}, false);
t.assert_equal("size", (size_t) 3, st.size());
t.assert_true("not empty", !st.empty());
t.assert_equal("operator[]", 2, st[1]);
st.push_back(4);
t.assert_equal("push_back", (size_t) 4, st.size());
t.assert_true("get_tokens", st.get_tokens() == llama_tokens({1, 2, 3, 4}));
st.insert({5, 6});
t.assert_true("insert appends", st.get_tokens() == llama_tokens({1, 2, 3, 4, 5, 6}));
st.set_token(0, 9);
t.assert_equal("set_token", 9, st[0]);
bool thrown = false;
try {
st.push_back(LLAMA_TOKEN_NULL);
} catch (const std::exception &) {
thrown = true;
}
t.assert_true("push_back(LLAMA_TOKEN_NULL) throws", thrown);
t.assert_equal("size unchanged after throw", (size_t) 6, st.size());
st.clear();
t.assert_true("clear", st.empty());
}
static void test_server_tokens_pos(testing & t) {
server_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
t.assert_equal("pos_next()", (llama_pos) 5, st.pos_next());
t.assert_equal("pos_next(0)", (llama_pos) 0, st.pos_next(0));
t.assert_equal("pos_next(3)", (llama_pos) 3, st.pos_next(3));
t.assert_equal("size_up_to_pos(0)", (size_t) 0, st.size_up_to_pos(0));
t.assert_equal("size_up_to_pos(3)", (size_t) 3, st.size_up_to_pos(3));
t.assert_equal("size_up_to_pos clamps", (size_t) 5, st.size_up_to_pos(100));
}
static void test_server_tokens_keep_first(testing & t) {
server_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
st.keep_first(5);
t.assert_equal("keep all", (size_t) 5, st.size());
st.keep_first(2);
t.assert_true("keep 2", st.get_tokens() == llama_tokens({1, 2}));
st.keep_first(0);
t.assert_true("keep 0", st.empty());
}
static void test_server_tokens_common_prefix(testing & t) {
server_tokens a(llama_tokens{1, 2, 3, 4}, false);
server_tokens empty(llama_tokens{}, false);
t.assert_equal("both empty", (size_t) 0, empty.get_common_prefix(empty));
t.assert_equal("one empty", (size_t) 0, a.get_common_prefix(empty));
t.assert_equal("identical", (size_t) 4, a.get_common_prefix(a));
t.assert_equal("diverge at 2", (size_t) 2, a.get_common_prefix(server_tokens(llama_tokens{1, 2, 9, 4}, false)));
t.assert_equal("diverge at 0", (size_t) 0, a.get_common_prefix(server_tokens(llama_tokens{9, 2, 3, 4}, false)));
t.assert_equal("b is prefix", (size_t) 2, a.get_common_prefix(server_tokens(llama_tokens{1, 2}, false)));
t.assert_equal("a is prefix", (size_t) 4, a.get_common_prefix(server_tokens(llama_tokens{1, 2, 3, 4, 5, 6}, false)));
}
static void test_server_tokens_serialize(testing & t) {
// same packing as slot save/restore in server-context.cpp
auto pack = [](const std::vector<char> & data) {
llama_tokens packed(data.size() / sizeof(llama_token));
memcpy(packed.data(), data.data(), data.size());
return packed;
};
t.test("roundtrip", [&](testing & t) {
server_tokens st(llama_tokens{10, 20, 30}, false);
std::vector<char> data = st.serialize();
t.assert_equal("aligned to llama_token", (size_t) 0, data.size() % sizeof(llama_token));
server_tokens restored = server_tokens::deserialize(pack(data), false);
t.assert_true("tokens", restored.get_tokens() == st.get_tokens());
});
t.test("roundtrip empty", [&](testing & t) {
server_tokens st(llama_tokens{}, false);
server_tokens restored = server_tokens::deserialize(pack(st.serialize()), false);
t.assert_true("empty", restored.empty());
});
t.test("legacy plain token list", [&](testing & t) {
server_tokens restored = server_tokens::deserialize(llama_tokens{7, 8, 9}, false);
t.assert_true("tokens", restored.get_tokens() == llama_tokens({7, 8, 9}));
});
auto expect_throw = [](testing & t, const std::string & msg, const llama_tokens & packed) {
bool thrown = false;
try {
server_tokens::deserialize(packed, false);
} catch (const std::exception &) {
thrown = true;
}
t.assert_true(msg, thrown);
};
t.test("corrupted", [&](testing & t) {
llama_tokens good = pack(server_tokens(llama_tokens{10, 20, 30}, false).serialize());
llama_tokens bad_version = good;
bad_version[1] = 999;
expect_throw(t, "bad version", bad_version);
llama_tokens truncated = good;
truncated.pop_back();
expect_throw(t, "truncated", truncated);
llama_tokens trailing = good;
trailing.push_back(42);
expect_throw(t, "trailing data", trailing);
llama_tokens huge_count = good;
huge_count[2] = 0x7fffffff;
expect_throw(t, "huge token count", huge_count);
expect_throw(t, "marker only", llama_tokens{LLAMA_TOKEN_NULL});
});
}
static void test_server_tokens_clone(testing & t) {
server_tokens st(llama_tokens{1, 2, 3}, false);
server_tokens cl = st.clone();
cl.push_back(4);
t.assert_equal("original untouched", (size_t) 3, st.size());
t.assert_equal("clone grew", (size_t) 4, cl.size());
t.assert_true("get_text_tokens", cl.get_text_tokens() == llama_tokens({1, 2, 3, 4}));
}
static void test_server_tokens_prefix_hashes(testing & t) {
const size_t n_blk = server_tokens::hash_block_size;
auto make_tokens = [](size_t n) {
llama_tokens res(n);
for (size_t i = 0; i < n; i++) {
res[i] = 100 + (llama_token) i;
}
return res;
};
{
server_tokens a(make_tokens(3 * n_blk + 10), false);
t.assert_equal("full blocks only", (size_t) 3, a.get_prefix_hashes().size());
t.assert_equal("same tokens", 3 * n_blk, a.get_common_prefix_lower(a.clone().get_prefix_hashes()));
t.assert_equal("no hashes", (size_t) 0, a.get_common_prefix_lower({}));
// the exact value is 2 * n_blk + 5
llama_tokens toks = make_tokens(3 * n_blk + 10);
toks[2 * n_blk + 5] = 7;
server_tokens b(toks, false);
t.assert_equal("diverge in block 2", 2 * n_blk, a.get_common_prefix_lower(b.get_prefix_hashes()));
t.assert_equal("exact, diverge in block 2", 2 * n_blk + 5, a.get_common_prefix(b));
t.assert_equal("exact, same tokens", 3 * n_blk + 10, a.get_common_prefix(a.clone()));
t.assert_equal("b is shorter", 1 * n_blk, a.get_common_prefix_lower(server_tokens(make_tokens(n_blk + 3), false).get_prefix_hashes()));
toks[0] = 7;
t.assert_equal("diverge in block 0", (size_t) 0, a.get_common_prefix_lower(server_tokens(toks, false).get_prefix_hashes()));
}
{
server_tokens a(make_tokens(3 * n_blk), false);
const auto hashes = a.get_prefix_hashes();
server_tokens b = a.clone();
b.set_token(n_blk + 1, 7);
t.assert_equal("set_token", n_blk, b.get_common_prefix_lower(hashes));
b = a.clone();
b.keep_first(n_blk + 1);
for (size_t i = n_blk + 1; i < 3 * n_blk; i++) {
b.push_back(7);
}
t.assert_equal("keep_first + push_back", n_blk, b.get_common_prefix_lower(hashes));
b.clear();
t.assert_equal("clear", (size_t) 0, b.get_prefix_hashes().size());
}
{
// the image chunk starts in block 0 and ends in a later block
mtmd::input_chunks chunks(mtmd_test_create_input_chunks());
const mtmd_input_chunk * image = chunks[1];
const size_t n_image = mtmd_input_chunk_get_n_tokens(image);
const size_t i_image = n_blk - 4;
auto make_mtmd = [&](bool with_image, llama_token last) {
server_tokens res(make_tokens(i_image), true);
if (with_image) {
res.push_back(image);
}
while (res.size() < i_image + n_image) {
res.push_back(5);
}
res.push_back(last);
return res;
};
server_tokens a = make_mtmd(true, 1);
t.assert_true("image is longer than a block", n_image > n_blk);
t.assert_equal("exact value", i_image + n_image, a.get_common_prefix(make_mtmd(true, 2)));
t.assert_equal("block ends inside the image", i_image, a.get_common_prefix_lower(make_mtmd(true, 2).get_prefix_hashes()));
t.assert_equal("text in place of the image", (size_t) 0, a.get_common_prefix_lower(make_mtmd(false, 1).get_prefix_hashes()));
}
}
static void test_server_tokens(testing & t) {
t.test("basic", test_server_tokens_basic);
t.test("pos", test_server_tokens_pos);
t.test("keep_first", test_server_tokens_keep_first);
t.test("common_prefix", test_server_tokens_common_prefix);
t.test("serialize", test_server_tokens_serialize);
t.test("clone", test_server_tokens_clone);
t.test("prefix_hashes", test_server_tokens_prefix_hashes);
}
//
// string / json helpers
//
static void test_utf8(testing & t) {
t.test("validate_utf8", [](testing & t) {
t.assert_equal("empty", (size_t) 0, validate_utf8(""));
t.assert_equal("ascii", (size_t) 3, validate_utf8("abc"));
t.assert_equal("full 2-byte", (size_t) 3, validate_utf8("a\xC3\xA9"));
t.assert_equal("full 3-byte", (size_t) 4, validate_utf8("a\xE2\x82\xAC"));
t.assert_equal("full 4-byte", (size_t) 5, validate_utf8("a\xF0\x9F\x98\x80"));
t.assert_equal("cut 2-byte (1/2)", (size_t) 1, validate_utf8("a\xC3"));
t.assert_equal("cut 3-byte (1/3)", (size_t) 1, validate_utf8("a\xE2"));
t.assert_equal("cut 3-byte (2/3)", (size_t) 1, validate_utf8("a\xE2\x82"));
t.assert_equal("cut 4-byte (1/4)", (size_t) 1, validate_utf8("a\xF0"));
t.assert_equal("cut 4-byte (2/4)", (size_t) 1, validate_utf8("a\xF0\x9F"));
t.assert_equal("cut 4-byte (3/4)", (size_t) 1, validate_utf8("a\xF0\x9F\x98"));
t.assert_equal("cut only", (size_t) 0, validate_utf8("\xF0\x9F\x98"));
t.assert_equal("full then cut", (size_t) 4, validate_utf8("\xF0\x9F\x98\x80\xC3"));
});
t.test("is_valid_utf8", [](testing & t) {
t.assert_true("empty", is_valid_utf8(""));
t.assert_true("ascii", is_valid_utf8("hello"));
t.assert_true("multi-byte", is_valid_utf8("\xC3\xA9\xE2\x82\xAC\xF0\x9F\x98\x80"));
t.assert_true("truncated", !is_valid_utf8("\xE2\x82"));
t.assert_true("bad cont. byte", !is_valid_utf8("\xC3\x41"));
t.assert_true("lone cont. byte", !is_valid_utf8("\x80"));
t.assert_true("invalid lead", !is_valid_utf8("\xFF"));
});
}
static void test_sse(testing & t) {
t.test("oai", [](testing & t) {
t.assert_equal("single", std::string("data: {\"a\":1}\n\n"), format_oai_sse(json{{"a", 1}}));
t.assert_equal("array", std::string("data: {\"a\":1}\n\ndata: {\"b\":2}\n\n"),
format_oai_sse(json::array({json{{"a", 1}}, json{{"b", 2}}})));
t.assert_equal("invalid utf8 replaced", std::string("data: \"x\xEF\xBF\xBD\"\n\n"), format_oai_sse(json("x\xC3")));
});
t.test("oai_resp", [](testing & t) {
json ev = {{"event", "response.created"}, {"data", {{"id", "r1"}}}};
t.assert_equal("single", std::string("event: response.created\ndata: {\"id\":\"r1\"}\n\n"), format_oai_resp_sse(ev));
t.assert_equal("array", std::string("event: response.created\ndata: {\"id\":\"r1\"}\n\nevent: response.created\ndata: {\"id\":\"r1\"}\n\n"),
format_oai_resp_sse(json::array({ev, ev})));
});
t.test("anthropic", [](testing & t) {
json ev = {{"event", "message_start"}, {"data", {{"type", "message_start"}}}};
t.assert_equal("with event", std::string("event: message_start\ndata: {\"type\":\"message_start\"}\n\n"), format_anthropic_sse(ev));
t.assert_equal("data only", std::string("data: {\"type\":\"ping\"}\n\n"), format_anthropic_sse(json{{"type", "ping"}}));
});
}
static void test_json_helpers(testing & t) {
t.test("json_is_array_of_numbers", [](testing & t) {
t.assert_true("ints", json_is_array_of_numbers(json::array({1, 2, 3})));
t.assert_true("empty array", json_is_array_of_numbers(json::array()));
t.assert_true("float", !json_is_array_of_numbers(json::array({1, 2.5})));
t.assert_true("string", !json_is_array_of_numbers(json::array({1, "a"})));
t.assert_true("not an array", !json_is_array_of_numbers(json(1)));
});
t.test("json_is_array_of_mixed_numbers_strings", [](testing & t) {
t.assert_true("mixed", json_is_array_of_mixed_numbers_strings(json::array({"a", 1})));
t.assert_true("numbers only", !json_is_array_of_mixed_numbers_strings(json::array({1, 2})));
t.assert_true("strings only", !json_is_array_of_mixed_numbers_strings(json::array({"a", "b"})));
t.assert_true("empty", !json_is_array_of_mixed_numbers_strings(json::array()));
});
t.test("json_is_array_and_contains_numbers", [](testing & t) {
t.assert_true("contains", json_is_array_and_contains_numbers(json::array({"a", 1})));
t.assert_true("strings only", !json_is_array_and_contains_numbers(json::array({"a"})));
t.assert_true("not an array", !json_is_array_and_contains_numbers(json(1)));
});
t.test("json_get_nested_values", [](testing & t) {
json js = {{"a", {{"b", {{"c", 1}}}}}, {"x", 2}};
json res = json_get_nested_values({"a/b/c", "x", "a/b", "a/missing", "x/y"}, js);
t.assert_equal("nested", 1, res.at("a/b/c").get<int>());
t.assert_equal("top", 2, res.at("x").get<int>());
t.assert_true("object", res.at("a/b") == json{{"c", 1}});
t.assert_true("missing", !res.contains("a/missing"));
t.assert_true("not obj", !res.contains("x/y"));
t.assert_equal("count", (size_t) 3, res.size());
});
}
//
// server_prompt
//
static void test_server_prompt_reuse(testing & t) {
// 110 text tokens, the position of a token is its index
auto make_prompt = [](std::vector<std::array<llama_pos, 2>> checkpoints) {
llama_tokens toks(110);
for (size_t i = 0; i < toks.size(); i++) {
toks[i] = 100 + (llama_token) i;
}
server_prompt res;
res.tokens = server_tokens(toks, false);
for (const auto & range : checkpoints) {
auto ckpt = std::make_shared<common_prompt_checkpoint>();
ckpt->pos_min = range[0];
ckpt->pos_max = range[1];
ckpt->n_tokens = range[1] + 1;
res.checkpoints.push_back(std::move(ckpt));
}
return res;
};
t.test("recurrent", [&](testing & t) {
// a recurrent state holds one position: the live state is at 109, n_swa is 0
const auto prompt = make_prompt({{95, 95}, {99, 99}, {105, 105}});
const llama_pos pos_min = 109;
auto res = prompt.get_reuse(110, true, 0, pos_min);
t.assert_true("new tokens after the prompt, no checkpoint", res.ckpt == nullptr);
t.assert_equal("new tokens after the prompt, n_past", 110, res.n_past);
res = prompt.get_reuse(110, false, 0, pos_min);
t.assert_true("same prompt, last checkpoint", res.ckpt && res.ckpt->pos_max == 105);
t.assert_equal("same prompt, n_past", 106, res.n_past);
t.assert_equal("same prompt, pos_next", 106, res.pos_next);
// the closest checkpoint before the new tokens is used, the one after them is not
res = prompt.get_reuse(104, true, 0, pos_min);
t.assert_true("cut at 104, checkpoint", res.ckpt && res.ckpt->pos_max == 99);
t.assert_equal("cut at 104, n_past", 100, res.n_past);
t.assert_equal("cut at 104, pos_next", 100, res.pos_next);
// the state at 99 already holds the token at 99, it cannot be used to process it again
res = prompt.get_reuse(99, true, 0, pos_min);
t.assert_true("cut at 99, checkpoint", res.ckpt && res.ckpt->pos_max == 95);
t.assert_equal("cut at 99, n_past", 96, res.n_past);
res = prompt.get_reuse(50, true, 0, pos_min);
t.assert_true("cut at 50, no checkpoint", res.ckpt == nullptr);
t.assert_equal("cut at 50, n_past", 0, res.n_past);
t.assert_equal("cut at 50, pos_next", 0, res.pos_next);
});
t.test("swa", [&](testing & t) {
// n_swa is 8: the live state starts at 102, the checkpoint holds the positions 92 to 99
const auto prompt = make_prompt({{92, 99}});
const llama_pos pos_min = 102;
auto res = prompt.get_reuse(110, true, 8, pos_min);
t.assert_true("new tokens after the prompt, no checkpoint", res.ckpt == nullptr);
t.assert_equal("new tokens after the prompt, n_past", 110, res.n_past);
// the token at 104 needs the positions 97 to 103, the live state starts too late for a cut at 104
res = prompt.get_reuse(104, true, 8, pos_min);
t.assert_true("cut at 104, checkpoint", res.ckpt != nullptr);
t.assert_equal("cut at 104, n_past", 99, res.n_past);
t.assert_equal("cut at 104, pos_next", 99, res.pos_next);
res = prompt.get_reuse(95, true, 8, pos_min);
t.assert_true("cut at 95, no checkpoint", res.ckpt == nullptr);
t.assert_equal("cut at 95, n_past", 0, res.n_past);
});
}
static void test_lora_helpers(testing & t) {
// only pointer identity is used, never dereferenced
auto * p0 = reinterpret_cast<llama_adapter_lora *>(0x10);
auto * p1 = reinterpret_cast<llama_adapter_lora *>(0x20);
auto make = [](llama_adapter_lora * ptr, float scale) {
common_adapter_lora_info info;
info.scale = scale;
info.ptr = ptr;
return info;
};
t.test("parse_lora_request", [](testing & t) {
auto res = parse_lora_request(json::array({
json{{"id", 0}, {"scale", 0.5}},
json{{"id", 2}, {"scale", 1.0}},
json{{"id", 0}, {"scale", 0.25}}, // last one wins
}));
t.assert_equal("count", (size_t) 2, res.size());
t.assert_equal("id 0", 0.25f, res.at(0));
t.assert_equal("id 2", 1.0f, res.at(2));
auto defaults = parse_lora_request(json::array({json::object()}));
t.assert_equal("default id/scale", 0.0f, defaults.at(-1));
});
t.test("are_lora_equal", [&](testing & t) {
t.assert_true("both empty", are_lora_equal({}, {}));
t.assert_true("same", are_lora_equal({make(p0, 1.0f)}, {make(p0, 1.0f)}));
t.assert_true("diff scale", !are_lora_equal({make(p0, 1.0f)}, {make(p0, 0.5f)}));
t.assert_true("diff ptr", !are_lora_equal({make(p0, 1.0f)}, {make(p1, 1.0f)}));
t.assert_true("diff size", !are_lora_equal({make(p0, 1.0f)}, {}));
});
t.test("lora_should_clear_cache", [&](testing & t) {
// the adapter pointer is only read when the configs differ
t.assert_true("both empty", !lora_should_clear_cache({}, {}));
t.assert_true("same", !lora_should_clear_cache({make(p0, 1.0f)}, {make(p0, 1.0f)}));
t.assert_true("to no lora", lora_should_clear_cache({make(p0, 0.0f)}, {}));
});
t.test("lora_get_enabled_ids", [&](testing & t) {
auto ids = lora_get_enabled_ids({make(p0, 0.0f), make(p1, 0.5f), make(p0, -1.0f), make(p1, 2.0f)});
t.assert_true("ids", ids == std::vector<size_t>({1, 3}));
});
}
//
// server_pipe
//
static void test_server_pipe(testing & t) {
auto stop_now = []() { return true; };
t.test("drain then eof", [](testing & t) {
server_pipe<int> p;
t.assert_true("write 1", p.write(1));
t.assert_true("write 2", p.write(2));
p.close_write();
int v = 0;
t.assert_true("read 1", p.read(v, nullptr) && v == 1);
t.assert_true("read 2", p.read(v, nullptr) && v == 2);
t.assert_true("eof", !p.read(v, nullptr));
});
t.test("broken pipe", [](testing & t) {
server_pipe<int> p;
p.close_read();
t.assert_true("write fails", !p.write(1));
});
t.test("max_size drops oldest", [](testing & t) {
server_pipe<int> p;
p.max_size = 2;
p.write(1);
p.write(2);
p.write(3);
p.close_write();
int v = 0;
t.assert_true("read 2", p.read(v, nullptr) && v == 2);
t.assert_true("read 3", p.read(v, nullptr) && v == 3);
t.assert_true("eof", !p.read(v, nullptr));
});
t.test("queued data wins over should_stop", [&](testing & t) {
server_pipe<int> p;
p.write(1);
int v = 0;
t.assert_true("read", p.read(v, stop_now) && v == 1);
});
t.test("should_stop closes the pipe", [&](testing & t) {
server_pipe<int> p;
int v = 0;
t.assert_true("read stopped", !p.read(v, stop_now));
t.assert_true("write fails", !p.write(1));
});
t.test("should_stop as deadline keeps the pipe open", [&](testing & t) {
server_pipe<int> p;
int v = 0;
t.assert_true("read stopped", !p.read(v, stop_now, /* close_on_stop */ false));
t.assert_true("write ok", p.write(1));
t.assert_true("read again", p.read(v, nullptr) && v == 1);
});
t.test("cross thread", [](testing & t) {
constexpr int n = 1000;
server_pipe<int> p;
std::thread producer([&]() {
for (int i = 0; i < n; i++) {
p.write(int(i));
}
p.close_write();
});
int v = 0;
int count = 0;
bool in_order = true;
while (p.read(v, nullptr)) {
in_order &= v == count;
count++;
}
producer.join();
t.assert_equal("count", n, count);
t.assert_true("in order", in_order);
});
}
//
// server_decision_escape_special_tokens
//
// "<|name|>" with U+00A6 in place of "|"
static std::string escaped(const std::string & name) {
return "<\xC2\xA6" + name + "\xC2\xA6>";
}
static void test_decision_escape(testing & t) {
t.test("cases", [](testing & t) {
const std::vector<std::pair<std::string, std::string>> cases = {
{ "", "" },
{ "no tokens", "no tokens" },
{ "<|im_start|>user", escaped("im_start") + "user" },
{ "<|a_1|><|B|>", escaped("a_1") + escaped("B") },
{ "<|<|a|>", "<|" + escaped("a") },
{ "<|a|>|>", escaped("a") + "|>" },
{ "<||>", "<||>" },
{ "<|a b|>", "<|a b|>" },
{ "<|a|b|>", "<|a|b|>" },
{ "<|caf\xC3\xA9|>", "<|caf\xC3\xA9|>" },
{ "<|abc", "<|abc" },
};
for (const auto & [text, expected] : cases) {
t.assert_equal(text, expected, server_decision_escape_special_tokens(text));
}
});
// every string of up to 6 chars from "<|>a-" gives the text of the regex
t.test("same as std::regex", [](testing & t) {
static const std::regex re_special("<\\|([A-Za-z0-9_]+)\\|>");
const std::string chars = "<|>a-";
size_t n_strings = 0;
size_t n_diff = 0;
for (size_t len = 0; len <= 6; len++) {
size_t n = 1;
for (size_t k = 0; k < len; k++) {
n *= chars.size();
}
for (size_t idx = 0; idx < n; idx++) {
std::string text;
for (size_t k = 0, v = idx; k < len; k++, v /= chars.size()) {
text += chars[v % chars.size()];
}
if (server_decision_escape_special_tokens(text) != std::regex_replace(text, re_special, "<\xC2\xA6$1\xC2\xA6>")) {
n_diff++;
}
n_strings++;
}
}
t.assert_equal("strings that differ, of " + std::to_string(n_strings), (size_t) 0, n_diff);
});
// std::regex overflowed the stack on these
t.test("long input", [](testing & t) {
const std::string name(1000000, 'a');
t.assert_true("not closed", server_decision_escape_special_tokens("<|" + name) == "<|" + name);
t.assert_true("closed", server_decision_escape_special_tokens("<|" + name + "|>") == escaped(name));
});
}
static void test_rerank_template(testing & t) {
struct test_case {
const char * name;
std::string tmpl;
std::string query;
std::string doc;
std::string expected;
};
const test_case cases[] = {
{"plain inputs", "Q: {query}\nD: {document}", "query", "doc", "Q: query\nD: doc"},
{"document placeholder in query", "Q: {query}\nD: {document}", "Explain {document}.", "doc", "Q: Explain {document}.\nD: doc"},
{"query placeholder in query", "{query}|{document}", "{query}", "doc", "{query}|doc"},
{"placeholders in both inputs", "{query}|{document}", "{query} {document}", "{query} {document}", "{query} {document}|{query} {document}"},
{"repeated placeholders", "{query}|{document}|{query}|{document}", "{document}", "doc", "{document}|doc|{document}|doc"},
{"adjacent reversed placeholders", "{document}{query}", "query", "doc", "docquery"},
{"query only", "{query}", "{document}", "doc", "{document}"},
{"document only", "{document}", "query", "{query}", "{query}"},
{"no placeholders", "fixed", "{document}", "{query}", "fixed"},
{"unknown and incomplete placeholders", "{unknown}|{query}|{document}|{query", "query", "doc", "{unknown}|query|doc|{query"},
{"nested braces", "{{query}}|{{document}}", "query", "doc", "{query}|{doc}"},
{"placeholder formed at boundary", "{query}ument}", "{doc", "doc", "{document}"},
{"empty inputs", "Q: {query}\nD: {document}", "", "", "Q: \nD: "},
{"empty template", "", "query", "doc", ""},
};
for (const auto & test : cases) {
t.assert_equal(test.name, test.expected, format_rerank_template(test.tmpl, test.query, test.doc));
}
}
//
// server_decision_context::format_answer, 2 class scores (minicpm-ocr-jev)
//
static void test_decision_class_scores(testing & t) {
server_decision_context ctx;
ctx.type = COMMON_DECISION_TYPE_MINICPM_OCR_JEV;
const server_decision_question choice = {"q", SERVER_DECISION_QUESTION_CHOICE, json(), {{"0", json()}, {"1", json()}}};
const server_decision_question noul = {"q", SERVER_DECISION_QUESTION_NOUL, json(), {{"false", json()}, {"true", json()}}};
const std::vector<std::pair<float, float>> cases = {
{0.0f, 0.0f}, {-1.5f, 2.25f}, {3.0f, -0.5f}, {80.0f, -80.0f}, {1000.0f, 1001.0f},
};
for (const auto & [z0, z1] : cases) {
const double p1 = 1.0 / (1.0 + std::exp((double) z0 - (double) z1));
const json a = ctx.format_answer(choice, {{z0, z1}});
const double ap0 = a.at("probabilities").at("0").get<double>();
const double ap1 = a.at("probabilities").at("1").get<double>();
t.assert_true("p1", std::fabs(ap1 - p1) <= 1e-6);
t.assert_true("sum", std::fabs(ap0 + ap1 - 1.0) <= 1e-9);
// a tie picks the first class
t.assert_equal("choice", std::string(z1 > z0 ? "1" : "0"), a.at("choice").get<std::string>());
t.assert_true("confidence", std::fabs(a.at("confidence").get<double>() - (2.0 * std::max(ap0, ap1) - 1.0)) <= 1e-9);
const json n = ctx.format_answer(noul, {{z0, z1}});
t.assert_true("noul is p1", n.at("noul").get<double>() == ap1);
}
for (const float bad : {std::nanf(""), INFINITY}) {
bool thrown = false;
try {
ctx.format_answer(choice, {{bad, 0.0f}});
} catch (const std::exception &) {
thrown = true;
}
t.assert_true("nan or +inf", thrown);
}
}
//
// server_decision_context::parse_questions / parse_state, the request checks of minicpm-ocr-jev
//
static void test_decision_ocr_jev_validation(testing & t) {
server_decision_context ctx;
ctx.type = COMMON_DECISION_TYPE_MINICPM_OCR_JEV;
// only base64-decoded, the image itself is not parsed here
const std::string image = "data:image/png;base64,AAAA";
const json choice = {
{"type", "choice"},
{"criteria", json{{"0", nullptr}, {"1", nullptr}}},
};
const auto questions_of = [](const json & question) {
return json{{"state", "some OCR text"}, {"questions", json{{"q", question}}}};
};
const auto files_of = [&](size_t n) {
json files = json::array();
for (size_t i = 0; i < n; i++) {
files.push_back(image);
}
return json{{"state", "some OCR text"}, {"files", files}};
};
const auto reject_questions = [&](const char * name, const json & body, const std::string & expected) {
std::string error;
try {
ctx.parse_questions(body);
} catch (const std::invalid_argument & e) {
error = e.what();
}
t.assert_true(std::string(name) + ": " + error, error.find(expected) != std::string::npos);
};
const auto reject_state = [&](const char * name, const json & body, const std::string & expected) {
std::vector<raw_buffer> files;
std::string error;
try {
ctx.parse_state(body, files);
} catch (const std::invalid_argument & e) {
error = e.what();
}
t.assert_true(std::string(name) + ": " + error, error.find(expected) != std::string::npos);
};
reject_state("state is text", {{"state", 42}}, "\"state\" must be the OCR text");
// one fixed question, no instructions
reject_questions("one question", {{"state", "x"}, {"questions", json{{"a", choice}, {"b", choice}}}},
"answers exactly one question");
reject_questions("no instructions",
questions_of(json{{"type", "choice"}, {"instructions", "judge it"}, {"criteria", json{{"0", nullptr}, {"1", nullptr}}}}),
"\"instructions\" is not used by this model");
// the classes are fixed, so are the criteria
reject_questions("criteria are the classes",
questions_of(json{{"type", "choice"}, {"criteria", json{{"yes", nullptr}, {"no", nullptr}}}}),
"\"criteria\" must be {\"0\": null, \"1\": null}");
reject_questions("noul without criteria",
questions_of(json{{"type", "noul"}, {"criteria", json{{"false", "no"}, {"true", "yes"}}}}),
"\"criteria\" is not supported for a noul question");
reject_questions("choice or noul only",
questions_of(json{{"type", "score"}, {"criteria", json::array({"low", "high"})}}),
"only choice and noul questions are supported");
// the prompt has the original image, then the one rendered from the OCR text
reject_state("two images", files_of(1), "this model needs exactly 2 images");
reject_state("two images, three given", files_of(3), "this model needs exactly 2 images");
}
// server_decision_context: the recursive functions stop at RECURSIVE_LIMIT
//
static void test_decision_depth(testing & t) {
const auto prepare = [](size_t depth) {
server_decision_context ctx;
ctx.type = COMMON_DECISION_TYPE_H2O_LIGHTNING; // the state goes through decision_py_dump
std::string error;
try {
ctx.prepare_state(json::parse(std::string(depth, '[') + std::string(depth, ']')));
} catch (const std::invalid_argument & e) {
error = e.what();
}
return error;
};
t.test("state at the limit", [&](testing & t) {
const std::string error = prepare(128);
t.assert_true("accepted: " + error, error.empty());
});
t.test("deep state", [&](testing & t) {
const std::string error = prepare(1000);
t.assert_true("rejected as nested: " + error, error.find("nested more than 128 levels deep") != std::string::npos);
});
}
// DRY sequence breakers
//
static void test_dry_sequence_breakers(testing & t) {
common_params params_base;
const auto eval = [&](const json & data) {
return server_schema::eval_llama_cmpl_schema(nullptr, params_base, {}, data);
};
{
json data;
data["dry_sequence_breakers"] = json::array({"ab", "ab", "cd", "ab"});
const auto params = eval(data);
const std::vector<std::string> expected = {"ab", "cd"};
t.assert_true("duplicates removed, order kept", params.sampling.dry_sequence_breakers == expected);
}
{
std::vector<std::string> many;
for (int i = 0; i <= LLAMA_DRY_MAX_SEQ_BREAKERS; ++i) {
many.push_back("b" + std::to_string(i));
}
json data;
data["dry_sequence_breakers"] = many;
bool thrown = false;
try {
eval(data);
} catch (const std::exception &) {
thrown = true;
}
t.assert_true("more than the cap is rejected", thrown);
}
}
static void test_stop_strings(testing & t) {
common_params params_base;
const auto eval = [&](const json & data) {
return server_schema::eval_llama_cmpl_schema(nullptr, params_base, {}, data);
};
{
json data;
data["stop"] = "";
t.assert_true("empty string is ignored", eval(data).antiprompt.empty());
}
{
json data;
data["stop"] = json::array({""});
t.assert_true("empty string in array is ignored", eval(data).antiprompt.empty());
}
{
json data;
data["stop"] = json::array({"", "zzz"});
const std::vector<std::string> expected = {"zzz"};
t.assert_true("only non-empty strings are kept", eval(data).antiprompt == expected);
}
{
params_base.antiprompt = {"User:"};
json data;
data["stop"] = json::array({""});
const std::vector<std::string> expected = {"User:"};
t.assert_true("falls back to CLI defaults", eval(data).antiprompt == expected);
}
}
//
// glob_match
//
static void test_glob_match(testing & t) {
t.test("patterns", [](testing & t) {
t.assert_true("literal", glob_match("src/main.cpp", "src/main.cpp"));
t.assert_true("star", glob_match("src/*.cpp", "src/main.cpp"));
t.assert_true("star stops at /", !glob_match("*.cpp", "src/main.cpp"));
t.assert_true("double star", glob_match("**/*.cpp", "a/b/c/main.cpp"));
t.assert_true("double star at end", glob_match("src/**", "src/a/b"));
t.assert_true("question mark", glob_match("ma?n.cpp", "main.cpp"));
t.assert_true("question mark not /", !glob_match("a?b", "a/b"));
t.assert_true("class", glob_match("[a-c]x", "bx"));
t.assert_true("negated class", !glob_match("[!a-c]x", "bx"));
t.assert_true("unclosed class", glob_match("[ab", "[ab"));
t.assert_true("mismatch", !glob_match("main.cpp", "main.c"));
});
t.test("long input", [](testing & t) {
const std::string path(100000, 'a');
t.assert_true("long literal", glob_match(path, path));
t.assert_true("long path", glob_match("**", path));
t.assert_true("too many wildcards", !glob_match(std::string(1000, '*'), "a"));
});
}
//
// OpenAI speech API
//
static void test_speech_convert(testing & t) {
const json base = {{"model", "tts"}, {"input", "Hello"}, {"voice", "alba"}};
auto with = [&](const char * key, const json & value) {
json body = base;
body[key] = value;
return body;
};
auto rejected = [&](const char * key, const json & value) {
try {
convert_speech_to_tts(with(key, value));
} catch (const std::invalid_argument &) {
return true;
}
return false;
};
// defaults
json out = convert_speech_to_tts(base);
t.assert_equal("input kept", std::string("Hello"), out.at("input").get<std::string>());
t.assert_equal("voice kept", std::string("alba"), out.at("voice").get<std::string>());
t.assert_equal("default format", std::string("wav"), out.at("response_format").get<std::string>());
t.assert_true ("no stream", !out.contains("stream"));
// response_format
out = convert_speech_to_tts(with("response_format", "wav"));
t.assert_equal("wav", std::string("wav"), out.at("response_format").get<std::string>());
out = convert_speech_to_tts(with("response_format", "pcm"));
t.assert_equal("pcm is 16-bit", std::string("pcm16"), out.at("response_format").get<std::string>());
for (const char * format : {"mp3", "opus", "aac", "flac", "pcm16"}) {
t.assert_true(std::string("rejects ") + format, rejected("response_format", format));
}
// custom voice
out = convert_speech_to_tts(with("voice", json{{"id", "voice_1234"}}));
t.assert_equal("voice object", std::string("voice_1234"), out.at("voice").get<std::string>());
// stream_format
out = convert_speech_to_tts(with("stream_format", "audio"));
t.assert_true ("audio streams", out.at("stream").get<bool>());
t.assert_true ("rejects sse", rejected("stream_format", "sse"));
// speed is forwarded, the model decides whether it is supported
out = convert_speech_to_tts(with("speed", 1.5));
t.assert_equal("speed forwarded", 1.5, out.at("speed").get<double>());
// instructions are ignored
t.assert_true ("instructions", !rejected("instructions", "whisper"));
}
static void test_pcm_f32_to_s16(testing & t) {
auto sample = [](const std::string & bytes, size_t i) {
int16_t v;
memcpy(&v, bytes.data() + i * sizeof(v), sizeof(v));
return (int) v;
};
const std::vector<float> samples = {0.0f, 1.0f, -1.0f, 0.5f, 2.0f, -2.0f};
const std::string bytes = pcm_f32_to_s16(samples.data(), samples.size());
t.assert_equal("size", (size_t) 12, bytes.size());
t.assert_equal("zero", 0, sample(bytes, 0));
t.assert_equal("max", 32767, sample(bytes, 1));
t.assert_equal("min", -32767, sample(bytes, 2));
t.assert_equal("rounds", 16384, sample(bytes, 3));
t.assert_equal("clamps above", 32767, sample(bytes, 4));
t.assert_equal("clamps below", -32767, sample(bytes, 5));
t.assert_true ("empty", pcm_f32_to_s16(nullptr, 0).empty());
}
int main(int argc, char ** argv) {
testing t;
const char * verbose = getenv("LLAMA_TEST_VERBOSE");
if (verbose) {
t.verbose = std::string(verbose) == "1";
}
if (argc > 1) {
t.set_filter(argv[1]);
}
t.test("server_tokens", test_server_tokens);
t.test("utf8", test_utf8);
t.test("sse", test_sse);
t.test("json_helpers", test_json_helpers);
t.test("lora_helpers", test_lora_helpers);
t.test("server_prompt_reuse", test_server_prompt_reuse);
t.test("server_pipe", test_server_pipe);
t.test("decision_escape", test_decision_escape);
t.test("rerank_template", test_rerank_template);
t.test("decision_class_scores", test_decision_class_scores);
t.test("decision_ocr_jev_validation", test_decision_ocr_jev_validation);
t.test("decision_depth", test_decision_depth);
t.test("glob_match", test_glob_match);
t.test("dry_sequence_breakers", test_dry_sequence_breakers);
t.test("stop_strings", test_stop_strings);
t.test("speech_convert", test_speech_convert);
t.test("pcm_f32_to_s16", test_pcm_f32_to_s16);
return t.summary();
}