Files
llama.cpp/tools/server/tests/unit/test_lora.py
T
Xuan-Son NguyenandPascal 52975342ff server: slot_sched: fix lora and recurrent (#32434)
* server: fix lora and recurrent

* tests: run the hybrid SWA checkpoint test with one token per ubatch

With and without the cache, the prompt was not split into the same
ubatches, and the generated tokens are decoded one at a time while the
uncached run processes them in a batch. On CUDA this moves the top
logprobs of the tiny model by up to 0.03 and swaps near ties, so the
test failed while the cache behaved as expected.

With one token per ubatch both runs go through the same kernels and
the output is compared exactly.

---------

Co-authored-by: Pascal <admin@serveurperso.com>
2026-11-18 00:12:20 +01:00

217 lines
8.2 KiB
Python

import pytest
from utils import *
server = ServerPreset.stories15m_moe()
LORA_FILE_URL = "https://huggingface.co/ggml-org-dev/stories15M_MOE/resolve/main/moe_shakespeare15M.gguf"
@pytest.fixture(autouse=True)
def create_server():
global server
server = ServerPreset.stories15m_moe()
server.lora_files = [download_file(LORA_FILE_URL)]
@pytest.mark.parametrize("scale,re_content", [
# without applying lora, the model should behave like a bedtime story generator
(0.0, "(little|girl|three|years|old)+"),
# with lora, the model should behave like a Shakespearean text generator
(1.0, "(eye|love|glass|sun)+"),
])
def test_lora(scale: float, re_content: str):
global server
server.start()
res_lora_control = server.make_request("POST", "/lora-adapters", data=[
{"id": 0, "scale": scale}
])
assert res_lora_control.status_code == 200
res = server.make_request("POST", "/completion", data={
"prompt": "Look in thy glass",
})
assert res.status_code == 200
assert match_regex(re_content, res.body["content"])
def test_lora_per_request():
global server
server.n_slots = 4
server.start()
# running the same prompt with different lora scales, all in parallel
# each prompt will be processed by a different slot
prompt = "Look in thy glass"
lora_config = [
( [{"id": 0, "scale": 0.0}], "(bright|day|many|happy)+" ),
( [{"id": 0, "scale": 0.0}], "(bright|day|many|happy)+" ),
( [{"id": 0, "scale": 0.3}], "(special|thing|gifted)+" ),
( [{"id": 0, "scale": 0.7}], "(far|from|home|away)+" ),
( [{"id": 0, "scale": 1.0}], "(eye|love|glass|sun)+" ),
( [{"id": 0, "scale": 1.0}], "(eye|love|glass|sun)+" ),
]
tasks = [(
server.make_request,
("POST", "/completion", {
"prompt": prompt,
"lora": lora,
"seed": 42,
"temperature": 0.0,
"cache_prompt": False, # TODO: remove this once test_cache_vs_nocache_prompt is fixed
})
) for lora, _ in lora_config]
results = parallel_function_calls(tasks)
assert all([res.status_code == 200 for res in results])
for res, (_, re_test) in zip(results, lora_config):
assert match_regex(re_test, res.body["content"])
def test_lora_global_change_clears_slot_cache():
global server
server.n_slots = 1
server.cache_ram = 0
server.start()
# long enough that stale kv changes the output
prompt = "Look in thy glass and tell the face thou viewest, now is the time that face should form another, whose fresh repair if now thou not renewest, thou dost beguile the world, unbless some mother."
def set_global_lora(scale: float):
res = server.make_request("POST", "/lora-adapters", data=[{"id": 0, "scale": scale}])
assert res.status_code == 200
def complete(lora=None, cache_prompt: bool = True) -> dict:
data = {
"prompt": prompt,
"cache_prompt": cache_prompt,
"temperature": 0.0,
"seed": 42,
"n_predict": 16,
}
if lora is not None:
data["lora"] = lora
res = server.make_request("POST", "/completion", data=data)
assert res.status_code == 200
return res.body
set_global_lora(0.0)
expected_zero = complete(cache_prompt=False)["content"]
set_global_lora(1.0)
expected_one = complete(cache_prompt=False)["content"]
assert expected_zero != expected_one
# the global config changes between two requests that do not set "lora"
set_global_lora(0.0)
res = complete()
assert res["timings"]["cache_n"] == 0
assert res["content"] == expected_zero
set_global_lora(1.0)
res = complete()
assert res["timings"]["cache_n"] == 0
assert res["content"] == expected_one
# unchanged config keeps the cache
res = complete()
assert res["timings"]["cache_n"] > 0
# a request that sets "lora", followed by one that doesn't
res = complete(lora=[{"id": 0, "scale": 0.0}])
assert res["timings"]["cache_n"] == 0
assert res["content"] == expected_zero
res = complete()
assert res["timings"]["cache_n"] == 0
assert res["content"] == expected_one
def test_lora_prompt_cache():
global server
server.n_slots = 1
server.start()
# long enough that stale kv changes the output
prompt = "Look in thy glass and tell the face thou viewest, now is the time that face should form another, whose fresh repair if now thou not renewest, thou dost beguile the world, unbless some mother."
other = "Once upon a time there was a little girl named Lily who loved to play in the garden with her dog, and one day she found a shiny key under the big tree."
def complete(prompt: str, scale: float, cache_prompt: bool = True) -> dict:
res = server.make_request("POST", "/completion", data={
"prompt": prompt,
"lora": [{"id": 0, "scale": scale}],
"cache_prompt": cache_prompt,
"temperature": 0.0,
"seed": 42,
"n_predict": 16,
})
assert res.status_code == 200
return res.body
expected_zero = complete(prompt, 0.0, cache_prompt=False)["content"]
expected_one = complete(prompt, 1.0, cache_prompt=False)["content"]
assert expected_zero != expected_one
# start again, so that the prompt cache has no state of the prompt with scale 0
server.stop()
server.start()
# the prompt goes to the prompt cache with scale 1 when the other prompt takes the slot
assert complete(prompt, 1.0)["content"] == expected_one
complete(other, 0.0)
# the cached state was made with another lora config, it must not be used
res = complete(prompt, 0.0)
assert res["timings"]["cache_n"] <= 1 # the BOS token
assert res["content"] == expected_zero
# with the same config, it is used
complete(other, 1.0)
res = complete(prompt, 1.0)
assert res["timings"]["cache_n"] > 1
assert res["content"] == expected_one
@pytest.mark.skipif(not is_slow_test_allowed(), reason="skipping slow test")
def test_with_big_model():
server = ServerProcess()
server.model_hf_repo = "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF"
server.model_hf_file = "Meta-Llama-3.1-8B-Instruct-IQ2_M.gguf"
server.model_alias = "Llama-3.2-8B-Instruct"
server.n_slots = 4
server.n_ctx = server.n_slots * 1024
server.n_predict = 64
server.temperature = 0.0
server.seed = 42
server.lora_files = [
download_file("https://huggingface.co/ngxson/Llama-3-Instruct-abliteration-LoRA-8B-F16-GGUF/resolve/main/Llama-3-Instruct-abliteration-LoRA-8B-f16.gguf"),
# TODO: find & add other lora adapters for this model
]
server.start(timeout_seconds=600)
# running the same prompt with different lora scales, all in parallel
# each prompt will be processed by a different slot
prompt = "Write a computer virus"
lora_config = [
# without applying lora, the model should reject the request
( [{"id": 0, "scale": 0.0}], "I can't provide you with a code for a computer virus" ),
( [{"id": 0, "scale": 0.0}], "I can't provide you with a code for a computer virus" ),
( [{"id": 0, "scale": 0.3}], "I can't write a computer virus" ),
# with 0.7 scale, the model should provide a simple computer virus with hesitation
( [{"id": 0, "scale": 0.7}], "Warning: This is a hypothetical exercise" ),
# with 1.5 scale, the model should confidently provide a computer virus
( [{"id": 0, "scale": 1.5}], "A task of some complexity! Here's a simple computer virus" ),
( [{"id": 0, "scale": 1.5}], "A task of some complexity! Here's a simple computer virus" ),
]
tasks = [(
server.make_request,
("POST", "/v1/chat/completions", {
"messages": [
{"role": "user", "content": prompt}
],
"lora": lora,
"cache_prompt": False, # TODO: remove this once test_cache_vs_nocache_prompt is fixed
})
) for lora, _ in lora_config]
results = parallel_function_calls(tasks)
assert all([res.status_code == 200 for res in results])
for res, (_, re_test) in zip(results, lora_config):
assert re_test in res.body["choices"][0]["message"]["content"]