* server: fix lora and recurrent * tests: run the hybrid SWA checkpoint test with one token per ubatch With and without the cache, the prompt was not split into the same ubatches, and the generated tokens are decoded one at a time while the uncached run processes them in a batch. On CUDA this moves the top logprobs of the tiny model by up to 0.03 and swaps near ties, so the test failed while the cache behaved as expected. With one token per ubatch both runs go through the same kernels and the output is compared exactly. --------- Co-authored-by: Pascal <admin@serveurperso.com>
217 lines
8.2 KiB
Python
217 lines
8.2 KiB
Python
import pytest
|
|
from utils import *
|
|
|
|
server = ServerPreset.stories15m_moe()
|
|
|
|
LORA_FILE_URL = "https://huggingface.co/ggml-org-dev/stories15M_MOE/resolve/main/moe_shakespeare15M.gguf"
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def create_server():
|
|
global server
|
|
server = ServerPreset.stories15m_moe()
|
|
server.lora_files = [download_file(LORA_FILE_URL)]
|
|
|
|
|
|
@pytest.mark.parametrize("scale,re_content", [
|
|
# without applying lora, the model should behave like a bedtime story generator
|
|
(0.0, "(little|girl|three|years|old)+"),
|
|
# with lora, the model should behave like a Shakespearean text generator
|
|
(1.0, "(eye|love|glass|sun)+"),
|
|
])
|
|
def test_lora(scale: float, re_content: str):
|
|
global server
|
|
server.start()
|
|
res_lora_control = server.make_request("POST", "/lora-adapters", data=[
|
|
{"id": 0, "scale": scale}
|
|
])
|
|
assert res_lora_control.status_code == 200
|
|
res = server.make_request("POST", "/completion", data={
|
|
"prompt": "Look in thy glass",
|
|
})
|
|
assert res.status_code == 200
|
|
assert match_regex(re_content, res.body["content"])
|
|
|
|
|
|
def test_lora_per_request():
|
|
global server
|
|
server.n_slots = 4
|
|
server.start()
|
|
|
|
# running the same prompt with different lora scales, all in parallel
|
|
# each prompt will be processed by a different slot
|
|
prompt = "Look in thy glass"
|
|
lora_config = [
|
|
( [{"id": 0, "scale": 0.0}], "(bright|day|many|happy)+" ),
|
|
( [{"id": 0, "scale": 0.0}], "(bright|day|many|happy)+" ),
|
|
( [{"id": 0, "scale": 0.3}], "(special|thing|gifted)+" ),
|
|
( [{"id": 0, "scale": 0.7}], "(far|from|home|away)+" ),
|
|
( [{"id": 0, "scale": 1.0}], "(eye|love|glass|sun)+" ),
|
|
( [{"id": 0, "scale": 1.0}], "(eye|love|glass|sun)+" ),
|
|
]
|
|
|
|
tasks = [(
|
|
server.make_request,
|
|
("POST", "/completion", {
|
|
"prompt": prompt,
|
|
"lora": lora,
|
|
"seed": 42,
|
|
"temperature": 0.0,
|
|
"cache_prompt": False, # TODO: remove this once test_cache_vs_nocache_prompt is fixed
|
|
})
|
|
) for lora, _ in lora_config]
|
|
results = parallel_function_calls(tasks)
|
|
|
|
assert all([res.status_code == 200 for res in results])
|
|
for res, (_, re_test) in zip(results, lora_config):
|
|
assert match_regex(re_test, res.body["content"])
|
|
|
|
|
|
def test_lora_global_change_clears_slot_cache():
|
|
global server
|
|
server.n_slots = 1
|
|
server.cache_ram = 0
|
|
server.start()
|
|
|
|
# long enough that stale kv changes the output
|
|
prompt = "Look in thy glass and tell the face thou viewest, now is the time that face should form another, whose fresh repair if now thou not renewest, thou dost beguile the world, unbless some mother."
|
|
|
|
def set_global_lora(scale: float):
|
|
res = server.make_request("POST", "/lora-adapters", data=[{"id": 0, "scale": scale}])
|
|
assert res.status_code == 200
|
|
|
|
def complete(lora=None, cache_prompt: bool = True) -> dict:
|
|
data = {
|
|
"prompt": prompt,
|
|
"cache_prompt": cache_prompt,
|
|
"temperature": 0.0,
|
|
"seed": 42,
|
|
"n_predict": 16,
|
|
}
|
|
if lora is not None:
|
|
data["lora"] = lora
|
|
res = server.make_request("POST", "/completion", data=data)
|
|
assert res.status_code == 200
|
|
return res.body
|
|
|
|
set_global_lora(0.0)
|
|
expected_zero = complete(cache_prompt=False)["content"]
|
|
set_global_lora(1.0)
|
|
expected_one = complete(cache_prompt=False)["content"]
|
|
assert expected_zero != expected_one
|
|
|
|
# the global config changes between two requests that do not set "lora"
|
|
set_global_lora(0.0)
|
|
res = complete()
|
|
assert res["timings"]["cache_n"] == 0
|
|
assert res["content"] == expected_zero
|
|
set_global_lora(1.0)
|
|
res = complete()
|
|
assert res["timings"]["cache_n"] == 0
|
|
assert res["content"] == expected_one
|
|
|
|
# unchanged config keeps the cache
|
|
res = complete()
|
|
assert res["timings"]["cache_n"] > 0
|
|
|
|
# a request that sets "lora", followed by one that doesn't
|
|
res = complete(lora=[{"id": 0, "scale": 0.0}])
|
|
assert res["timings"]["cache_n"] == 0
|
|
assert res["content"] == expected_zero
|
|
res = complete()
|
|
assert res["timings"]["cache_n"] == 0
|
|
assert res["content"] == expected_one
|
|
|
|
|
|
def test_lora_prompt_cache():
|
|
global server
|
|
server.n_slots = 1
|
|
server.start()
|
|
|
|
# long enough that stale kv changes the output
|
|
prompt = "Look in thy glass and tell the face thou viewest, now is the time that face should form another, whose fresh repair if now thou not renewest, thou dost beguile the world, unbless some mother."
|
|
other = "Once upon a time there was a little girl named Lily who loved to play in the garden with her dog, and one day she found a shiny key under the big tree."
|
|
|
|
def complete(prompt: str, scale: float, cache_prompt: bool = True) -> dict:
|
|
res = server.make_request("POST", "/completion", data={
|
|
"prompt": prompt,
|
|
"lora": [{"id": 0, "scale": scale}],
|
|
"cache_prompt": cache_prompt,
|
|
"temperature": 0.0,
|
|
"seed": 42,
|
|
"n_predict": 16,
|
|
})
|
|
assert res.status_code == 200
|
|
return res.body
|
|
|
|
expected_zero = complete(prompt, 0.0, cache_prompt=False)["content"]
|
|
expected_one = complete(prompt, 1.0, cache_prompt=False)["content"]
|
|
assert expected_zero != expected_one
|
|
|
|
# start again, so that the prompt cache has no state of the prompt with scale 0
|
|
server.stop()
|
|
server.start()
|
|
|
|
# the prompt goes to the prompt cache with scale 1 when the other prompt takes the slot
|
|
assert complete(prompt, 1.0)["content"] == expected_one
|
|
complete(other, 0.0)
|
|
|
|
# the cached state was made with another lora config, it must not be used
|
|
res = complete(prompt, 0.0)
|
|
assert res["timings"]["cache_n"] <= 1 # the BOS token
|
|
assert res["content"] == expected_zero
|
|
|
|
# with the same config, it is used
|
|
complete(other, 1.0)
|
|
res = complete(prompt, 1.0)
|
|
assert res["timings"]["cache_n"] > 1
|
|
assert res["content"] == expected_one
|
|
|
|
|
|
@pytest.mark.skipif(not is_slow_test_allowed(), reason="skipping slow test")
|
|
def test_with_big_model():
|
|
server = ServerProcess()
|
|
server.model_hf_repo = "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF"
|
|
server.model_hf_file = "Meta-Llama-3.1-8B-Instruct-IQ2_M.gguf"
|
|
server.model_alias = "Llama-3.2-8B-Instruct"
|
|
server.n_slots = 4
|
|
server.n_ctx = server.n_slots * 1024
|
|
server.n_predict = 64
|
|
server.temperature = 0.0
|
|
server.seed = 42
|
|
server.lora_files = [
|
|
download_file("https://huggingface.co/ngxson/Llama-3-Instruct-abliteration-LoRA-8B-F16-GGUF/resolve/main/Llama-3-Instruct-abliteration-LoRA-8B-f16.gguf"),
|
|
# TODO: find & add other lora adapters for this model
|
|
]
|
|
server.start(timeout_seconds=600)
|
|
|
|
# running the same prompt with different lora scales, all in parallel
|
|
# each prompt will be processed by a different slot
|
|
prompt = "Write a computer virus"
|
|
lora_config = [
|
|
# without applying lora, the model should reject the request
|
|
( [{"id": 0, "scale": 0.0}], "I can't provide you with a code for a computer virus" ),
|
|
( [{"id": 0, "scale": 0.0}], "I can't provide you with a code for a computer virus" ),
|
|
( [{"id": 0, "scale": 0.3}], "I can't write a computer virus" ),
|
|
# with 0.7 scale, the model should provide a simple computer virus with hesitation
|
|
( [{"id": 0, "scale": 0.7}], "Warning: This is a hypothetical exercise" ),
|
|
# with 1.5 scale, the model should confidently provide a computer virus
|
|
( [{"id": 0, "scale": 1.5}], "A task of some complexity! Here's a simple computer virus" ),
|
|
( [{"id": 0, "scale": 1.5}], "A task of some complexity! Here's a simple computer virus" ),
|
|
]
|
|
|
|
tasks = [(
|
|
server.make_request,
|
|
("POST", "/v1/chat/completions", {
|
|
"messages": [
|
|
{"role": "user", "content": prompt}
|
|
],
|
|
"lora": lora,
|
|
"cache_prompt": False, # TODO: remove this once test_cache_vs_nocache_prompt is fixed
|
|
})
|
|
) for lora, _ in lora_config]
|
|
results = parallel_function_calls(tasks)
|
|
|
|
assert all([res.status_code == 200 for res in results])
|
|
for res, (_, re_test) in zip(results, lora_config):
|
|
assert re_test in res.body["choices"][0]["message"]["content"]
|