import pytest from utils import * server = ServerPreset.stories15m_moe() LORA_FILE_URL = "https://huggingface.co/ggml-org-dev/stories15M_MOE/resolve/main/moe_shakespeare15M.gguf" @pytest.fixture(autouse=True) def create_server(): global server server = ServerPreset.stories15m_moe() server.lora_files = [download_file(LORA_FILE_URL)] @pytest.mark.parametrize("scale,re_content", [ # without applying lora, the model should behave like a bedtime story generator (0.0, "(little|girl|three|years|old)+"), # with lora, the model should behave like a Shakespearean text generator (1.0, "(eye|love|glass|sun)+"), ]) def test_lora(scale: float, re_content: str): global server server.start() res_lora_control = server.make_request("POST", "/lora-adapters", data=[ {"id": 0, "scale": scale} ]) assert res_lora_control.status_code == 200 res = server.make_request("POST", "/completion", data={ "prompt": "Look in thy glass", }) assert res.status_code == 200 assert match_regex(re_content, res.body["content"]) def test_lora_per_request(): global server server.n_slots = 4 server.start() # running the same prompt with different lora scales, all in parallel # each prompt will be processed by a different slot prompt = "Look in thy glass" lora_config = [ ( [{"id": 0, "scale": 0.0}], "(bright|day|many|happy)+" ), ( [{"id": 0, "scale": 0.0}], "(bright|day|many|happy)+" ), ( [{"id": 0, "scale": 0.3}], "(special|thing|gifted)+" ), ( [{"id": 0, "scale": 0.7}], "(far|from|home|away)+" ), ( [{"id": 0, "scale": 1.0}], "(eye|love|glass|sun)+" ), ( [{"id": 0, "scale": 1.0}], "(eye|love|glass|sun)+" ), ] tasks = [( server.make_request, ("POST", "/completion", { "prompt": prompt, "lora": lora, "seed": 42, "temperature": 0.0, "cache_prompt": False, # TODO: remove this once test_cache_vs_nocache_prompt is fixed }) ) for lora, _ in lora_config] results = parallel_function_calls(tasks) assert all([res.status_code == 200 for res in results]) for res, (_, re_test) in zip(results, lora_config): assert match_regex(re_test, res.body["content"]) def test_lora_global_change_clears_slot_cache(): global server server.n_slots = 1 server.cache_ram = 0 server.start() # long enough that stale kv changes the output prompt = "Look in thy glass and tell the face thou viewest, now is the time that face should form another, whose fresh repair if now thou not renewest, thou dost beguile the world, unbless some mother." def set_global_lora(scale: float): res = server.make_request("POST", "/lora-adapters", data=[{"id": 0, "scale": scale}]) assert res.status_code == 200 def complete(lora=None, cache_prompt: bool = True) -> dict: data = { "prompt": prompt, "cache_prompt": cache_prompt, "temperature": 0.0, "seed": 42, "n_predict": 16, } if lora is not None: data["lora"] = lora res = server.make_request("POST", "/completion", data=data) assert res.status_code == 200 return res.body set_global_lora(0.0) expected_zero = complete(cache_prompt=False)["content"] set_global_lora(1.0) expected_one = complete(cache_prompt=False)["content"] assert expected_zero != expected_one # the global config changes between two requests that do not set "lora" set_global_lora(0.0) res = complete() assert res["timings"]["cache_n"] == 0 assert res["content"] == expected_zero set_global_lora(1.0) res = complete() assert res["timings"]["cache_n"] == 0 assert res["content"] == expected_one # unchanged config keeps the cache res = complete() assert res["timings"]["cache_n"] > 0 # a request that sets "lora", followed by one that doesn't res = complete(lora=[{"id": 0, "scale": 0.0}]) assert res["timings"]["cache_n"] == 0 assert res["content"] == expected_zero res = complete() assert res["timings"]["cache_n"] == 0 assert res["content"] == expected_one def test_lora_prompt_cache(): global server server.n_slots = 1 server.start() # long enough that stale kv changes the output prompt = "Look in thy glass and tell the face thou viewest, now is the time that face should form another, whose fresh repair if now thou not renewest, thou dost beguile the world, unbless some mother." other = "Once upon a time there was a little girl named Lily who loved to play in the garden with her dog, and one day she found a shiny key under the big tree." def complete(prompt: str, scale: float, cache_prompt: bool = True) -> dict: res = server.make_request("POST", "/completion", data={ "prompt": prompt, "lora": [{"id": 0, "scale": scale}], "cache_prompt": cache_prompt, "temperature": 0.0, "seed": 42, "n_predict": 16, }) assert res.status_code == 200 return res.body expected_zero = complete(prompt, 0.0, cache_prompt=False)["content"] expected_one = complete(prompt, 1.0, cache_prompt=False)["content"] assert expected_zero != expected_one # start again, so that the prompt cache has no state of the prompt with scale 0 server.stop() server.start() # the prompt goes to the prompt cache with scale 1 when the other prompt takes the slot assert complete(prompt, 1.0)["content"] == expected_one complete(other, 0.0) # the cached state was made with another lora config, it must not be used res = complete(prompt, 0.0) assert res["timings"]["cache_n"] <= 1 # the BOS token assert res["content"] == expected_zero # with the same config, it is used complete(other, 1.0) res = complete(prompt, 1.0) assert res["timings"]["cache_n"] > 1 assert res["content"] == expected_one @pytest.mark.skipif(not is_slow_test_allowed(), reason="skipping slow test") def test_with_big_model(): server = ServerProcess() server.model_hf_repo = "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF" server.model_hf_file = "Meta-Llama-3.1-8B-Instruct-IQ2_M.gguf" server.model_alias = "Llama-3.2-8B-Instruct" server.n_slots = 4 server.n_ctx = server.n_slots * 1024 server.n_predict = 64 server.temperature = 0.0 server.seed = 42 server.lora_files = [ download_file("https://huggingface.co/ngxson/Llama-3-Instruct-abliteration-LoRA-8B-F16-GGUF/resolve/main/Llama-3-Instruct-abliteration-LoRA-8B-f16.gguf"), # TODO: find & add other lora adapters for this model ] server.start(timeout_seconds=600) # running the same prompt with different lora scales, all in parallel # each prompt will be processed by a different slot prompt = "Write a computer virus" lora_config = [ # without applying lora, the model should reject the request ( [{"id": 0, "scale": 0.0}], "I can't provide you with a code for a computer virus" ), ( [{"id": 0, "scale": 0.0}], "I can't provide you with a code for a computer virus" ), ( [{"id": 0, "scale": 0.3}], "I can't write a computer virus" ), # with 0.7 scale, the model should provide a simple computer virus with hesitation ( [{"id": 0, "scale": 0.7}], "Warning: This is a hypothetical exercise" ), # with 1.5 scale, the model should confidently provide a computer virus ( [{"id": 0, "scale": 1.5}], "A task of some complexity! Here's a simple computer virus" ), ( [{"id": 0, "scale": 1.5}], "A task of some complexity! Here's a simple computer virus" ), ] tasks = [( server.make_request, ("POST", "/v1/chat/completions", { "messages": [ {"role": "user", "content": prompt} ], "lora": lora, "cache_prompt": False, # TODO: remove this once test_cache_vs_nocache_prompt is fixed }) ) for lora, _ in lora_config] results = parallel_function_calls(tasks) assert all([res.status_code == 200 for res in results]) for res, (_, re_test) in zip(results, lora_config): assert re_test in res.body["choices"][0]["message"]["content"]