Files
llama.cpp/tests/test_kv_cache_scales.py
T
Oliver Simons 202c1a229f llama : Add FP8 KV-cache support (#28392)
* Add FP8 KV-cache support

* Add tests for scaled, FP8 KV-cache to test-backend-ops

* Fix get_*_scale mapping and dequnatize FP8 also before rope shift

* Normalizes cache-scale names independently of projection layout.

While fused KQV is currently impractival for non-shared scales, this is
an improvement for future use regardless

* Enable Gemma 4 Assistant with FP8 by storing and fetching scale ptrs

* Update comment wording

* Move E2E FP8 KV cache tests to test-llama-archs

They belong there as they are not just a test for ggml's backend ops

* Increase support surface, remove seed

* make scaled kv generic

* Fix CI failings for test-kv-cache

https://github.com/ggml-org/llama.cpp/actions/runs/38718763447/job/116367842578?pr=28392

* CI: execute kv-cache-scale tests

* exclude fp8-kv tests from OpenVINO

* Fix linting errors of test_kv_cache_scales

* Disable kv-ffload=false + multi-GPU tests for FP8

See https://github.com/ggml-org/llama.cpp/actions/runs/38952722084/job/117148353699?pr=28413

* Adjust test label for test-kv-cache.cpp

* Fix conflict-resolution mistakes in test-kv-cache.cpp

https://github.com/ggml-org/llama.cpp/actions/runs/40554674463/job/122367426461?pr=28392

* Move KV tests to test-fp8.cpp

https://github.com/ggml-org/llama.cpp/pull/27512#discussion_r4410518158

* Remove tmp file for test-kv-cache-scales.py on Windows as well

* Fix fp8 kv cache test by population kv_source_impl

* Exclude models that consume KV cache directly

* Remove obsolete comment

* Allow BF16 scales as well for KV cache

* Rename *_cache_scale to *_cache_s

https://github.com/ggml-org/llama.cpp/pull/28392#discussion_r4517194999

* Move `is_type_scaled()` to `llama-kv-cache.cpp`
2026-11-10 18:08:32 +02:00

58 lines
2.5 KiB
Python

"""Test KV cache scales with synthetic tensors; no GPU or model downloads required.
Install dependencies and run from the repository root:
python -m pip install -r requirements/requirements-convert_hf_to_gguf.txt
python -m pip install -e ./gguf-py
python -m unittest discover -s tests -p 'test_kv_cache_scales.py' -v
Use unittest discovery so the repository root is available for imports.
"""
import tempfile
import unittest
from pathlib import Path
import torch
from conversion.base import ModelBase
import gguf
class TestKVCacheScales(unittest.TestCase):
def test_projection_names(self):
cases = (
(gguf.MODEL_ARCH.PHI3, "model.layers.2.self_attn.qkv_proj", "model.layers.2.self_attn.qkv_proj"),
(gguf.MODEL_ARCH.GPT2, "transformer.h.2.attn.c_attn", "transformer.h.2.attn.c_attn"),
(gguf.MODEL_ARCH.LLAMA, "model.layers.2.self_attn.k_proj", "model.layers.2.self_attn.v_proj"),
)
for arch, k_name, v_name in cases:
for dtype in (torch.float32, torch.bfloat16):
with self.subTest(arch=arch, dtype=dtype), tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "scales.gguf"
model = object.__new__(ModelBase)
model.tensor_map = gguf.get_tensor_name_map(arch, 3)
model.gguf_writer = gguf.GGUFWriter(path, gguf.MODEL_ARCH_NAMES[arch])
model.model_tensors = {
k_name + ".k_scale": lambda: torch.tensor(0.5, dtype=dtype),
v_name + ".v_scale": lambda: torch.tensor(1.0, dtype=dtype),
}
model._prepare_kv_cache_scales()
self.assertFalse(model.model_tensors)
model.gguf_writer.write_header_to_file()
model.gguf_writer.write_kv_data_to_file()
model.gguf_writer.write_tensors_to_file()
model.gguf_writer.close()
# Release the reader's memory map before Windows removes the temporary directory.
self._check_scales(path)
def _check_scales(self, path):
reader = gguf.GGUFReader(path)
self.assertEqual([t.name for t in reader.tensors], ["blk.2.attn_k.k_scale", "blk.2.attn_v.v_scale"])
self.assertEqual([t.tensor_type for t in reader.tensors], [gguf.GGMLQuantizationType.F32] * 2)
self.assertEqual([float(t.data[0]) for t in reader.tensors], [0.5, 1.0])
if __name__ == "__main__":
unittest.main()