* Add FP8 KV-cache support * Add tests for scaled, FP8 KV-cache to test-backend-ops * Fix get_*_scale mapping and dequnatize FP8 also before rope shift * Normalizes cache-scale names independently of projection layout. While fused KQV is currently impractival for non-shared scales, this is an improvement for future use regardless * Enable Gemma 4 Assistant with FP8 by storing and fetching scale ptrs * Update comment wording * Move E2E FP8 KV cache tests to test-llama-archs They belong there as they are not just a test for ggml's backend ops * Increase support surface, remove seed * make scaled kv generic * Fix CI failings for test-kv-cache https://github.com/ggml-org/llama.cpp/actions/runs/38718763447/job/116367842578?pr=28392 * CI: execute kv-cache-scale tests * exclude fp8-kv tests from OpenVINO * Fix linting errors of test_kv_cache_scales * Disable kv-ffload=false + multi-GPU tests for FP8 See https://github.com/ggml-org/llama.cpp/actions/runs/38952722084/job/117148353699?pr=28413 * Adjust test label for test-kv-cache.cpp * Fix conflict-resolution mistakes in test-kv-cache.cpp https://github.com/ggml-org/llama.cpp/actions/runs/40554674463/job/122367426461?pr=28392 * Move KV tests to test-fp8.cpp https://github.com/ggml-org/llama.cpp/pull/27512#discussion_r4410518158 * Remove tmp file for test-kv-cache-scales.py on Windows as well * Fix fp8 kv cache test by population kv_source_impl * Exclude models that consume KV cache directly * Remove obsolete comment * Allow BF16 scales as well for KV cache * Rename *_cache_scale to *_cache_s https://github.com/ggml-org/llama.cpp/pull/28392#discussion_r4517194999 * Move `is_type_scaled()` to `llama-kv-cache.cpp`
58 lines
2.5 KiB
Python
58 lines
2.5 KiB
Python
"""Test KV cache scales with synthetic tensors; no GPU or model downloads required.
|
|
|
|
Install dependencies and run from the repository root:
|
|
python -m pip install -r requirements/requirements-convert_hf_to_gguf.txt
|
|
python -m pip install -e ./gguf-py
|
|
python -m unittest discover -s tests -p 'test_kv_cache_scales.py' -v
|
|
|
|
Use unittest discovery so the repository root is available for imports.
|
|
"""
|
|
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
import torch
|
|
|
|
from conversion.base import ModelBase
|
|
import gguf
|
|
|
|
|
|
class TestKVCacheScales(unittest.TestCase):
|
|
def test_projection_names(self):
|
|
cases = (
|
|
(gguf.MODEL_ARCH.PHI3, "model.layers.2.self_attn.qkv_proj", "model.layers.2.self_attn.qkv_proj"),
|
|
(gguf.MODEL_ARCH.GPT2, "transformer.h.2.attn.c_attn", "transformer.h.2.attn.c_attn"),
|
|
(gguf.MODEL_ARCH.LLAMA, "model.layers.2.self_attn.k_proj", "model.layers.2.self_attn.v_proj"),
|
|
)
|
|
for arch, k_name, v_name in cases:
|
|
for dtype in (torch.float32, torch.bfloat16):
|
|
with self.subTest(arch=arch, dtype=dtype), tempfile.TemporaryDirectory() as tmp:
|
|
path = Path(tmp) / "scales.gguf"
|
|
model = object.__new__(ModelBase)
|
|
model.tensor_map = gguf.get_tensor_name_map(arch, 3)
|
|
model.gguf_writer = gguf.GGUFWriter(path, gguf.MODEL_ARCH_NAMES[arch])
|
|
model.model_tensors = {
|
|
k_name + ".k_scale": lambda: torch.tensor(0.5, dtype=dtype),
|
|
v_name + ".v_scale": lambda: torch.tensor(1.0, dtype=dtype),
|
|
}
|
|
model._prepare_kv_cache_scales()
|
|
self.assertFalse(model.model_tensors)
|
|
model.gguf_writer.write_header_to_file()
|
|
model.gguf_writer.write_kv_data_to_file()
|
|
model.gguf_writer.write_tensors_to_file()
|
|
model.gguf_writer.close()
|
|
|
|
# Release the reader's memory map before Windows removes the temporary directory.
|
|
self._check_scales(path)
|
|
|
|
def _check_scales(self, path):
|
|
reader = gguf.GGUFReader(path)
|
|
self.assertEqual([t.name for t in reader.tensors], ["blk.2.attn_k.k_scale", "blk.2.attn_v.v_scale"])
|
|
self.assertEqual([t.tensor_type for t in reader.tensors], [gguf.GGMLQuantizationType.F32] * 2)
|
|
self.assertEqual([float(t.data[0]) for t in reader.tensors], [0.5, 1.0])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|