* Add FP8 KV-cache support * Add tests for scaled, FP8 KV-cache to test-backend-ops * Fix get_*_scale mapping and dequnatize FP8 also before rope shift * Normalizes cache-scale names independently of projection layout. While fused KQV is currently impractival for non-shared scales, this is an improvement for future use regardless * Enable Gemma 4 Assistant with FP8 by storing and fetching scale ptrs * Update comment wording * Move E2E FP8 KV cache tests to test-llama-archs They belong there as they are not just a test for ggml's backend ops * Increase support surface, remove seed * make scaled kv generic * Fix CI failings for test-kv-cache https://github.com/ggml-org/llama.cpp/actions/runs/38718763447/job/116367842578?pr=28392 * CI: execute kv-cache-scale tests * exclude fp8-kv tests from OpenVINO * Fix linting errors of test_kv_cache_scales * Disable kv-ffload=false + multi-GPU tests for FP8 See https://github.com/ggml-org/llama.cpp/actions/runs/38952722084/job/117148353699?pr=28413 * Adjust test label for test-kv-cache.cpp * Fix conflict-resolution mistakes in test-kv-cache.cpp https://github.com/ggml-org/llama.cpp/actions/runs/40554674463/job/122367426461?pr=28392 * Move KV tests to test-fp8.cpp https://github.com/ggml-org/llama.cpp/pull/27512#discussion_r4410518158 * Remove tmp file for test-kv-cache-scales.py on Windows as well * Fix fp8 kv cache test by population kv_source_impl * Exclude models that consume KV cache directly * Remove obsolete comment * Allow BF16 scales as well for KV cache * Rename *_cache_scale to *_cache_s https://github.com/ggml-org/llama.cpp/pull/28392#discussion_r4517194999 * Move `is_type_scaled()` to `llama-kv-cache.cpp`
106 lines
5.1 KiB
C++
106 lines
5.1 KiB
C++
#include "llama-kv-cache.h"
|
|
#include "llama-model.h"
|
|
|
|
#include <numeric>
|
|
#include <string>
|
|
|
|
static void init_model(llama_model & model, uint32_t n_layer) {
|
|
model.hparams = {};
|
|
model.hparams.n_layer_all = n_layer;
|
|
model.hparams.n_embd_head_k_full = 4;
|
|
model.hparams.n_embd_head_v_full = 4;
|
|
model.hparams.n_head_arr.fill(1);
|
|
model.hparams.n_head_kv_arr.fill(1);
|
|
std::iota(model.hparams.kv_source_impl.begin(), model.hparams.kv_source_impl.end(), 0);
|
|
model.layers.resize(n_layer);
|
|
}
|
|
|
|
int main() {
|
|
llama_backend_init();
|
|
|
|
{
|
|
std::string errors;
|
|
llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
|
|
if (level == GGML_LOG_LEVEL_ERROR) {
|
|
*static_cast<std::string *>(user_data) += text;
|
|
}
|
|
}, &errors);
|
|
|
|
for (llm_arch arch : { LLM_ARCH_DEEPSEEK2, LLM_ARCH_DEEPSEEK32, LLM_ARCH_DEEPSEEK4,
|
|
LLM_ARCH_DEEPSEEK41, LLM_ARCH_GLM_DSA, LLM_ARCH_GLM5_NEXT, LLM_ARCH_DOTS3NOTE,
|
|
LLM_ARCH_MISTRAL_LARGE4, LLM_ARCH_HY_V4, LLM_ARCH_MINIMAX_M3, LLM_ARCH_QWEN4EXP,
|
|
LLM_ARCH_DFLASH }) {
|
|
std::unique_ptr<llama_model> model(llama_model_create(arch, llama_model_default_params()));
|
|
init_model(*model, 1);
|
|
if (arch == LLM_ARCH_DEEPSEEK2) {
|
|
model->hparams.n_embd_head_k_mla_impl = 4;
|
|
model->hparams.n_embd_head_v_mla_impl = 4;
|
|
}
|
|
|
|
for (ggml_type type_k : { GGML_TYPE_F16, GGML_TYPE_F8_E4M3 }) {
|
|
for (ggml_type type_v : { GGML_TYPE_F16, GGML_TYPE_F8_E4M3 }) {
|
|
if (type_k == GGML_TYPE_F16 && type_v == GGML_TYPE_F16) {
|
|
continue;
|
|
}
|
|
auto params = llama_context_default_params();
|
|
params.type_k = type_k;
|
|
params.type_v = type_v;
|
|
errors.clear();
|
|
llama_context * ctx = llama_init_from_model(model.get(), params);
|
|
GGML_ASSERT(ctx == nullptr);
|
|
if (type_k == type_v || (!model->hparams.is_mla() && arch != LLM_ARCH_DEEPSEEK4 && arch != LLM_ARCH_DEEPSEEK41)) {
|
|
GGML_ASSERT(errors.find("FP8 cache is not supported for architecture") != std::string::npos);
|
|
} else {
|
|
GGML_ASSERT(errors.find("model does not support different K") != std::string::npos);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
llama_log_set(nullptr, nullptr);
|
|
}
|
|
|
|
{
|
|
std::unique_ptr<llama_model> target(llama_model_create(LLM_ARCH_GEMMA4, llama_model_default_params()));
|
|
std::unique_ptr<llama_model> assistant(llama_model_create(LLM_ARCH_GEMMA4_ASSISTANT, llama_model_default_params()));
|
|
init_model(*target, 3);
|
|
init_model(*assistant, 2);
|
|
|
|
ggml_context_ptr ctx(ggml_init({ 4096, nullptr, false }));
|
|
target->layers[2].k_cache_s = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_F32, 1);
|
|
target->layers[2].v_cache_s = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_F32, 1);
|
|
assistant->layers[0].k_cache_s = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_F32, 1);
|
|
assistant->layers[0].v_cache_s = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_F32, 1);
|
|
assistant->layers[1].k_cache_s = assistant->layers[0].k_cache_s;
|
|
assistant->layers[1].v_cache_s = assistant->layers[0].v_cache_s;
|
|
ggml_set_f32(target->layers[2].k_cache_s, 0.5f);
|
|
ggml_set_f32(target->layers[2].v_cache_s, 0.25f);
|
|
ggml_set_f32(assistant->layers[0].k_cache_s, 2.0f);
|
|
ggml_set_f32(assistant->layers[0].v_cache_s, 4.0f);
|
|
|
|
for (ggml_type type_k : { GGML_TYPE_F16, GGML_TYPE_F8_E4M3 }) {
|
|
for (ggml_type type_v : { GGML_TYPE_F16, GGML_TYPE_F8_E4M3 }) {
|
|
llama_kv_cache source(*target, false, target->hparams, type_k, type_v,
|
|
false, false, true, 32, 1, 0, LLAMA_SWA_TYPE_NONE, nullptr, nullptr,
|
|
[](uint32_t il) { return il == 0 ? 2 : -1; }, nullptr);
|
|
llama_kv_cache shared(*assistant, false, assistant->hparams, type_k, type_v,
|
|
false, false, true, 32, 1, 0, LLAMA_SWA_TYPE_NONE, &source, nullptr, nullptr,
|
|
[](int32_t il) { return il == 0 ? 2 : 1; });
|
|
|
|
GGML_ASSERT(source.get_k_scale(2) == (type_k == GGML_TYPE_F8_E4M3 ? target->layers[2].k_cache_s : nullptr));
|
|
GGML_ASSERT(source.get_v_scale(2) == (type_v == GGML_TYPE_F8_E4M3 ? target->layers[2].v_cache_s : nullptr));
|
|
GGML_ASSERT(source.get_k_scale(0) == source.get_k_scale(2));
|
|
GGML_ASSERT(source.get_v_scale(0) == source.get_v_scale(2));
|
|
GGML_ASSERT(shared.get_k_storage(0) == source.get_k_storage(2));
|
|
GGML_ASSERT(shared.get_k_scale(0) == source.get_k_scale(2));
|
|
GGML_ASSERT(shared.get_v_scale(0) == source.get_v_scale(2));
|
|
GGML_ASSERT(shared.get_k_scale(1) == nullptr);
|
|
GGML_ASSERT(shared.get_v_scale(1) == nullptr);
|
|
}
|
|
}
|
|
}
|
|
|
|
llama_backend_free();
|
|
return 0;
|
|
}
|