* Add FP8 KV-cache support * Add tests for scaled, FP8 KV-cache to test-backend-ops * Fix get_*_scale mapping and dequnatize FP8 also before rope shift * Normalizes cache-scale names independently of projection layout. While fused KQV is currently impractival for non-shared scales, this is an improvement for future use regardless * Enable Gemma 4 Assistant with FP8 by storing and fetching scale ptrs * Update comment wording * Move E2E FP8 KV cache tests to test-llama-archs They belong there as they are not just a test for ggml's backend ops * Increase support surface, remove seed * make scaled kv generic * Fix CI failings for test-kv-cache https://github.com/ggml-org/llama.cpp/actions/runs/38718763447/job/116367842578?pr=28392 * CI: execute kv-cache-scale tests * exclude fp8-kv tests from OpenVINO * Fix linting errors of test_kv_cache_scales * Disable kv-ffload=false + multi-GPU tests for FP8 See https://github.com/ggml-org/llama.cpp/actions/runs/38952722084/job/117148353699?pr=28413 * Adjust test label for test-kv-cache.cpp * Fix conflict-resolution mistakes in test-kv-cache.cpp https://github.com/ggml-org/llama.cpp/actions/runs/40554674463/job/122367426461?pr=28392 * Move KV tests to test-fp8.cpp https://github.com/ggml-org/llama.cpp/pull/27512#discussion_r4410518158 * Remove tmp file for test-kv-cache-scales.py on Windows as well * Fix fp8 kv cache test by population kv_source_impl * Exclude models that consume KV cache directly * Remove obsolete comment * Allow BF16 scales as well for KV cache * Rename *_cache_scale to *_cache_s https://github.com/ggml-org/llama.cpp/pull/28392#discussion_r4517194999 * Move `is_type_scaled()` to `llama-kv-cache.cpp`
719 lines
33 KiB
C++
719 lines
33 KiB
C++
#include "common.h"
|
|
#include "log.h"
|
|
#include "ggml.h"
|
|
#include "ggml-backend.h"
|
|
#include "ggml-cpu.h"
|
|
#include "ggml-cpp.h"
|
|
#include "gguf.h"
|
|
#include "llama.h"
|
|
#include "llama-cpp.h"
|
|
#include "../src/llama-context.h"
|
|
#include "../src/llama-kv-cache.h"
|
|
#include "../src/llama-model.h"
|
|
|
|
#include <algorithm>
|
|
#include <cmath>
|
|
#include <cstdio>
|
|
#include <cstring>
|
|
#include <filesystem>
|
|
#include <memory>
|
|
#include <random>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
// normalized mean squared error = mse(a, b) / mse(a, 0)
|
|
static double nmse(const std::vector<float> & a, const std::vector<float> & b) {
|
|
GGML_ASSERT(a.size() == b.size());
|
|
double mse_a_b = 0.0;
|
|
double mse_a_0 = 0.0;
|
|
|
|
for (size_t i = 0; i < a.size(); i++) {
|
|
float a_i = a[i];
|
|
float b_i = b[i];
|
|
|
|
mse_a_b += (a_i - b_i) * (a_i - b_i);
|
|
mse_a_0 += a_i * a_i;
|
|
}
|
|
|
|
return mse_a_b / mse_a_0;
|
|
}
|
|
|
|
static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed) {
|
|
std::mt19937 gen(seed);
|
|
std::uniform_int_distribution<> dis(0, n_vocab - 1);
|
|
std::vector<llama_token> ret;
|
|
ret.reserve(n_tokens);
|
|
for (uint32_t i = 0; i < n_tokens; i++) {
|
|
ret.push_back(dis(gen));
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
static std::vector<float> get_logits(
|
|
llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {
|
|
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
|
|
const uint32_t n_ctx = llama_n_ctx(lctx);
|
|
const uint32_t n_tokens = tokens.size();
|
|
common_batch batch(lctx);
|
|
GGML_ASSERT(n_tokens <= n_ctx);
|
|
for (uint32_t pos = 0; pos < n_tokens; pos++) {
|
|
batch.add(tokens[pos], pos, 0, true);
|
|
}
|
|
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
|
|
throw std::runtime_error("failed to decode batch");
|
|
}
|
|
|
|
std::vector<float> ret;
|
|
ret.reserve(n_tokens*n_vocab);
|
|
for (uint32_t i = 0; i < n_tokens; i++) {
|
|
const float * logits_ith = llama_get_logits_ith(lctx, i);
|
|
for (uint32_t j = 0; j < n_vocab; j++) {
|
|
ret.push_back(logits_ith[j]);
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
static const char * FP8_TEST_WEIGHTS[] = {
|
|
"output.weight", "blk.0.attn_q.weight",
|
|
"blk.0.ffn_gate_exps.weight", "blk.0.ffn_up_exps.weight", "blk.0.ffn_down_exps.weight",
|
|
};
|
|
|
|
static int64_t prepare_models(const std::string & path, const std::string & input, const std::string & reference) {
|
|
ggml_context * ctx_raw = nullptr;
|
|
gguf_context_ptr source(gguf_init_from_file(path.c_str(), {false, &ctx_raw}));
|
|
ggml_context_ptr source_ctx(ctx_raw);
|
|
if (!source || !source_ctx) {
|
|
throw std::runtime_error("failed to read FP8 test fixture");
|
|
}
|
|
ggml_tensor * attn_q = ggml_get_tensor(source_ctx.get(), "blk.0.attn_q.weight");
|
|
if (!attn_q || !ggml_get_tensor(source_ctx.get(), "output.weight")) {
|
|
throw std::runtime_error("FP8 test fixture must have attention Q and output weights");
|
|
}
|
|
gguf_context_ptr fp8(gguf_init_empty());
|
|
gguf_context_ptr folded(gguf_init_empty());
|
|
gguf_set_kv(fp8.get(), source.get());
|
|
gguf_set_kv(folded.get(), source.get());
|
|
for (int64_t i = 0; i < gguf_get_n_tensors(source.get()); ++i) {
|
|
ggml_tensor * tensor = ggml_get_tensor(source_ctx.get(), gguf_get_tensor_name(source.get(), i));
|
|
gguf_add_tensor(fp8.get(), tensor);
|
|
gguf_add_tensor(folded.get(), tensor);
|
|
}
|
|
|
|
ggml_context_ptr scale_ctx(ggml_init({16 * ggml_tensor_overhead(), nullptr, true}));
|
|
// GGUF keeps pointers into these buffers until the files are written.
|
|
std::vector<std::vector<uint8_t>> fp8_data;
|
|
std::vector<std::vector<ggml_fp16_t>> folded_data;
|
|
std::vector<std::vector<float>> scale_data;
|
|
for (const char * name : FP8_TEST_WEIGHTS) {
|
|
ggml_tensor * tensor = ggml_get_tensor(source_ctx.get(), name);
|
|
if (!tensor) {
|
|
continue;
|
|
}
|
|
std::vector<float> values(ggml_nelements(tensor));
|
|
if (tensor->type == GGML_TYPE_F32) {
|
|
memcpy(values.data(), tensor->data, ggml_nbytes(tensor));
|
|
} else {
|
|
const auto * traits = ggml_get_type_traits(tensor->type);
|
|
if (!traits->to_float) {
|
|
throw std::runtime_error("unsupported FP8 test fixture tensor type");
|
|
}
|
|
traits->to_float(tensor->data, values.data(), values.size());
|
|
}
|
|
const auto * traits = ggml_get_type_traits(GGML_TYPE_F8_E4M3);
|
|
fp8_data.emplace_back(values.size());
|
|
traits->from_float_ref(values.data(), fp8_data.back().data(), values.size());
|
|
gguf_set_tensor_type(fp8.get(), name, GGML_TYPE_F8_E4M3);
|
|
gguf_set_tensor_data(fp8.get(), name, fp8_data.back().data());
|
|
|
|
traits->to_float(fp8_data.back().data(), values.data(), values.size());
|
|
const size_t expert_size = tensor->ne[0] * tensor->ne[1];
|
|
for (size_t i = 0; i < values.size(); ++i) {
|
|
values[i] *= i < expert_size ? 2.0f : 4.0f;
|
|
}
|
|
folded_data.emplace_back(values.size());
|
|
ggml_fp32_to_fp16_row(values.data(), folded_data.back().data(), values.size());
|
|
gguf_set_tensor_type(folded.get(), name, GGML_TYPE_F16);
|
|
gguf_set_tensor_data(folded.get(), name, folded_data.back().data());
|
|
|
|
const std::string prefix(name, strlen(name) - strlen("weight"));
|
|
for (const char * suffix : {"scale", "input_scale"}) {
|
|
const std::string scale_name = prefix + suffix;
|
|
ggml_tensor * scale = ggml_get_tensor(source_ctx.get(), scale_name.c_str());
|
|
if (scale) {
|
|
if (scale->type != GGML_TYPE_F32 || ggml_nelements(scale) != tensor->ne[2]) {
|
|
throw std::runtime_error("unsupported FP8 test fixture scale shape or type");
|
|
}
|
|
scale_data.emplace_back(tensor->ne[2], 1.0f);
|
|
gguf_set_tensor_data(folded.get(), scale_name.c_str(), scale_data.back().data());
|
|
} else {
|
|
scale = ggml_new_tensor_1d(scale_ctx.get(), GGML_TYPE_F32, tensor->ne[2]);
|
|
ggml_set_name(scale, scale_name.c_str());
|
|
gguf_add_tensor(fp8.get(), scale);
|
|
}
|
|
// Scales above one avoid subnormal Q6_K block scales in the folded reference.
|
|
scale_data.emplace_back(tensor->ne[2], 4.0f);
|
|
if (strcmp(suffix, "scale") == 0) {
|
|
scale_data.back()[0] = 2.0f;
|
|
}
|
|
gguf_set_tensor_data(fp8.get(), scale_name.c_str(), scale_data.back().data());
|
|
}
|
|
}
|
|
if (!gguf_write_to_file(fp8.get(), input.c_str(), false) ||
|
|
!gguf_write_to_file(folded.get(), reference.c_str(), false)) {
|
|
throw std::runtime_error("failed to write FP8 test models");
|
|
}
|
|
return attn_q->ne[0];
|
|
}
|
|
|
|
static bool silent_model_load_progress(float /*progress*/, void * /*user_data*/) {
|
|
return true;
|
|
}
|
|
|
|
static llama_model_ptr load_model(const std::string & path, const llama_model_params & params, FILE * file = nullptr) {
|
|
llama_model_ptr model(file ? llama_model_load_from_file_ptr(file, params) :
|
|
llama_model_load_from_file(path.c_str(), params));
|
|
if (!model) {
|
|
throw std::runtime_error("failed to load FP8 test model");
|
|
}
|
|
return model;
|
|
}
|
|
|
|
static std::vector<float> load_logits(const std::string & path, const llama_model_params & model_params, bool split = false) {
|
|
std::unique_ptr<FILE, int (*)(FILE *)> file(split ? nullptr : fopen(path.c_str(), "rb"), fclose);
|
|
if (!split && !file) {
|
|
throw std::runtime_error("failed to open FP8 test model");
|
|
}
|
|
llama_model_ptr model = load_model(path, model_params, file.get());
|
|
auto ctx_params = llama_context_default_params();
|
|
ctx_params.n_ctx = 0;
|
|
ctx_params.n_ubatch = 64;
|
|
ctx_params.n_threads = ctx_params.n_threads_batch = 4;
|
|
llama_context_ptr lctx(llama_init_from_model(model.get(), ctx_params));
|
|
if (!lctx) {
|
|
throw std::runtime_error("failed to create FP8 test context");
|
|
}
|
|
const auto tokens = get_tokens(8, llama_vocab_n_tokens(llama_model_get_vocab(model.get())), 1);
|
|
return get_logits(model.get(), lctx.get(), tokens);
|
|
}
|
|
|
|
static void check_model_save(const std::string & path, const llama_model_params & model_params) {
|
|
const std::string saved = path + ".saved.gguf";
|
|
const auto logits = load_logits(path, model_params);
|
|
auto save_params = model_params;
|
|
// Packed CPU buffers do not support reading weights back for saving.
|
|
save_params.use_extra_bufts = false;
|
|
save_params.load_mode = LLAMA_LOAD_MODE_NONE;
|
|
{
|
|
llama_model_ptr model = load_model(path, save_params);
|
|
llama_model_save_to_file(model.get(), saved.c_str());
|
|
}
|
|
const double error = nmse(logits, load_logits(saved, model_params));
|
|
LOG("FP8 model save: %s nmse=%.2e\n", path.c_str(), error);
|
|
if (!(error < 1e-10)) {
|
|
throw std::runtime_error("saved FP8 test model differs from original model");
|
|
}
|
|
std::filesystem::rename(saved, path);
|
|
}
|
|
|
|
static void check_fp8_scales(const std::string & input, const std::string & reference, const llama_model_params & model_params) {
|
|
const auto reference_logits = load_logits(reference, model_params);
|
|
const double source_error = nmse(reference_logits, load_logits(input, model_params));
|
|
LOG("FP8 sidecar scales: nmse=%.2e\n", source_error);
|
|
if (!(source_error < 1e-5)) {
|
|
throw std::runtime_error("FP8 sidecar scales differ from folded weights");
|
|
}
|
|
}
|
|
|
|
static void check_split_model(const std::string & path, const std::filesystem::path & dir, const llama_model_params & model_params) {
|
|
ggml_context * ctx_raw = nullptr;
|
|
gguf_context_ptr metadata(gguf_init_from_file(path.c_str(), {false, &ctx_raw}));
|
|
ggml_context_ptr ctx(ctx_raw);
|
|
if (!metadata || !ctx) {
|
|
throw std::runtime_error("failed to read FP8 test model for splitting");
|
|
}
|
|
const auto split_first = (dir / "split-00001-of-00002.gguf").string();
|
|
const auto split_second = (dir / "split-00002-of-00002.gguf").string();
|
|
for (int split = 0; split < 2; ++split) {
|
|
gguf_context_ptr shard(gguf_init_empty());
|
|
gguf_set_kv(shard.get(), metadata.get());
|
|
gguf_set_val_u16(shard.get(), "split.no", split);
|
|
gguf_set_val_u16(shard.get(), "split.count", 2);
|
|
gguf_set_val_i32(shard.get(), "split.tensors.count", gguf_get_n_tensors(metadata.get()));
|
|
for (int64_t i = 0; i < gguf_get_n_tensors(metadata.get()); ++i) {
|
|
auto * tensor = ggml_get_tensor(ctx.get(), gguf_get_tensor_name(metadata.get(), i));
|
|
// Put output scales in the second shard to test lookup across splits.
|
|
const bool output_scale = strcmp(tensor->name, "output.scale") == 0 || strcmp(tensor->name, "output.input_scale") == 0;
|
|
if (output_scale == (split == 1)) {
|
|
gguf_add_tensor(shard.get(), tensor);
|
|
}
|
|
}
|
|
if (!gguf_write_to_file(shard.get(), (split == 0 ? split_first : split_second).c_str(), false)) {
|
|
throw std::runtime_error("failed to write split FP8 test model");
|
|
}
|
|
}
|
|
const double split_error = nmse(load_logits(path, model_params), load_logits(split_first, model_params, true));
|
|
LOG("FP8 split model: nmse=%.2e\n", split_error);
|
|
if (!(split_error < 1e-10)) {
|
|
throw std::runtime_error("split FP8 model differs from unsplit model");
|
|
}
|
|
}
|
|
|
|
static void check_fp8_requantize(
|
|
const std::string & input, const std::string & reference,
|
|
const std::string & output, const std::string & reference_output,
|
|
int64_t n_embd, const llama_model_params & model_params) {
|
|
for (llama_ftype ftype : {LLAMA_FTYPE_ALL_F32, LLAMA_FTYPE_MOSTLY_F16, LLAMA_FTYPE_MOSTLY_BF16, LLAMA_FTYPE_MOSTLY_Q4_K_M}) {
|
|
for (int nthread : {1, 3}) {
|
|
std::vector<float> importance(n_embd, 1.0f);
|
|
llama_model_imatrix_data imatrix[] = {
|
|
{"blk.0.attn_q.weight", importance.data(), importance.size()},
|
|
{nullptr, nullptr, 0},
|
|
};
|
|
auto params = llama_model_quantize_default_params();
|
|
params.ftype = ftype;
|
|
params.nthread = nthread;
|
|
if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M && nthread == 3) {
|
|
params.imatrix = imatrix;
|
|
}
|
|
if (llama_model_quantize(input.c_str(), output.c_str(), ¶ms) != 0 ||
|
|
llama_model_quantize(reference.c_str(), reference_output.c_str(), ¶ms) != 0) {
|
|
throw std::runtime_error("FP8 model requantization failed");
|
|
}
|
|
const double error = nmse(load_logits(reference_output, model_params), load_logits(output, model_params));
|
|
LOG("FP8 requantization: ftype=%d threads=%d nmse=%.2e\n", ftype, nthread, error);
|
|
if (!(error < 1e-4)) {
|
|
throw std::runtime_error("FP8 requantization differs from folded weights");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
static void check_fp8_attention(llama_model * model, bool offload_kqv = true) {
|
|
// TODO: enable CPU KV cache tests with tensor split after the meta backend handles the attention transfers.
|
|
if (model->split_mode() == LLAMA_SPLIT_MODE_TENSOR && !offload_kqv) {
|
|
LOG("FP8 attention: tensor split with CPU KV cache is not supported, skipping\n");
|
|
return;
|
|
}
|
|
|
|
ggml_context_ptr scale_ctx(ggml_init({ 4096, nullptr, true }));
|
|
ggml_tensor * k_scale = ggml_new_tensor_1d(scale_ctx.get(), GGML_TYPE_F32, 1);
|
|
ggml_tensor * v_scale = ggml_new_tensor_1d(scale_ctx.get(), GGML_TYPE_F32, 1);
|
|
ggml_set_name(k_scale, "blk.0.attn_k.k_scale");
|
|
ggml_set_name(v_scale, "blk.0.attn_v.v_scale");
|
|
ggml_backend_buffer_ptr scale_buf(ggml_backend_alloc_ctx_tensors_from_buft(scale_ctx.get(), ggml_backend_dev_buffer_type(model->dev_layer(0))));
|
|
ggml_tensor * old_k_scale = model->layers[0].k_cache_s;
|
|
ggml_tensor * old_v_scale = model->layers[0].v_cache_s;
|
|
|
|
const int64_t head_size = model->hparams.n_embd_head_k();
|
|
const int64_t n_head = model->hparams.n_head();
|
|
const int64_t n_tokens = 4;
|
|
GGML_ASSERT(n_head == model->hparams.n_head_kv());
|
|
GGML_ASSERT(head_size == model->hparams.n_embd_head_v());
|
|
|
|
for (llama_flash_attn_type flash : { LLAMA_FLASH_ATTN_TYPE_DISABLED, LLAMA_FLASH_ATTN_TYPE_ENABLED }) {
|
|
if (model->split_mode() == LLAMA_SPLIT_MODE_TENSOR && flash == LLAMA_FLASH_ATTN_TYPE_DISABLED) {
|
|
LOG("FP8 attention: tensor split requires flash attention, skipping disabled cases\n");
|
|
continue;
|
|
}
|
|
for (int scaled : { 1, 2, 3 }) {
|
|
const float sk = scaled & 1 ? 0.03125f : 1.0f;
|
|
const float sv = scaled & 2 ? 0.125f : 1.0f;
|
|
ggml_backend_tensor_set(k_scale, &sk, 0, sizeof(sk));
|
|
ggml_backend_tensor_set(v_scale, &sv, 0, sizeof(sv));
|
|
model->layers[0].k_cache_s = scaled & 1 ? k_scale : nullptr;
|
|
model->layers[0].v_cache_s = scaled & 2 ? v_scale : nullptr;
|
|
|
|
std::vector<float> tensors[4];
|
|
auto params = llama_context_default_params();
|
|
params.n_ctx = 256;
|
|
params.n_batch = n_tokens;
|
|
params.n_ubatch = n_tokens;
|
|
params.n_threads = 1;
|
|
params.n_threads_batch = 1;
|
|
params.type_k = scaled & 1 ? GGML_TYPE_F8_E4M3 : GGML_TYPE_F16;
|
|
params.type_v = scaled & 2 ? GGML_TYPE_F8_E4M3 : GGML_TYPE_F16;
|
|
params.flash_attn_type = flash;
|
|
params.offload_kqv = offload_kqv;
|
|
if (model->split_mode() == LLAMA_SPLIT_MODE_TENSOR) {
|
|
ggml_context_ptr ctx(ggml_init({ 8*ggml_tensor_overhead(), nullptr, true }));
|
|
ggml_tensor * q = ggml_new_tensor_4d(ctx.get(), GGML_TYPE_F32, head_size, n_tokens, n_head, 1);
|
|
ggml_tensor * k = ggml_new_tensor_4d(ctx.get(), params.type_k, head_size, params.n_ctx, n_head, 1);
|
|
ggml_tensor * v = ggml_new_tensor_4d(ctx.get(), params.type_v, head_size, params.n_ctx, n_head, 1);
|
|
ggml_tensor * mask = ggml_new_tensor_4d(ctx.get(), GGML_TYPE_F16, params.n_ctx, params.n_ctx, 1, 1);
|
|
ggml_tensor * attn = ggml_flash_attn_ext(ctx.get(), q, k, v, mask, 1.0f/sqrtf(head_size), 0.0f, 0.0f);
|
|
auto * dev = model->dev_layer(0);
|
|
if (!ggml_backend_dev_supports_op(dev, attn)) {
|
|
LOG("FP8 attention: scaled=%d offload=%d: tensor-split flash attention not supported on %s, skipping\n",
|
|
scaled, offload_kqv, ggml_backend_dev_name(dev));
|
|
continue;
|
|
}
|
|
}
|
|
if (offload_kqv) {
|
|
// Cache writes cannot fall back from a preallocated device buffer.
|
|
llama_kv_cache probe(*model, false, model->hparams, params.type_k, params.type_v,
|
|
flash == LLAMA_FLASH_ATTN_TYPE_DISABLED, false, true, params.n_ctx, 1,
|
|
0, LLAMA_SWA_TYPE_NONE, nullptr, nullptr, nullptr, nullptr);
|
|
ggml_context_ptr ctx(ggml_init({ 64*ggml_tensor_overhead(), nullptr, true }));
|
|
llama_kv_cache::slot_info slots;
|
|
bool supported = true;
|
|
for (uint32_t il = 0; il < model->hparams.n_layer(); ++il) {
|
|
ggml_tensor * k = ggml_new_tensor_3d(ctx.get(), GGML_TYPE_F32, head_size, n_head, n_tokens);
|
|
ggml_tensor * v = ggml_new_tensor_3d(ctx.get(), GGML_TYPE_F32, head_size, n_head, n_tokens);
|
|
ggml_tensor * k_idxs = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I64, n_tokens);
|
|
ggml_tensor * v_idxs = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I64,
|
|
flash == LLAMA_FLASH_ATTN_TYPE_DISABLED ? n_tokens*head_size*n_head : n_tokens);
|
|
auto * dev = model->dev_layer(il);
|
|
if (!ggml_backend_dev_supports_op(dev, probe.cpy_k(ctx.get(), k, k_idxs, il, slots)) ||
|
|
!ggml_backend_dev_supports_op(dev, probe.cpy_v(ctx.get(), v, v_idxs, il, slots))) {
|
|
LOG("FP8 attention: flash=%s scaled=%d offload=1: cache writes not supported on %s, skipping\n",
|
|
llama_flash_attn_type_name(flash), scaled, ggml_backend_dev_name(dev));
|
|
supported = false;
|
|
break;
|
|
}
|
|
}
|
|
if (!supported) {
|
|
continue;
|
|
}
|
|
}
|
|
params.cb_eval_user_data = tensors;
|
|
params.cb_eval = [](ggml_tensor * t, bool ask, void * user_data) {
|
|
const char * names[] = { "Qcur-0", "Kcur-0", "Vcur-0", "kqv_out-0" };
|
|
for (int i = 0; i < 4; ++i) {
|
|
if (strcmp(t->name, names[i]) != 0) {
|
|
continue;
|
|
}
|
|
if (!ask) {
|
|
GGML_ASSERT(t->type == GGML_TYPE_F32 && ggml_is_contiguous(t));
|
|
auto & data = static_cast<std::vector<float> *>(user_data)[i];
|
|
data.resize(ggml_nelements(t));
|
|
ggml_backend_tensor_get(t, data.data(), 0, data.size()*sizeof(float));
|
|
}
|
|
return true;
|
|
}
|
|
return false;
|
|
};
|
|
llama_context_ptr lctx(llama_init_from_model(model, params));
|
|
GGML_ASSERT(lctx);
|
|
GGML_ASSERT(lctx->get_cparams().flash_attn == (flash == LLAMA_FLASH_ATTN_TYPE_ENABLED));
|
|
get_logits(model, lctx.get(), { 1, 2, 3, 4 });
|
|
llama_synchronize(lctx.get());
|
|
for (const auto & data : tensors) {
|
|
GGML_ASSERT(data.size() == (size_t) (head_size*n_head*n_tokens));
|
|
}
|
|
|
|
// Round the original K/V independently of the cache write graph.
|
|
for (int i : { 1, 2 }) {
|
|
const float scale = i == 1 ? sk : sv;
|
|
const ggml_type type = i == 1 ? params.type_k : params.type_v;
|
|
auto & data = tensors[i];
|
|
if (type == GGML_TYPE_F8_E4M3) {
|
|
for (float & value : data) {
|
|
value /= scale;
|
|
}
|
|
std::vector<uint8_t> quantized(data.size());
|
|
ggml_get_type_traits(type)->from_float_ref(data.data(), quantized.data(), data.size());
|
|
ggml_get_type_traits(type)->to_float(quantized.data(), data.data(), data.size());
|
|
for (float & value : data) {
|
|
value *= scale;
|
|
}
|
|
} else {
|
|
for (float & value : data) {
|
|
value = ggml_fp16_to_fp32(ggml_fp32_to_fp16(value));
|
|
}
|
|
}
|
|
}
|
|
|
|
std::vector<float> expected(tensors[3].size(), 0.0f);
|
|
for (int64_t token = 0; token < n_tokens; ++token) {
|
|
for (int64_t head = 0; head < n_head; ++head) {
|
|
const int64_t q_offset = (token*n_head + head)*head_size;
|
|
std::vector<double> scores(token + 1);
|
|
double max_score = -INFINITY;
|
|
for (int64_t past = 0; past <= token; ++past) {
|
|
const int64_t kv_offset = (past*n_head + head)*head_size;
|
|
double dot = 0.0;
|
|
for (int64_t d = 0; d < head_size; ++d) {
|
|
dot += (double) tensors[0][q_offset + d]*tensors[1][kv_offset + d];
|
|
}
|
|
scores[past] = dot/sqrt((double) head_size);
|
|
max_score = std::max(max_score, scores[past]);
|
|
}
|
|
double sum = 0.0;
|
|
for (double & score : scores) {
|
|
score = exp(score - max_score);
|
|
sum += score;
|
|
}
|
|
for (int64_t past = 0; past <= token; ++past) {
|
|
const int64_t kv_offset = (past*n_head + head)*head_size;
|
|
for (int64_t d = 0; d < head_size; ++d) {
|
|
expected[q_offset + d] += scores[past]/sum*tensors[2][kv_offset + d];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
const double error = nmse(expected, tensors[3]);
|
|
if (!std::isfinite(error) || error >= 1e-5) {
|
|
LOG_ERR("FP8 attention: flash=%s scaled=%d offload=%d nmse=%.2e\n", llama_flash_attn_type_name(flash), scaled, offload_kqv, error);
|
|
}
|
|
GGML_ASSERT(std::isfinite(error) && error < 1e-5);
|
|
}
|
|
}
|
|
model->layers[0].k_cache_s = old_k_scale;
|
|
model->layers[0].v_cache_s = old_v_scale;
|
|
}
|
|
|
|
static void check_fp8_context_shift(llama_model * model) {
|
|
ggml_context_ptr scale_ctx(ggml_init({ 4096, nullptr, true }));
|
|
ggml_tensor * scale = ggml_new_tensor_1d(scale_ctx.get(), GGML_TYPE_F32, 1);
|
|
ggml_backend_buffer_ptr scale_buf(ggml_backend_alloc_ctx_tensors_from_buft(scale_ctx.get(), ggml_backend_cpu_buffer_type()));
|
|
ggml_tensor * old_scale = model->layers[0].k_cache_s;
|
|
|
|
for (float scale_value : { 1.0f, 0.5f }) {
|
|
ggml_backend_tensor_set(scale, &scale_value, 0, sizeof(scale_value));
|
|
model->layers[0].k_cache_s = scale;
|
|
|
|
auto params = llama_context_default_params();
|
|
params.n_ctx = 256;
|
|
params.n_batch = 4;
|
|
params.n_ubatch = 4;
|
|
params.n_threads = 1;
|
|
params.n_threads_batch = 1;
|
|
params.type_k = GGML_TYPE_F8_E4M3;
|
|
params.type_v = GGML_TYPE_F8_E4M3;
|
|
params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED;
|
|
params.offload_kqv = false;
|
|
llama_context_ptr lctx(llama_init_from_model(model, params));
|
|
GGML_ASSERT(lctx);
|
|
get_logits(model, lctx.get(), { 1, 2, 3, 4 });
|
|
llama_synchronize(lctx.get());
|
|
|
|
auto * kv = dynamic_cast<llama_kv_cache *>(llama_get_memory(lctx.get()));
|
|
GGML_ASSERT(kv && kv->get_can_shift());
|
|
ggml_tensor * k = kv->get_k_storage(0);
|
|
std::vector<uint8_t> before(ggml_nbytes(k));
|
|
ggml_backend_tensor_get(k, before.data(), 0, before.size());
|
|
|
|
ggml_context_ptr ctx(ggml_init({ 2*1024*1024, nullptr, false }));
|
|
const auto & hp = model->hparams;
|
|
const auto & cp = lctx->get_cparams();
|
|
ggml_tensor * keys = ggml_new_tensor_3d(ctx.get(), GGML_TYPE_F32, hp.n_embd_head_k(), hp.n_head_kv(), kv->get_size());
|
|
ggml_get_type_traits(GGML_TYPE_F8_E4M3)->to_float(before.data(), (float *) keys->data, ggml_nelements(keys));
|
|
ggml_tensor * shifts = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I32, kv->get_size());
|
|
ggml_set_i32(shifts, 0);
|
|
for (int i = 1; i < 4; ++i) {
|
|
((int32_t *) shifts->data)[i] = -1;
|
|
}
|
|
ggml_tensor * expected_f32 = ggml_rope_ext(ctx.get(), keys, shifts, model->get_rope_factors(cp, 0), hp.n_rot(), hp.rope_type,
|
|
cp.n_ctx_orig_yarn, model->get_rope_freq_base(cp, 0), model->get_rope_freq_scale(cp, 0),
|
|
cp.yarn_ext_factor, cp.yarn_attn_factor, cp.yarn_beta_fast, cp.yarn_beta_slow);
|
|
ggml_cgraph * gf = ggml_new_graph(ctx.get());
|
|
ggml_build_forward_expand(gf, expected_f32);
|
|
GGML_ASSERT(ggml_graph_compute_with_ctx(ctx.get(), gf, 1) == GGML_STATUS_SUCCESS);
|
|
std::vector<uint8_t> expected(before.size());
|
|
ggml_quantize_chunk(GGML_TYPE_F8_E4M3, (float *) expected_f32->data, expected.data(), 0,
|
|
ggml_nelements(expected_f32)/keys->ne[0], keys->ne[0], nullptr);
|
|
|
|
llama_memory_seq_add(kv, 0, 1, 4, -1);
|
|
GGML_ASSERT(kv->get_has_shift());
|
|
GGML_ASSERT(lctx->memory_update(false));
|
|
llama_synchronize(lctx.get());
|
|
GGML_ASSERT(!kv->get_has_shift());
|
|
std::vector<uint8_t> after(before.size());
|
|
ggml_backend_tensor_get(k, after.data(), 0, after.size());
|
|
GGML_ASSERT(after != before);
|
|
GGML_ASSERT(after == expected);
|
|
lctx.reset();
|
|
}
|
|
|
|
model->layers[0].k_cache_s = old_scale;
|
|
}
|
|
|
|
enum class test_status {
|
|
PASS,
|
|
FAIL,
|
|
SKIP,
|
|
};
|
|
|
|
static test_status test_fp8_kv(const std::string & path, const char * target_backend) {
|
|
ggml_backend_dev_t cpu_devices[] = {nullptr};
|
|
auto params = llama_model_default_params();
|
|
params.devices = cpu_devices;
|
|
params.progress_callback = silent_model_load_progress;
|
|
{
|
|
llama_model_ptr model = load_model(path, params);
|
|
if (model->arch != LLM_ARCH_LLAMA || model->hparams.n_expert > 0) {
|
|
LOG("FP8 KV: only dense Llama fixtures are supported, skipping %s\n", path.c_str());
|
|
return test_status::SKIP;
|
|
}
|
|
LOG("FP8 KV: CPU attention and context shift\n");
|
|
check_fp8_attention(model.get());
|
|
check_fp8_context_shift(model.get());
|
|
}
|
|
|
|
std::vector<ggml_backend_dev_t> devices_meta;
|
|
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
|
|
auto * dev = ggml_backend_dev_get(i);
|
|
if (target_backend && strcmp(target_backend, ggml_backend_dev_name(dev)) != 0) {
|
|
continue;
|
|
}
|
|
if (ggml_backend_dev_buffer_type(dev) != ggml_backend_cpu_buffer_type()) {
|
|
devices_meta.push_back(dev);
|
|
}
|
|
if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {
|
|
continue;
|
|
}
|
|
ggml_backend_dev_t devices[] = {dev, nullptr};
|
|
params.devices = devices;
|
|
LOG("FP8 KV: device=%s\n", ggml_backend_dev_name(dev));
|
|
llama_model_ptr model = load_model(path, params);
|
|
check_fp8_attention(model.get());
|
|
check_fp8_attention(model.get(), false);
|
|
}
|
|
if (!target_backend && !devices_meta.empty()) {
|
|
devices_meta.push_back(nullptr);
|
|
params.devices = devices_meta.data();
|
|
params.split_mode = LLAMA_SPLIT_MODE_TENSOR;
|
|
LOG("FP8 KV: tensor split\n");
|
|
llama_model_ptr model = load_model(path, params);
|
|
check_fp8_attention(model.get());
|
|
check_fp8_attention(model.get(), false);
|
|
}
|
|
return test_status::PASS;
|
|
}
|
|
|
|
static void test_fp8(const std::string & model_path) {
|
|
const auto dir = std::filesystem::temp_directory_path() / ("llama-fp8-" + std::to_string(std::random_device{}()));
|
|
if (!std::filesystem::create_directory(dir)) {
|
|
throw std::runtime_error("failed to create FP8 test directory");
|
|
}
|
|
struct test_files {
|
|
std::filesystem::path dir;
|
|
~test_files() {
|
|
std::error_code ec;
|
|
std::filesystem::remove_all(dir, ec);
|
|
}
|
|
} files{dir};
|
|
|
|
const auto input = (files.dir / "fp8.gguf").string();
|
|
const auto reference = (files.dir / "reference.gguf").string();
|
|
const auto output = (files.dir / "output.gguf").string();
|
|
const auto reference_output = (files.dir / "reference-output.gguf").string();
|
|
|
|
// Use the same CPU kernels for scaled weights and the folded reference.
|
|
const llama_model_tensor_buft_override overrides[] = {
|
|
{".*", ggml_backend_cpu_buffer_type()},
|
|
{nullptr, nullptr},
|
|
};
|
|
ggml_backend_dev_t devices[] = {nullptr};
|
|
auto model_params = llama_model_default_params();
|
|
model_params.devices = devices;
|
|
model_params.tensor_buft_overrides = overrides;
|
|
model_params.progress_callback = silent_model_load_progress;
|
|
|
|
const int64_t n_embd = prepare_models(model_path, input, reference);
|
|
check_model_save(input, model_params);
|
|
check_model_save(reference, model_params);
|
|
check_fp8_scales(input, reference, model_params);
|
|
check_split_model(reference, files.dir, model_params);
|
|
check_split_model(input, files.dir, model_params);
|
|
check_fp8_requantize(input, reference, output, reference_output, n_embd, model_params);
|
|
}
|
|
|
|
static const char * test_status_str(test_status status) {
|
|
switch (status) {
|
|
case test_status::PASS: return "PASS";
|
|
case test_status::FAIL: return "FAIL";
|
|
case test_status::SKIP: return "SKIP";
|
|
}
|
|
GGML_ABORT("invalid test status");
|
|
}
|
|
|
|
int main(int argc, char ** argv) {
|
|
if (argc == 2 && (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0)) {
|
|
printf("Usage: %s (-m model.gguf | --models DIR) [--kv [-b backend]]\n", argv[0]);
|
|
return 0;
|
|
}
|
|
std::string model_file;
|
|
std::string models_dir;
|
|
const char * target_backend = nullptr;
|
|
bool kv = false;
|
|
bool valid = true;
|
|
for (int i = 1; i < argc; ++i) {
|
|
if ((strcmp(argv[i], "-m") == 0 || strcmp(argv[i], "--model") == 0 || strcmp(argv[i], "--models") == 0) && i + 1 < argc) {
|
|
if (!model_file.empty() || !models_dir.empty()) {
|
|
valid = false;
|
|
break;
|
|
}
|
|
const bool directory = strcmp(argv[i], "--models") == 0;
|
|
if (directory) {
|
|
models_dir = argv[++i];
|
|
} else {
|
|
model_file = argv[++i];
|
|
}
|
|
} else if (strcmp(argv[i], "--kv") == 0) {
|
|
kv = true;
|
|
} else if ((strcmp(argv[i], "-b") == 0 || strcmp(argv[i], "--backend") == 0) && i + 1 < argc) {
|
|
target_backend = argv[++i];
|
|
} else {
|
|
valid = false;
|
|
break;
|
|
}
|
|
}
|
|
if (!valid || (model_file.empty() && models_dir.empty()) || (target_backend && !kv)) {
|
|
fprintf(stderr, "Usage: %s (-m model.gguf | --models DIR) [--kv [-b backend]]\n", argv[0]);
|
|
return 1;
|
|
}
|
|
common_init();
|
|
llama_backend_init();
|
|
int result = 0;
|
|
try {
|
|
if (target_backend && !ggml_backend_dev_by_name(target_backend)) {
|
|
throw std::runtime_error("unknown backend device: " + std::string(target_backend));
|
|
}
|
|
std::vector<std::string> models;
|
|
if (!models_dir.empty()) {
|
|
if (!std::filesystem::is_directory(models_dir)) {
|
|
throw std::runtime_error("models directory does not exist: " + models_dir);
|
|
}
|
|
for (const auto & entry : std::filesystem::directory_iterator(models_dir)) {
|
|
if (entry.is_regular_file() && entry.path().extension() == ".gguf") {
|
|
models.push_back(entry.path().string());
|
|
}
|
|
}
|
|
std::sort(models.begin(), models.end());
|
|
if (models.empty()) {
|
|
throw std::runtime_error("no .gguf models found in: " + models_dir);
|
|
}
|
|
} else {
|
|
models.push_back(model_file);
|
|
}
|
|
|
|
std::vector<test_status> results;
|
|
for (const auto & model : models) {
|
|
try {
|
|
if (kv) {
|
|
results.push_back(test_fp8_kv(model, target_backend));
|
|
} else {
|
|
test_fp8(model);
|
|
results.push_back(test_status::PASS);
|
|
}
|
|
} catch (const std::exception & err) {
|
|
fprintf(stderr, "FP8 test failed for '%s': %s\n", model.c_str(), err.what());
|
|
results.push_back(test_status::FAIL);
|
|
result = 1;
|
|
}
|
|
}
|
|
printf("\n| Model | Result |\n|-------|--------|\n");
|
|
for (size_t i = 0; i < models.size(); ++i) {
|
|
printf("| %s | %s |\n", std::filesystem::path(models[i]).filename().string().c_str(), test_status_str(results[i]));
|
|
}
|
|
} catch (const std::exception & err) {
|
|
fprintf(stderr, "FP8 test failed: %s\n", err.what());
|
|
result = 1;
|
|
}
|
|
llama_backend_free();
|
|
return result;
|
|
}
|