Files
llama.cpp/tests/test-fp8.cpp
T

725 lines
33 KiB
C++

#include "common.h"
#include "log.h"
#include "ggml.h"
#include "ggml-backend.h"
#include "ggml-cpp.h"
#include "gguf.h"
#include "llama.h"
#include "llama-cpp.h"
#include "../src/llama-context.h"
#include "../src/llama-kv-cache.h"
#include "../src/llama-model.h"
#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <filesystem>
#include <memory>
#include <random>
#include <stdexcept>
#include <string>
#include <vector>
// normalized mean squared error = mse(a, b) / mse(a, 0)
static double nmse(const std::vector<float> & a, const std::vector<float> & b) {
GGML_ASSERT(a.size() == b.size());
double mse_a_b = 0.0;
double mse_a_0 = 0.0;
for (size_t i = 0; i < a.size(); i++) {
float a_i = a[i];
float b_i = b[i];
mse_a_b += (a_i - b_i) * (a_i - b_i);
mse_a_0 += a_i * a_i;
}
return mse_a_b / mse_a_0;
}
static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed) {
std::mt19937 gen(seed);
std::uniform_int_distribution<> dis(0, n_vocab - 1);
std::vector<llama_token> ret;
ret.reserve(n_tokens);
for (uint32_t i = 0; i < n_tokens; i++) {
ret.push_back(dis(gen));
}
return ret;
}
static std::vector<float> get_logits(
llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
const uint32_t n_ctx = llama_n_ctx(lctx);
const uint32_t n_tokens = tokens.size();
common_batch batch(lctx);
GGML_ASSERT(n_tokens <= n_ctx);
for (uint32_t pos = 0; pos < n_tokens; pos++) {
batch.add(tokens[pos], pos, 0, true);
}
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
throw std::runtime_error("failed to decode batch");
}
std::vector<float> ret;
ret.reserve(n_tokens*n_vocab);
for (uint32_t i = 0; i < n_tokens; i++) {
const float * logits_ith = llama_get_logits_ith(lctx, i);
for (uint32_t j = 0; j < n_vocab; j++) {
ret.push_back(logits_ith[j]);
}
}
return ret;
}
static const char * FP8_TEST_WEIGHTS[] = {
"output.weight", "blk.0.attn_q.weight",
"blk.0.ffn_gate_exps.weight", "blk.0.ffn_up_exps.weight", "blk.0.ffn_down_exps.weight",
};
static int64_t prepare_models(const std::string & path, const std::string & input, const std::string & reference) {
ggml_context * ctx_raw = nullptr;
gguf_context_ptr source(gguf_init_from_file(path.c_str(), {false, &ctx_raw}));
ggml_context_ptr source_ctx(ctx_raw);
if (!source || !source_ctx) {
throw std::runtime_error("failed to read FP8 test fixture");
}
ggml_tensor * attn_q = ggml_get_tensor(source_ctx.get(), "blk.0.attn_q.weight");
if (!attn_q || !ggml_get_tensor(source_ctx.get(), "output.weight")) {
throw std::runtime_error("FP8 test fixture must have attention Q and output weights");
}
gguf_context_ptr fp8(gguf_init_empty());
gguf_context_ptr folded(gguf_init_empty());
gguf_set_kv(fp8.get(), source.get());
gguf_set_kv(folded.get(), source.get());
for (int64_t i = 0; i < gguf_get_n_tensors(source.get()); ++i) {
ggml_tensor * tensor = ggml_get_tensor(source_ctx.get(), gguf_get_tensor_name(source.get(), i));
gguf_add_tensor(fp8.get(), tensor);
gguf_add_tensor(folded.get(), tensor);
}
ggml_context_ptr scale_ctx(ggml_init({16 * ggml_tensor_overhead(), nullptr, true}));
// GGUF keeps pointers into these buffers until the files are written.
std::vector<std::vector<uint8_t>> fp8_data;
std::vector<std::vector<ggml_fp16_t>> folded_data;
std::vector<std::vector<float>> scale_data;
for (const char * name : FP8_TEST_WEIGHTS) {
ggml_tensor * tensor = ggml_get_tensor(source_ctx.get(), name);
if (!tensor) {
continue;
}
std::vector<float> values(ggml_nelements(tensor));
if (tensor->type == GGML_TYPE_F32) {
memcpy(values.data(), tensor->data, ggml_nbytes(tensor));
} else {
const auto * traits = ggml_get_type_traits(tensor->type);
if (!traits->to_float) {
throw std::runtime_error("unsupported FP8 test fixture tensor type");
}
traits->to_float(tensor->data, values.data(), values.size());
}
const auto * traits = ggml_get_type_traits(GGML_TYPE_F8_E4M3);
fp8_data.emplace_back(values.size());
traits->from_float_ref(values.data(), fp8_data.back().data(), values.size());
gguf_set_tensor_type(fp8.get(), name, GGML_TYPE_F8_E4M3);
gguf_set_tensor_data(fp8.get(), name, fp8_data.back().data());
traits->to_float(fp8_data.back().data(), values.data(), values.size());
const size_t expert_size = tensor->ne[0] * tensor->ne[1];
for (size_t i = 0; i < values.size(); ++i) {
values[i] *= i < expert_size ? 2.0f : 4.0f;
}
folded_data.emplace_back(values.size());
ggml_fp32_to_fp16_row(values.data(), folded_data.back().data(), values.size());
gguf_set_tensor_type(folded.get(), name, GGML_TYPE_F16);
gguf_set_tensor_data(folded.get(), name, folded_data.back().data());
const std::string prefix(name, strlen(name) - strlen("weight"));
for (const char * suffix : {"scale", "input_scale"}) {
const std::string scale_name = prefix + suffix;
ggml_tensor * scale = ggml_get_tensor(source_ctx.get(), scale_name.c_str());
if (scale) {
if (scale->type != GGML_TYPE_F32 || ggml_nelements(scale) != tensor->ne[2]) {
throw std::runtime_error("unsupported FP8 test fixture scale shape or type");
}
scale_data.emplace_back(tensor->ne[2], 1.0f);
gguf_set_tensor_data(folded.get(), scale_name.c_str(), scale_data.back().data());
} else {
scale = ggml_new_tensor_1d(scale_ctx.get(), GGML_TYPE_F32, tensor->ne[2]);
ggml_set_name(scale, scale_name.c_str());
gguf_add_tensor(fp8.get(), scale);
}
// Scales above one avoid subnormal Q6_K block scales in the folded reference.
scale_data.emplace_back(tensor->ne[2], 4.0f);
if (strcmp(suffix, "scale") == 0) {
scale_data.back()[0] = 2.0f;
}
gguf_set_tensor_data(fp8.get(), scale_name.c_str(), scale_data.back().data());
}
}
if (!gguf_write_to_file(fp8.get(), input.c_str(), false) ||
!gguf_write_to_file(folded.get(), reference.c_str(), false)) {
throw std::runtime_error("failed to write FP8 test models");
}
return attn_q->ne[0];
}
static bool silent_model_load_progress(float /*progress*/, void * /*user_data*/) {
return true;
}
static llama_model_ptr load_model(const std::string & path, const llama_model_params & params, FILE * file = nullptr) {
llama_model_ptr model(file ? llama_model_load_from_file_ptr(file, params) :
llama_model_load_from_file(path.c_str(), params));
if (!model) {
throw std::runtime_error("failed to load FP8 test model");
}
return model;
}
static std::vector<float> load_logits(const std::string & path, const llama_model_params & model_params, bool split = false) {
std::unique_ptr<FILE, int (*)(FILE *)> file(split ? nullptr : fopen(path.c_str(), "rb"), fclose);
if (!split && !file) {
throw std::runtime_error("failed to open FP8 test model");
}
llama_model_ptr model = load_model(path, model_params, file.get());
auto ctx_params = llama_context_default_params();
ctx_params.n_ctx = 0;
ctx_params.n_ubatch = 64;
ctx_params.n_threads = ctx_params.n_threads_batch = 4;
llama_context_ptr lctx(llama_init_from_model(model.get(), ctx_params));
if (!lctx) {
throw std::runtime_error("failed to create FP8 test context");
}
const auto tokens = get_tokens(8, llama_vocab_n_tokens(llama_model_get_vocab(model.get())), 1);
return get_logits(model.get(), lctx.get(), tokens);
}
static void check_model_save(const std::string & path, const llama_model_params & model_params) {
const std::string saved = path + ".saved.gguf";
const auto logits = load_logits(path, model_params);
auto save_params = model_params;
// Packed CPU buffers do not support reading weights back for saving.
save_params.use_extra_bufts = false;
save_params.load_mode = LLAMA_LOAD_MODE_NONE;
{
llama_model_ptr model = load_model(path, save_params);
llama_model_save_to_file(model.get(), saved.c_str());
}
const double error = nmse(logits, load_logits(saved, model_params));
LOG("FP8 model save: %s nmse=%.2e\n", path.c_str(), error);
if (!(error < 1e-10)) {
throw std::runtime_error("saved FP8 test model differs from original model");
}
std::filesystem::rename(saved, path);
}
static void check_fp8_scales(const std::string & input, const std::string & reference, const llama_model_params & model_params) {
const auto reference_logits = load_logits(reference, model_params);
const double source_error = nmse(reference_logits, load_logits(input, model_params));
LOG("FP8 sidecar scales: nmse=%.2e\n", source_error);
if (!(source_error < 1e-5)) {
throw std::runtime_error("FP8 sidecar scales differ from folded weights");
}
}
static void check_split_model(const std::string & path, const std::filesystem::path & dir, const llama_model_params & model_params) {
ggml_context * ctx_raw = nullptr;
gguf_context_ptr metadata(gguf_init_from_file(path.c_str(), {false, &ctx_raw}));
ggml_context_ptr ctx(ctx_raw);
if (!metadata || !ctx) {
throw std::runtime_error("failed to read FP8 test model for splitting");
}
const auto split_first = (dir / "split-00001-of-00002.gguf").string();
const auto split_second = (dir / "split-00002-of-00002.gguf").string();
for (int split = 0; split < 2; ++split) {
gguf_context_ptr shard(gguf_init_empty());
gguf_set_kv(shard.get(), metadata.get());
gguf_set_val_u16(shard.get(), "split.no", split);
gguf_set_val_u16(shard.get(), "split.count", 2);
gguf_set_val_i32(shard.get(), "split.tensors.count", gguf_get_n_tensors(metadata.get()));
for (int64_t i = 0; i < gguf_get_n_tensors(metadata.get()); ++i) {
auto * tensor = ggml_get_tensor(ctx.get(), gguf_get_tensor_name(metadata.get(), i));
// Put output scales in the second shard to test lookup across splits.
const bool output_scale = strcmp(tensor->name, "output.scale") == 0 || strcmp(tensor->name, "output.input_scale") == 0;
if (output_scale == (split == 1)) {
gguf_add_tensor(shard.get(), tensor);
}
}
if (!gguf_write_to_file(shard.get(), (split == 0 ? split_first : split_second).c_str(), false)) {
throw std::runtime_error("failed to write split FP8 test model");
}
}
const double split_error = nmse(load_logits(path, model_params), load_logits(split_first, model_params, true));
LOG("FP8 split model: nmse=%.2e\n", split_error);
if (!(split_error < 1e-10)) {
throw std::runtime_error("split FP8 model differs from unsplit model");
}
}
static void check_fp8_requantize(
const std::string & input, const std::string & reference,
const std::string & output, const std::string & reference_output,
int64_t n_embd, const llama_model_params & model_params) {
for (llama_ftype ftype : {LLAMA_FTYPE_ALL_F32, LLAMA_FTYPE_MOSTLY_F16, LLAMA_FTYPE_MOSTLY_BF16, LLAMA_FTYPE_MOSTLY_Q4_K_M}) {
for (int nthread : {1, 3}) {
std::vector<float> importance(n_embd, 1.0f);
llama_model_imatrix_data imatrix[] = {
{"blk.0.attn_q.weight", importance.data(), importance.size()},
{nullptr, nullptr, 0},
};
auto params = llama_model_quantize_default_params();
params.ftype = ftype;
params.nthread = nthread;
if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M && nthread == 3) {
params.imatrix = imatrix;
}
if (llama_model_quantize(input.c_str(), output.c_str(), &params) != 0 ||
llama_model_quantize(reference.c_str(), reference_output.c_str(), &params) != 0) {
throw std::runtime_error("FP8 model requantization failed");
}
const double error = nmse(load_logits(reference_output, model_params), load_logits(output, model_params));
LOG("FP8 requantization: ftype=%d threads=%d nmse=%.2e\n", ftype, nthread, error);
if (!(error < 1e-4)) {
throw std::runtime_error("FP8 requantization differs from folded weights");
}
}
}
}
static void check_fp8_attention(llama_model * model, bool offload_kqv = true) {
// TODO: enable CPU KV cache tests with tensor split after the meta backend handles the attention transfers.
if (model->split_mode() == LLAMA_SPLIT_MODE_TENSOR && !offload_kqv) {
LOG("FP8 attention: tensor split with CPU KV cache is not supported, skipping\n");
return;
}
ggml_context_ptr scale_ctx(ggml_init({ 4096, nullptr, true }));
ggml_tensor * k_scale = ggml_new_tensor_1d(scale_ctx.get(), GGML_TYPE_F32, 1);
ggml_tensor * v_scale = ggml_new_tensor_1d(scale_ctx.get(), GGML_TYPE_F32, 1);
ggml_set_name(k_scale, "blk.0.attn_k.k_scale");
ggml_set_name(v_scale, "blk.0.attn_v.v_scale");
ggml_backend_buffer_ptr scale_buf(ggml_backend_alloc_ctx_tensors_from_buft(scale_ctx.get(), ggml_backend_dev_buffer_type(model->dev_layer(0))));
ggml_tensor * old_k_scale = model->layers[0].k_cache_s;
ggml_tensor * old_v_scale = model->layers[0].v_cache_s;
const int64_t head_size = model->hparams.n_embd_head_k();
const int64_t n_head = model->hparams.n_head();
const int64_t n_tokens = 4;
GGML_ASSERT(n_head == model->hparams.n_head_kv());
GGML_ASSERT(head_size == model->hparams.n_embd_head_v());
for (llama_flash_attn_type flash : { LLAMA_FLASH_ATTN_TYPE_DISABLED, LLAMA_FLASH_ATTN_TYPE_ENABLED }) {
if (model->split_mode() == LLAMA_SPLIT_MODE_TENSOR && flash == LLAMA_FLASH_ATTN_TYPE_DISABLED) {
LOG("FP8 attention: tensor split requires flash attention, skipping disabled cases\n");
continue;
}
for (int scaled : { 1, 2, 3 }) {
const float sk = scaled & 1 ? 0.03125f : 1.0f;
const float sv = scaled & 2 ? 0.125f : 1.0f;
ggml_backend_tensor_set(k_scale, &sk, 0, sizeof(sk));
ggml_backend_tensor_set(v_scale, &sv, 0, sizeof(sv));
model->layers[0].k_cache_s = scaled & 1 ? k_scale : nullptr;
model->layers[0].v_cache_s = scaled & 2 ? v_scale : nullptr;
std::vector<float> tensors[4];
auto params = llama_context_default_params();
params.n_ctx = 256;
params.n_batch = n_tokens;
params.n_ubatch = n_tokens;
params.n_threads = 1;
params.n_threads_batch = 1;
params.type_k = scaled & 1 ? GGML_TYPE_F8_E4M3 : GGML_TYPE_F16;
params.type_v = scaled & 2 ? GGML_TYPE_F8_E4M3 : GGML_TYPE_F16;
params.flash_attn_type = flash;
params.offload_kqv = offload_kqv;
if (model->split_mode() == LLAMA_SPLIT_MODE_TENSOR) {
ggml_context_ptr ctx(ggml_init({ 8*ggml_tensor_overhead(), nullptr, true }));
ggml_tensor * q = ggml_new_tensor_4d(ctx.get(), GGML_TYPE_F32, head_size, n_tokens, n_head, 1);
ggml_tensor * k = ggml_new_tensor_4d(ctx.get(), params.type_k, head_size, params.n_ctx, n_head, 1);
ggml_tensor * v = ggml_new_tensor_4d(ctx.get(), params.type_v, head_size, params.n_ctx, n_head, 1);
ggml_tensor * mask = ggml_new_tensor_4d(ctx.get(), GGML_TYPE_F16, params.n_ctx, params.n_ctx, 1, 1);
ggml_tensor * attn = ggml_flash_attn_ext(ctx.get(), q, k, v, mask, 1.0f/sqrtf(head_size), 0.0f, 0.0f);
auto * dev = model->dev_layer(0);
if (!ggml_backend_dev_supports_op(dev, attn)) {
LOG("FP8 attention: scaled=%d offload=%d: tensor-split flash attention not supported on %s, skipping\n",
scaled, offload_kqv, ggml_backend_dev_name(dev));
continue;
}
}
if (offload_kqv) {
// Cache writes cannot fall back from a preallocated device buffer.
llama_kv_cache probe(*model, false, model->hparams, params.type_k, params.type_v,
flash == LLAMA_FLASH_ATTN_TYPE_DISABLED, false, true, params.n_ctx, 1,
0, LLAMA_SWA_TYPE_NONE, nullptr, nullptr, nullptr, nullptr);
ggml_context_ptr ctx(ggml_init({ 64*ggml_tensor_overhead(), nullptr, true }));
llama_kv_cache::slot_info slots;
bool supported = true;
for (uint32_t il = 0; il < model->hparams.n_layer(); ++il) {
ggml_tensor * k = ggml_new_tensor_3d(ctx.get(), GGML_TYPE_F32, head_size, n_head, n_tokens);
ggml_tensor * v = ggml_new_tensor_3d(ctx.get(), GGML_TYPE_F32, head_size, n_head, n_tokens);
ggml_tensor * k_idxs = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I64, n_tokens);
ggml_tensor * v_idxs = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I64,
flash == LLAMA_FLASH_ATTN_TYPE_DISABLED ? n_tokens*head_size*n_head : n_tokens);
auto * dev = model->dev_layer(il);
if (!ggml_backend_dev_supports_op(dev, probe.cpy_k(ctx.get(), k, k_idxs, il, slots)) ||
!ggml_backend_dev_supports_op(dev, probe.cpy_v(ctx.get(), v, v_idxs, il, slots))) {
LOG("FP8 attention: flash=%s scaled=%d offload=1: cache writes not supported on %s, skipping\n",
llama_flash_attn_type_name(flash), scaled, ggml_backend_dev_name(dev));
supported = false;
break;
}
}
if (!supported) {
continue;
}
}
params.cb_eval_user_data = tensors;
params.cb_eval = [](ggml_tensor * t, bool ask, void * user_data) {
const char * names[] = { "Qcur-0", "Kcur-0", "Vcur-0", "kqv_out-0" };
for (int i = 0; i < 4; ++i) {
if (strcmp(t->name, names[i]) != 0) {
continue;
}
if (!ask) {
GGML_ASSERT(t->type == GGML_TYPE_F32 && ggml_is_contiguous(t));
auto & data = static_cast<std::vector<float> *>(user_data)[i];
data.resize(ggml_nelements(t));
ggml_backend_tensor_get(t, data.data(), 0, data.size()*sizeof(float));
}
return true;
}
return false;
};
llama_context_ptr lctx(llama_init_from_model(model, params));
GGML_ASSERT(lctx);
GGML_ASSERT(lctx->get_cparams().flash_attn == (flash == LLAMA_FLASH_ATTN_TYPE_ENABLED));
get_logits(model, lctx.get(), { 1, 2, 3, 4 });
llama_synchronize(lctx.get());
for (const auto & data : tensors) {
GGML_ASSERT(data.size() == (size_t) (head_size*n_head*n_tokens));
}
// Round the original K/V independently of the cache write graph.
for (int i : { 1, 2 }) {
const float scale = i == 1 ? sk : sv;
const ggml_type type = i == 1 ? params.type_k : params.type_v;
auto & data = tensors[i];
if (type == GGML_TYPE_F8_E4M3) {
for (float & value : data) {
value /= scale;
}
std::vector<uint8_t> quantized(data.size());
ggml_get_type_traits(type)->from_float_ref(data.data(), quantized.data(), data.size());
ggml_get_type_traits(type)->to_float(quantized.data(), data.data(), data.size());
for (float & value : data) {
value *= scale;
}
} else {
for (float & value : data) {
value = ggml_fp16_to_fp32(ggml_fp32_to_fp16(value));
}
}
}
std::vector<float> expected(tensors[3].size(), 0.0f);
for (int64_t token = 0; token < n_tokens; ++token) {
for (int64_t head = 0; head < n_head; ++head) {
const int64_t q_offset = (token*n_head + head)*head_size;
std::vector<double> scores(token + 1);
double max_score = -INFINITY;
for (int64_t past = 0; past <= token; ++past) {
const int64_t kv_offset = (past*n_head + head)*head_size;
double dot = 0.0;
for (int64_t d = 0; d < head_size; ++d) {
dot += (double) tensors[0][q_offset + d]*tensors[1][kv_offset + d];
}
scores[past] = dot/sqrt((double) head_size);
max_score = std::max(max_score, scores[past]);
}
double sum = 0.0;
for (double & score : scores) {
score = exp(score - max_score);
sum += score;
}
for (int64_t past = 0; past <= token; ++past) {
const int64_t kv_offset = (past*n_head + head)*head_size;
for (int64_t d = 0; d < head_size; ++d) {
expected[q_offset + d] += scores[past]/sum*tensors[2][kv_offset + d];
}
}
}
}
const double error = nmse(expected, tensors[3]);
if (!std::isfinite(error) || error >= 1e-5) {
LOG_ERR("FP8 attention: flash=%s scaled=%d offload=%d nmse=%.2e\n", llama_flash_attn_type_name(flash), scaled, offload_kqv, error);
}
GGML_ASSERT(std::isfinite(error) && error < 1e-5);
}
}
model->layers[0].k_cache_s = old_k_scale;
model->layers[0].v_cache_s = old_v_scale;
}
static void check_fp8_context_shift(llama_model * model) {
ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr));
GGML_ASSERT(backend);
auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend.get()));
auto set_n_threads = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
GGML_ASSERT(set_n_threads);
set_n_threads(backend.get(), 1);
ggml_context_ptr scale_ctx(ggml_init({ 4096, nullptr, true }));
ggml_tensor * scale = ggml_new_tensor_1d(scale_ctx.get(), GGML_TYPE_F32, 1);
ggml_backend_buffer_ptr scale_buf(ggml_backend_alloc_ctx_tensors_from_buft(scale_ctx.get(), ggml_backend_cpu_buffer_type()));
ggml_tensor * old_scale = model->layers[0].k_cache_s;
for (float scale_value : { 1.0f, 0.5f }) {
ggml_backend_tensor_set(scale, &scale_value, 0, sizeof(scale_value));
model->layers[0].k_cache_s = scale;
auto params = llama_context_default_params();
params.n_ctx = 256;
params.n_batch = 4;
params.n_ubatch = 4;
params.n_threads = 1;
params.n_threads_batch = 1;
params.type_k = GGML_TYPE_F8_E4M3;
params.type_v = GGML_TYPE_F8_E4M3;
params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED;
params.offload_kqv = false;
llama_context_ptr lctx(llama_init_from_model(model, params));
GGML_ASSERT(lctx);
get_logits(model, lctx.get(), { 1, 2, 3, 4 });
llama_synchronize(lctx.get());
auto * kv = dynamic_cast<llama_kv_cache *>(llama_get_memory(lctx.get()));
GGML_ASSERT(kv && kv->get_can_shift());
ggml_tensor * k = kv->get_k_storage(0);
std::vector<uint8_t> before(ggml_nbytes(k));
ggml_backend_tensor_get(k, before.data(), 0, before.size());
ggml_context_ptr ctx(ggml_init({ 2*1024*1024, nullptr, false }));
const auto & hp = model->hparams;
const auto & cp = lctx->get_cparams();
ggml_tensor * keys = ggml_new_tensor_3d(ctx.get(), GGML_TYPE_F32, hp.n_embd_head_k(), hp.n_head_kv(), kv->get_size());
ggml_get_type_traits(GGML_TYPE_F8_E4M3)->to_float(before.data(), (float *) keys->data, ggml_nelements(keys));
ggml_tensor * shifts = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I32, kv->get_size());
std::fill_n(static_cast<int32_t *>(shifts->data), ggml_nelements(shifts), 0);
for (int i = 1; i < 4; ++i) {
((int32_t *) shifts->data)[i] = -1;
}
ggml_tensor * expected_f32 = ggml_rope_ext(ctx.get(), keys, shifts, model->get_rope_factors(cp, 0), hp.n_rot(), hp.rope_type,
cp.n_ctx_orig_yarn, model->get_rope_freq_base(cp, 0), model->get_rope_freq_scale(cp, 0),
cp.yarn_ext_factor, cp.yarn_attn_factor, cp.yarn_beta_fast, cp.yarn_beta_slow);
ggml_cgraph * gf = ggml_new_graph(ctx.get());
ggml_build_forward_expand(gf, expected_f32);
GGML_ASSERT(ggml_backend_graph_compute(backend.get(), gf) == GGML_STATUS_SUCCESS);
std::vector<uint8_t> expected(before.size());
ggml_quantize_chunk(GGML_TYPE_F8_E4M3, (float *) expected_f32->data, expected.data(), 0,
ggml_nelements(expected_f32)/keys->ne[0], keys->ne[0], nullptr);
llama_memory_seq_add(kv, 0, 1, 4, -1);
GGML_ASSERT(kv->get_has_shift());
GGML_ASSERT(lctx->memory_update(false));
llama_synchronize(lctx.get());
GGML_ASSERT(!kv->get_has_shift());
std::vector<uint8_t> after(before.size());
ggml_backend_tensor_get(k, after.data(), 0, after.size());
GGML_ASSERT(after != before);
GGML_ASSERT(after == expected);
lctx.reset();
}
model->layers[0].k_cache_s = old_scale;
}
enum class test_status {
PASS,
FAIL,
SKIP,
};
static test_status test_fp8_kv(const std::string & path, const char * target_backend) {
ggml_backend_dev_t cpu_devices[] = {nullptr};
auto params = llama_model_default_params();
params.devices = cpu_devices;
params.progress_callback = silent_model_load_progress;
{
llama_model_ptr model = load_model(path, params);
if (model->arch != LLM_ARCH_LLAMA || model->hparams.n_expert > 0) {
LOG("FP8 KV: only dense Llama fixtures are supported, skipping %s\n", path.c_str());
return test_status::SKIP;
}
LOG("FP8 KV: CPU attention and context shift\n");
check_fp8_attention(model.get());
check_fp8_context_shift(model.get());
}
std::vector<ggml_backend_dev_t> devices_meta;
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
auto * dev = ggml_backend_dev_get(i);
if (target_backend && strcmp(target_backend, ggml_backend_dev_name(dev)) != 0) {
continue;
}
if (ggml_backend_dev_buffer_type(dev) != ggml_backend_cpu_buffer_type()) {
devices_meta.push_back(dev);
}
if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {
continue;
}
ggml_backend_dev_t devices[] = {dev, nullptr};
params.devices = devices;
LOG("FP8 KV: device=%s\n", ggml_backend_dev_name(dev));
llama_model_ptr model = load_model(path, params);
check_fp8_attention(model.get());
check_fp8_attention(model.get(), false);
}
if (!target_backend && !devices_meta.empty()) {
devices_meta.push_back(nullptr);
params.devices = devices_meta.data();
params.split_mode = LLAMA_SPLIT_MODE_TENSOR;
LOG("FP8 KV: tensor split\n");
llama_model_ptr model = load_model(path, params);
check_fp8_attention(model.get());
check_fp8_attention(model.get(), false);
}
return test_status::PASS;
}
static void test_fp8(const std::string & model_path) {
const auto dir = std::filesystem::temp_directory_path() / ("llama-fp8-" + std::to_string(std::random_device{}()));
if (!std::filesystem::create_directory(dir)) {
throw std::runtime_error("failed to create FP8 test directory");
}
struct test_files {
std::filesystem::path dir;
~test_files() {
std::error_code ec;
std::filesystem::remove_all(dir, ec);
}
} files{dir};
const auto input = (files.dir / "fp8.gguf").string();
const auto reference = (files.dir / "reference.gguf").string();
const auto output = (files.dir / "output.gguf").string();
const auto reference_output = (files.dir / "reference-output.gguf").string();
// Use the same CPU kernels for scaled weights and the folded reference.
const llama_model_tensor_buft_override overrides[] = {
{".*", ggml_backend_cpu_buffer_type()},
{nullptr, nullptr},
};
ggml_backend_dev_t devices[] = {nullptr};
auto model_params = llama_model_default_params();
model_params.devices = devices;
model_params.tensor_buft_overrides = overrides;
model_params.progress_callback = silent_model_load_progress;
const int64_t n_embd = prepare_models(model_path, input, reference);
check_model_save(input, model_params);
check_model_save(reference, model_params);
check_fp8_scales(input, reference, model_params);
check_split_model(reference, files.dir, model_params);
check_split_model(input, files.dir, model_params);
check_fp8_requantize(input, reference, output, reference_output, n_embd, model_params);
}
static const char * test_status_str(test_status status) {
switch (status) {
case test_status::PASS: return "PASS";
case test_status::FAIL: return "FAIL";
case test_status::SKIP: return "SKIP";
}
GGML_ABORT("invalid test status");
}
int main(int argc, char ** argv) {
if (argc == 2 && (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0)) {
printf("Usage: %s (-m model.gguf | --models DIR) [--kv [-b backend]]\n", argv[0]);
return 0;
}
std::string model_file;
std::string models_dir;
const char * target_backend = nullptr;
bool kv = false;
bool valid = true;
for (int i = 1; i < argc; ++i) {
if ((strcmp(argv[i], "-m") == 0 || strcmp(argv[i], "--model") == 0 || strcmp(argv[i], "--models") == 0) && i + 1 < argc) {
if (!model_file.empty() || !models_dir.empty()) {
valid = false;
break;
}
const bool directory = strcmp(argv[i], "--models") == 0;
if (directory) {
models_dir = argv[++i];
} else {
model_file = argv[++i];
}
} else if (strcmp(argv[i], "--kv") == 0) {
kv = true;
} else if ((strcmp(argv[i], "-b") == 0 || strcmp(argv[i], "--backend") == 0) && i + 1 < argc) {
target_backend = argv[++i];
} else {
valid = false;
break;
}
}
if (!valid || (model_file.empty() && models_dir.empty()) || (target_backend && !kv)) {
fprintf(stderr, "Usage: %s (-m model.gguf | --models DIR) [--kv [-b backend]]\n", argv[0]);
return 1;
}
common_init();
llama_backend_init();
int result = 0;
try {
if (target_backend && !ggml_backend_dev_by_name(target_backend)) {
throw std::runtime_error("unknown backend device: " + std::string(target_backend));
}
std::vector<std::string> models;
if (!models_dir.empty()) {
if (!std::filesystem::is_directory(models_dir)) {
throw std::runtime_error("models directory does not exist: " + models_dir);
}
for (const auto & entry : std::filesystem::directory_iterator(models_dir)) {
if (entry.is_regular_file() && entry.path().extension() == ".gguf") {
models.push_back(entry.path().string());
}
}
std::sort(models.begin(), models.end());
if (models.empty()) {
throw std::runtime_error("no .gguf models found in: " + models_dir);
}
} else {
models.push_back(model_file);
}
std::vector<test_status> results;
for (const auto & model : models) {
try {
if (kv) {
results.push_back(test_fp8_kv(model, target_backend));
} else {
test_fp8(model);
results.push_back(test_status::PASS);
}
} catch (const std::exception & err) {
fprintf(stderr, "FP8 test failed for '%s': %s\n", model.c_str(), err.what());
results.push_back(test_status::FAIL);
result = 1;
}
}
printf("\n| Model | Result |\n|-------|--------|\n");
for (size_t i = 0; i < models.size(); ++i) {
printf("| %s | %s |\n", std::filesystem::path(models[i]).filename().string().c_str(), test_status_str(results[i]));
}
} catch (const std::exception & err) {
fprintf(stderr, "FP8 test failed: %s\n", err.what());
result = 1;
}
llama_backend_free();
return result;
}