#include "common.h" #include "log.h" #include "ggml.h" #include "ggml-backend.h" #include "ggml-cpp.h" #include "gguf.h" #include "llama.h" #include "llama-cpp.h" #include "../src/llama-context.h" #include "../src/llama-kv-cache.h" #include "../src/llama-model.h" #include #include #include #include #include #include #include #include #include #include // normalized mean squared error = mse(a, b) / mse(a, 0) static double nmse(const std::vector & a, const std::vector & b) { GGML_ASSERT(a.size() == b.size()); double mse_a_b = 0.0; double mse_a_0 = 0.0; for (size_t i = 0; i < a.size(); i++) { float a_i = a[i]; float b_i = b[i]; mse_a_b += (a_i - b_i) * (a_i - b_i); mse_a_0 += a_i * a_i; } return mse_a_b / mse_a_0; } static std::vector get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed) { std::mt19937 gen(seed); std::uniform_int_distribution<> dis(0, n_vocab - 1); std::vector ret; ret.reserve(n_tokens); for (uint32_t i = 0; i < n_tokens; i++) { ret.push_back(dis(gen)); } return ret; } static std::vector get_logits( llama_model * model, llama_context * lctx, const std::vector & tokens) { const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); const uint32_t n_ctx = llama_n_ctx(lctx); const uint32_t n_tokens = tokens.size(); common_batch batch(lctx); GGML_ASSERT(n_tokens <= n_ctx); for (uint32_t pos = 0; pos < n_tokens; pos++) { batch.add(tokens[pos], pos, 0, true); } if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) { throw std::runtime_error("failed to decode batch"); } std::vector ret; ret.reserve(n_tokens*n_vocab); for (uint32_t i = 0; i < n_tokens; i++) { const float * logits_ith = llama_get_logits_ith(lctx, i); for (uint32_t j = 0; j < n_vocab; j++) { ret.push_back(logits_ith[j]); } } return ret; } static const char * FP8_TEST_WEIGHTS[] = { "output.weight", "blk.0.attn_q.weight", "blk.0.ffn_gate_exps.weight", "blk.0.ffn_up_exps.weight", "blk.0.ffn_down_exps.weight", }; static int64_t prepare_models(const std::string & path, const std::string & input, const std::string & reference) { ggml_context * ctx_raw = nullptr; gguf_context_ptr source(gguf_init_from_file(path.c_str(), {false, &ctx_raw})); ggml_context_ptr source_ctx(ctx_raw); if (!source || !source_ctx) { throw std::runtime_error("failed to read FP8 test fixture"); } ggml_tensor * attn_q = ggml_get_tensor(source_ctx.get(), "blk.0.attn_q.weight"); if (!attn_q || !ggml_get_tensor(source_ctx.get(), "output.weight")) { throw std::runtime_error("FP8 test fixture must have attention Q and output weights"); } gguf_context_ptr fp8(gguf_init_empty()); gguf_context_ptr folded(gguf_init_empty()); gguf_set_kv(fp8.get(), source.get()); gguf_set_kv(folded.get(), source.get()); for (int64_t i = 0; i < gguf_get_n_tensors(source.get()); ++i) { ggml_tensor * tensor = ggml_get_tensor(source_ctx.get(), gguf_get_tensor_name(source.get(), i)); gguf_add_tensor(fp8.get(), tensor); gguf_add_tensor(folded.get(), tensor); } ggml_context_ptr scale_ctx(ggml_init({16 * ggml_tensor_overhead(), nullptr, true})); // GGUF keeps pointers into these buffers until the files are written. std::vector> fp8_data; std::vector> folded_data; std::vector> scale_data; for (const char * name : FP8_TEST_WEIGHTS) { ggml_tensor * tensor = ggml_get_tensor(source_ctx.get(), name); if (!tensor) { continue; } std::vector values(ggml_nelements(tensor)); if (tensor->type == GGML_TYPE_F32) { memcpy(values.data(), tensor->data, ggml_nbytes(tensor)); } else { const auto * traits = ggml_get_type_traits(tensor->type); if (!traits->to_float) { throw std::runtime_error("unsupported FP8 test fixture tensor type"); } traits->to_float(tensor->data, values.data(), values.size()); } const auto * traits = ggml_get_type_traits(GGML_TYPE_F8_E4M3); fp8_data.emplace_back(values.size()); traits->from_float_ref(values.data(), fp8_data.back().data(), values.size()); gguf_set_tensor_type(fp8.get(), name, GGML_TYPE_F8_E4M3); gguf_set_tensor_data(fp8.get(), name, fp8_data.back().data()); traits->to_float(fp8_data.back().data(), values.data(), values.size()); const size_t expert_size = tensor->ne[0] * tensor->ne[1]; for (size_t i = 0; i < values.size(); ++i) { values[i] *= i < expert_size ? 2.0f : 4.0f; } folded_data.emplace_back(values.size()); ggml_fp32_to_fp16_row(values.data(), folded_data.back().data(), values.size()); gguf_set_tensor_type(folded.get(), name, GGML_TYPE_F16); gguf_set_tensor_data(folded.get(), name, folded_data.back().data()); const std::string prefix(name, strlen(name) - strlen("weight")); for (const char * suffix : {"scale", "input_scale"}) { const std::string scale_name = prefix + suffix; ggml_tensor * scale = ggml_get_tensor(source_ctx.get(), scale_name.c_str()); if (scale) { if (scale->type != GGML_TYPE_F32 || ggml_nelements(scale) != tensor->ne[2]) { throw std::runtime_error("unsupported FP8 test fixture scale shape or type"); } scale_data.emplace_back(tensor->ne[2], 1.0f); gguf_set_tensor_data(folded.get(), scale_name.c_str(), scale_data.back().data()); } else { scale = ggml_new_tensor_1d(scale_ctx.get(), GGML_TYPE_F32, tensor->ne[2]); ggml_set_name(scale, scale_name.c_str()); gguf_add_tensor(fp8.get(), scale); } // Scales above one avoid subnormal Q6_K block scales in the folded reference. scale_data.emplace_back(tensor->ne[2], 4.0f); if (strcmp(suffix, "scale") == 0) { scale_data.back()[0] = 2.0f; } gguf_set_tensor_data(fp8.get(), scale_name.c_str(), scale_data.back().data()); } } if (!gguf_write_to_file(fp8.get(), input.c_str(), false) || !gguf_write_to_file(folded.get(), reference.c_str(), false)) { throw std::runtime_error("failed to write FP8 test models"); } return attn_q->ne[0]; } static bool silent_model_load_progress(float /*progress*/, void * /*user_data*/) { return true; } static llama_model_ptr load_model(const std::string & path, const llama_model_params & params, FILE * file = nullptr) { llama_model_ptr model(file ? llama_model_load_from_file_ptr(file, params) : llama_model_load_from_file(path.c_str(), params)); if (!model) { throw std::runtime_error("failed to load FP8 test model"); } return model; } static std::vector load_logits(const std::string & path, const llama_model_params & model_params, bool split = false) { std::unique_ptr file(split ? nullptr : fopen(path.c_str(), "rb"), fclose); if (!split && !file) { throw std::runtime_error("failed to open FP8 test model"); } llama_model_ptr model = load_model(path, model_params, file.get()); auto ctx_params = llama_context_default_params(); ctx_params.n_ctx = 0; ctx_params.n_ubatch = 64; ctx_params.n_threads = ctx_params.n_threads_batch = 4; llama_context_ptr lctx(llama_init_from_model(model.get(), ctx_params)); if (!lctx) { throw std::runtime_error("failed to create FP8 test context"); } const auto tokens = get_tokens(8, llama_vocab_n_tokens(llama_model_get_vocab(model.get())), 1); return get_logits(model.get(), lctx.get(), tokens); } static void check_model_save(const std::string & path, const llama_model_params & model_params) { const std::string saved = path + ".saved.gguf"; const auto logits = load_logits(path, model_params); auto save_params = model_params; // Packed CPU buffers do not support reading weights back for saving. save_params.use_extra_bufts = false; save_params.load_mode = LLAMA_LOAD_MODE_NONE; { llama_model_ptr model = load_model(path, save_params); llama_model_save_to_file(model.get(), saved.c_str()); } const double error = nmse(logits, load_logits(saved, model_params)); LOG("FP8 model save: %s nmse=%.2e\n", path.c_str(), error); if (!(error < 1e-10)) { throw std::runtime_error("saved FP8 test model differs from original model"); } std::filesystem::rename(saved, path); } static void check_fp8_scales(const std::string & input, const std::string & reference, const llama_model_params & model_params) { const auto reference_logits = load_logits(reference, model_params); const double source_error = nmse(reference_logits, load_logits(input, model_params)); LOG("FP8 sidecar scales: nmse=%.2e\n", source_error); if (!(source_error < 1e-5)) { throw std::runtime_error("FP8 sidecar scales differ from folded weights"); } } static void check_split_model(const std::string & path, const std::filesystem::path & dir, const llama_model_params & model_params) { ggml_context * ctx_raw = nullptr; gguf_context_ptr metadata(gguf_init_from_file(path.c_str(), {false, &ctx_raw})); ggml_context_ptr ctx(ctx_raw); if (!metadata || !ctx) { throw std::runtime_error("failed to read FP8 test model for splitting"); } const auto split_first = (dir / "split-00001-of-00002.gguf").string(); const auto split_second = (dir / "split-00002-of-00002.gguf").string(); for (int split = 0; split < 2; ++split) { gguf_context_ptr shard(gguf_init_empty()); gguf_set_kv(shard.get(), metadata.get()); gguf_set_val_u16(shard.get(), "split.no", split); gguf_set_val_u16(shard.get(), "split.count", 2); gguf_set_val_i32(shard.get(), "split.tensors.count", gguf_get_n_tensors(metadata.get())); for (int64_t i = 0; i < gguf_get_n_tensors(metadata.get()); ++i) { auto * tensor = ggml_get_tensor(ctx.get(), gguf_get_tensor_name(metadata.get(), i)); // Put output scales in the second shard to test lookup across splits. const bool output_scale = strcmp(tensor->name, "output.scale") == 0 || strcmp(tensor->name, "output.input_scale") == 0; if (output_scale == (split == 1)) { gguf_add_tensor(shard.get(), tensor); } } if (!gguf_write_to_file(shard.get(), (split == 0 ? split_first : split_second).c_str(), false)) { throw std::runtime_error("failed to write split FP8 test model"); } } const double split_error = nmse(load_logits(path, model_params), load_logits(split_first, model_params, true)); LOG("FP8 split model: nmse=%.2e\n", split_error); if (!(split_error < 1e-10)) { throw std::runtime_error("split FP8 model differs from unsplit model"); } } static void check_fp8_requantize( const std::string & input, const std::string & reference, const std::string & output, const std::string & reference_output, int64_t n_embd, const llama_model_params & model_params) { for (llama_ftype ftype : {LLAMA_FTYPE_ALL_F32, LLAMA_FTYPE_MOSTLY_F16, LLAMA_FTYPE_MOSTLY_BF16, LLAMA_FTYPE_MOSTLY_Q4_K_M}) { for (int nthread : {1, 3}) { std::vector importance(n_embd, 1.0f); llama_model_imatrix_data imatrix[] = { {"blk.0.attn_q.weight", importance.data(), importance.size()}, {nullptr, nullptr, 0}, }; auto params = llama_model_quantize_default_params(); params.ftype = ftype; params.nthread = nthread; if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M && nthread == 3) { params.imatrix = imatrix; } if (llama_model_quantize(input.c_str(), output.c_str(), ¶ms) != 0 || llama_model_quantize(reference.c_str(), reference_output.c_str(), ¶ms) != 0) { throw std::runtime_error("FP8 model requantization failed"); } const double error = nmse(load_logits(reference_output, model_params), load_logits(output, model_params)); LOG("FP8 requantization: ftype=%d threads=%d nmse=%.2e\n", ftype, nthread, error); if (!(error < 1e-4)) { throw std::runtime_error("FP8 requantization differs from folded weights"); } } } } static void check_fp8_attention(llama_model * model, bool offload_kqv = true) { // TODO: enable CPU KV cache tests with tensor split after the meta backend handles the attention transfers. if (model->split_mode() == LLAMA_SPLIT_MODE_TENSOR && !offload_kqv) { LOG("FP8 attention: tensor split with CPU KV cache is not supported, skipping\n"); return; } ggml_context_ptr scale_ctx(ggml_init({ 4096, nullptr, true })); ggml_tensor * k_scale = ggml_new_tensor_1d(scale_ctx.get(), GGML_TYPE_F32, 1); ggml_tensor * v_scale = ggml_new_tensor_1d(scale_ctx.get(), GGML_TYPE_F32, 1); ggml_set_name(k_scale, "blk.0.attn_k.k_scale"); ggml_set_name(v_scale, "blk.0.attn_v.v_scale"); ggml_backend_buffer_ptr scale_buf(ggml_backend_alloc_ctx_tensors_from_buft(scale_ctx.get(), ggml_backend_dev_buffer_type(model->dev_layer(0)))); ggml_tensor * old_k_scale = model->layers[0].k_cache_s; ggml_tensor * old_v_scale = model->layers[0].v_cache_s; const int64_t head_size = model->hparams.n_embd_head_k(); const int64_t n_head = model->hparams.n_head(); const int64_t n_tokens = 4; GGML_ASSERT(n_head == model->hparams.n_head_kv()); GGML_ASSERT(head_size == model->hparams.n_embd_head_v()); for (llama_flash_attn_type flash : { LLAMA_FLASH_ATTN_TYPE_DISABLED, LLAMA_FLASH_ATTN_TYPE_ENABLED }) { if (model->split_mode() == LLAMA_SPLIT_MODE_TENSOR && flash == LLAMA_FLASH_ATTN_TYPE_DISABLED) { LOG("FP8 attention: tensor split requires flash attention, skipping disabled cases\n"); continue; } for (int scaled : { 1, 2, 3 }) { const float sk = scaled & 1 ? 0.03125f : 1.0f; const float sv = scaled & 2 ? 0.125f : 1.0f; ggml_backend_tensor_set(k_scale, &sk, 0, sizeof(sk)); ggml_backend_tensor_set(v_scale, &sv, 0, sizeof(sv)); model->layers[0].k_cache_s = scaled & 1 ? k_scale : nullptr; model->layers[0].v_cache_s = scaled & 2 ? v_scale : nullptr; std::vector tensors[4]; auto params = llama_context_default_params(); params.n_ctx = 256; params.n_batch = n_tokens; params.n_ubatch = n_tokens; params.n_threads = 1; params.n_threads_batch = 1; params.type_k = scaled & 1 ? GGML_TYPE_F8_E4M3 : GGML_TYPE_F16; params.type_v = scaled & 2 ? GGML_TYPE_F8_E4M3 : GGML_TYPE_F16; params.flash_attn_type = flash; params.offload_kqv = offload_kqv; if (model->split_mode() == LLAMA_SPLIT_MODE_TENSOR) { ggml_context_ptr ctx(ggml_init({ 8*ggml_tensor_overhead(), nullptr, true })); ggml_tensor * q = ggml_new_tensor_4d(ctx.get(), GGML_TYPE_F32, head_size, n_tokens, n_head, 1); ggml_tensor * k = ggml_new_tensor_4d(ctx.get(), params.type_k, head_size, params.n_ctx, n_head, 1); ggml_tensor * v = ggml_new_tensor_4d(ctx.get(), params.type_v, head_size, params.n_ctx, n_head, 1); ggml_tensor * mask = ggml_new_tensor_4d(ctx.get(), GGML_TYPE_F16, params.n_ctx, params.n_ctx, 1, 1); ggml_tensor * attn = ggml_flash_attn_ext(ctx.get(), q, k, v, mask, 1.0f/sqrtf(head_size), 0.0f, 0.0f); auto * dev = model->dev_layer(0); if (!ggml_backend_dev_supports_op(dev, attn)) { LOG("FP8 attention: scaled=%d offload=%d: tensor-split flash attention not supported on %s, skipping\n", scaled, offload_kqv, ggml_backend_dev_name(dev)); continue; } } if (offload_kqv) { // Cache writes cannot fall back from a preallocated device buffer. llama_kv_cache probe(*model, false, model->hparams, params.type_k, params.type_v, flash == LLAMA_FLASH_ATTN_TYPE_DISABLED, false, true, params.n_ctx, 1, 0, LLAMA_SWA_TYPE_NONE, nullptr, nullptr, nullptr, nullptr); ggml_context_ptr ctx(ggml_init({ 64*ggml_tensor_overhead(), nullptr, true })); llama_kv_cache::slot_info slots; bool supported = true; for (uint32_t il = 0; il < model->hparams.n_layer(); ++il) { ggml_tensor * k = ggml_new_tensor_3d(ctx.get(), GGML_TYPE_F32, head_size, n_head, n_tokens); ggml_tensor * v = ggml_new_tensor_3d(ctx.get(), GGML_TYPE_F32, head_size, n_head, n_tokens); ggml_tensor * k_idxs = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I64, n_tokens); ggml_tensor * v_idxs = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I64, flash == LLAMA_FLASH_ATTN_TYPE_DISABLED ? n_tokens*head_size*n_head : n_tokens); auto * dev = model->dev_layer(il); if (!ggml_backend_dev_supports_op(dev, probe.cpy_k(ctx.get(), k, k_idxs, il, slots)) || !ggml_backend_dev_supports_op(dev, probe.cpy_v(ctx.get(), v, v_idxs, il, slots))) { LOG("FP8 attention: flash=%s scaled=%d offload=1: cache writes not supported on %s, skipping\n", llama_flash_attn_type_name(flash), scaled, ggml_backend_dev_name(dev)); supported = false; break; } } if (!supported) { continue; } } params.cb_eval_user_data = tensors; params.cb_eval = [](ggml_tensor * t, bool ask, void * user_data) { const char * names[] = { "Qcur-0", "Kcur-0", "Vcur-0", "kqv_out-0" }; for (int i = 0; i < 4; ++i) { if (strcmp(t->name, names[i]) != 0) { continue; } if (!ask) { GGML_ASSERT(t->type == GGML_TYPE_F32 && ggml_is_contiguous(t)); auto & data = static_cast *>(user_data)[i]; data.resize(ggml_nelements(t)); ggml_backend_tensor_get(t, data.data(), 0, data.size()*sizeof(float)); } return true; } return false; }; llama_context_ptr lctx(llama_init_from_model(model, params)); GGML_ASSERT(lctx); GGML_ASSERT(lctx->get_cparams().flash_attn == (flash == LLAMA_FLASH_ATTN_TYPE_ENABLED)); get_logits(model, lctx.get(), { 1, 2, 3, 4 }); llama_synchronize(lctx.get()); for (const auto & data : tensors) { GGML_ASSERT(data.size() == (size_t) (head_size*n_head*n_tokens)); } // Round the original K/V independently of the cache write graph. for (int i : { 1, 2 }) { const float scale = i == 1 ? sk : sv; const ggml_type type = i == 1 ? params.type_k : params.type_v; auto & data = tensors[i]; if (type == GGML_TYPE_F8_E4M3) { for (float & value : data) { value /= scale; } std::vector quantized(data.size()); ggml_get_type_traits(type)->from_float_ref(data.data(), quantized.data(), data.size()); ggml_get_type_traits(type)->to_float(quantized.data(), data.data(), data.size()); for (float & value : data) { value *= scale; } } else { for (float & value : data) { value = ggml_fp16_to_fp32(ggml_fp32_to_fp16(value)); } } } std::vector expected(tensors[3].size(), 0.0f); for (int64_t token = 0; token < n_tokens; ++token) { for (int64_t head = 0; head < n_head; ++head) { const int64_t q_offset = (token*n_head + head)*head_size; std::vector scores(token + 1); double max_score = -INFINITY; for (int64_t past = 0; past <= token; ++past) { const int64_t kv_offset = (past*n_head + head)*head_size; double dot = 0.0; for (int64_t d = 0; d < head_size; ++d) { dot += (double) tensors[0][q_offset + d]*tensors[1][kv_offset + d]; } scores[past] = dot/sqrt((double) head_size); max_score = std::max(max_score, scores[past]); } double sum = 0.0; for (double & score : scores) { score = exp(score - max_score); sum += score; } for (int64_t past = 0; past <= token; ++past) { const int64_t kv_offset = (past*n_head + head)*head_size; for (int64_t d = 0; d < head_size; ++d) { expected[q_offset + d] += scores[past]/sum*tensors[2][kv_offset + d]; } } } } const double error = nmse(expected, tensors[3]); if (!std::isfinite(error) || error >= 1e-5) { LOG_ERR("FP8 attention: flash=%s scaled=%d offload=%d nmse=%.2e\n", llama_flash_attn_type_name(flash), scaled, offload_kqv, error); } GGML_ASSERT(std::isfinite(error) && error < 1e-5); } } model->layers[0].k_cache_s = old_k_scale; model->layers[0].v_cache_s = old_v_scale; } static void check_fp8_context_shift(llama_model * model) { ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr)); GGML_ASSERT(backend); auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend.get())); auto set_n_threads = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); GGML_ASSERT(set_n_threads); set_n_threads(backend.get(), 1); ggml_context_ptr scale_ctx(ggml_init({ 4096, nullptr, true })); ggml_tensor * scale = ggml_new_tensor_1d(scale_ctx.get(), GGML_TYPE_F32, 1); ggml_backend_buffer_ptr scale_buf(ggml_backend_alloc_ctx_tensors_from_buft(scale_ctx.get(), ggml_backend_cpu_buffer_type())); ggml_tensor * old_scale = model->layers[0].k_cache_s; for (float scale_value : { 1.0f, 0.5f }) { ggml_backend_tensor_set(scale, &scale_value, 0, sizeof(scale_value)); model->layers[0].k_cache_s = scale; auto params = llama_context_default_params(); params.n_ctx = 256; params.n_batch = 4; params.n_ubatch = 4; params.n_threads = 1; params.n_threads_batch = 1; params.type_k = GGML_TYPE_F8_E4M3; params.type_v = GGML_TYPE_F8_E4M3; params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED; params.offload_kqv = false; llama_context_ptr lctx(llama_init_from_model(model, params)); GGML_ASSERT(lctx); get_logits(model, lctx.get(), { 1, 2, 3, 4 }); llama_synchronize(lctx.get()); auto * kv = dynamic_cast(llama_get_memory(lctx.get())); GGML_ASSERT(kv && kv->get_can_shift()); ggml_tensor * k = kv->get_k_storage(0); std::vector before(ggml_nbytes(k)); ggml_backend_tensor_get(k, before.data(), 0, before.size()); ggml_context_ptr ctx(ggml_init({ 2*1024*1024, nullptr, false })); const auto & hp = model->hparams; const auto & cp = lctx->get_cparams(); ggml_tensor * keys = ggml_new_tensor_3d(ctx.get(), GGML_TYPE_F32, hp.n_embd_head_k(), hp.n_head_kv(), kv->get_size()); ggml_get_type_traits(GGML_TYPE_F8_E4M3)->to_float(before.data(), (float *) keys->data, ggml_nelements(keys)); ggml_tensor * shifts = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I32, kv->get_size()); std::fill_n(static_cast(shifts->data), ggml_nelements(shifts), 0); for (int i = 1; i < 4; ++i) { ((int32_t *) shifts->data)[i] = -1; } ggml_tensor * expected_f32 = ggml_rope_ext(ctx.get(), keys, shifts, model->get_rope_factors(cp, 0), hp.n_rot(), hp.rope_type, cp.n_ctx_orig_yarn, model->get_rope_freq_base(cp, 0), model->get_rope_freq_scale(cp, 0), cp.yarn_ext_factor, cp.yarn_attn_factor, cp.yarn_beta_fast, cp.yarn_beta_slow); ggml_cgraph * gf = ggml_new_graph(ctx.get()); ggml_build_forward_expand(gf, expected_f32); GGML_ASSERT(ggml_backend_graph_compute(backend.get(), gf) == GGML_STATUS_SUCCESS); std::vector expected(before.size()); ggml_quantize_chunk(GGML_TYPE_F8_E4M3, (float *) expected_f32->data, expected.data(), 0, ggml_nelements(expected_f32)/keys->ne[0], keys->ne[0], nullptr); llama_memory_seq_add(kv, 0, 1, 4, -1); GGML_ASSERT(kv->get_has_shift()); GGML_ASSERT(lctx->memory_update(false)); llama_synchronize(lctx.get()); GGML_ASSERT(!kv->get_has_shift()); std::vector after(before.size()); ggml_backend_tensor_get(k, after.data(), 0, after.size()); GGML_ASSERT(after != before); GGML_ASSERT(after == expected); lctx.reset(); } model->layers[0].k_cache_s = old_scale; } enum class test_status { PASS, FAIL, SKIP, }; static test_status test_fp8_kv(const std::string & path, const char * target_backend) { ggml_backend_dev_t cpu_devices[] = {nullptr}; auto params = llama_model_default_params(); params.devices = cpu_devices; params.progress_callback = silent_model_load_progress; { llama_model_ptr model = load_model(path, params); if (model->arch != LLM_ARCH_LLAMA || model->hparams.n_expert > 0) { LOG("FP8 KV: only dense Llama fixtures are supported, skipping %s\n", path.c_str()); return test_status::SKIP; } LOG("FP8 KV: CPU attention and context shift\n"); check_fp8_attention(model.get()); check_fp8_context_shift(model.get()); } std::vector devices_meta; for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { auto * dev = ggml_backend_dev_get(i); if (target_backend && strcmp(target_backend, ggml_backend_dev_name(dev)) != 0) { continue; } if (ggml_backend_dev_buffer_type(dev) != ggml_backend_cpu_buffer_type()) { devices_meta.push_back(dev); } if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) { continue; } ggml_backend_dev_t devices[] = {dev, nullptr}; params.devices = devices; LOG("FP8 KV: device=%s\n", ggml_backend_dev_name(dev)); llama_model_ptr model = load_model(path, params); check_fp8_attention(model.get()); check_fp8_attention(model.get(), false); } if (!target_backend && !devices_meta.empty()) { devices_meta.push_back(nullptr); params.devices = devices_meta.data(); params.split_mode = LLAMA_SPLIT_MODE_TENSOR; LOG("FP8 KV: tensor split\n"); llama_model_ptr model = load_model(path, params); check_fp8_attention(model.get()); check_fp8_attention(model.get(), false); } return test_status::PASS; } static void test_fp8(const std::string & model_path) { const auto dir = std::filesystem::temp_directory_path() / ("llama-fp8-" + std::to_string(std::random_device{}())); if (!std::filesystem::create_directory(dir)) { throw std::runtime_error("failed to create FP8 test directory"); } struct test_files { std::filesystem::path dir; ~test_files() { std::error_code ec; std::filesystem::remove_all(dir, ec); } } files{dir}; const auto input = (files.dir / "fp8.gguf").string(); const auto reference = (files.dir / "reference.gguf").string(); const auto output = (files.dir / "output.gguf").string(); const auto reference_output = (files.dir / "reference-output.gguf").string(); // Use the same CPU kernels for scaled weights and the folded reference. const llama_model_tensor_buft_override overrides[] = { {".*", ggml_backend_cpu_buffer_type()}, {nullptr, nullptr}, }; ggml_backend_dev_t devices[] = {nullptr}; auto model_params = llama_model_default_params(); model_params.devices = devices; model_params.tensor_buft_overrides = overrides; model_params.progress_callback = silent_model_load_progress; const int64_t n_embd = prepare_models(model_path, input, reference); check_model_save(input, model_params); check_model_save(reference, model_params); check_fp8_scales(input, reference, model_params); check_split_model(reference, files.dir, model_params); check_split_model(input, files.dir, model_params); check_fp8_requantize(input, reference, output, reference_output, n_embd, model_params); } static const char * test_status_str(test_status status) { switch (status) { case test_status::PASS: return "PASS"; case test_status::FAIL: return "FAIL"; case test_status::SKIP: return "SKIP"; } GGML_ABORT("invalid test status"); } int main(int argc, char ** argv) { if (argc == 2 && (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0)) { printf("Usage: %s (-m model.gguf | --models DIR) [--kv [-b backend]]\n", argv[0]); return 0; } std::string model_file; std::string models_dir; const char * target_backend = nullptr; bool kv = false; bool valid = true; for (int i = 1; i < argc; ++i) { if ((strcmp(argv[i], "-m") == 0 || strcmp(argv[i], "--model") == 0 || strcmp(argv[i], "--models") == 0) && i + 1 < argc) { if (!model_file.empty() || !models_dir.empty()) { valid = false; break; } const bool directory = strcmp(argv[i], "--models") == 0; if (directory) { models_dir = argv[++i]; } else { model_file = argv[++i]; } } else if (strcmp(argv[i], "--kv") == 0) { kv = true; } else if ((strcmp(argv[i], "-b") == 0 || strcmp(argv[i], "--backend") == 0) && i + 1 < argc) { target_backend = argv[++i]; } else { valid = false; break; } } if (!valid || (model_file.empty() && models_dir.empty()) || (target_backend && !kv)) { fprintf(stderr, "Usage: %s (-m model.gguf | --models DIR) [--kv [-b backend]]\n", argv[0]); return 1; } common_init(); llama_backend_init(); int result = 0; try { if (target_backend && !ggml_backend_dev_by_name(target_backend)) { throw std::runtime_error("unknown backend device: " + std::string(target_backend)); } std::vector models; if (!models_dir.empty()) { if (!std::filesystem::is_directory(models_dir)) { throw std::runtime_error("models directory does not exist: " + models_dir); } for (const auto & entry : std::filesystem::directory_iterator(models_dir)) { if (entry.is_regular_file() && entry.path().extension() == ".gguf") { models.push_back(entry.path().string()); } } std::sort(models.begin(), models.end()); if (models.empty()) { throw std::runtime_error("no .gguf models found in: " + models_dir); } } else { models.push_back(model_file); } std::vector results; for (const auto & model : models) { try { if (kv) { results.push_back(test_fp8_kv(model, target_backend)); } else { test_fp8(model); results.push_back(test_status::PASS); } } catch (const std::exception & err) { fprintf(stderr, "FP8 test failed for '%s': %s\n", model.c_str(), err.what()); results.push_back(test_status::FAIL); result = 1; } } printf("\n| Model | Result |\n|-------|--------|\n"); for (size_t i = 0; i < models.size(); ++i) { printf("| %s | %s |\n", std::filesystem::path(models[i]).filename().string().c_str(), test_status_str(results[i])); } } catch (const std::exception & err) { fprintf(stderr, "FP8 test failed: %s\n", err.what()); result = 1; } llama_backend_free(); return result; }