ggml : add MUL_MAT src1 precision tests and gating (#31896)

This commit is contained in:
Georgi Gerganov authored and GitHub committed 2026-11-10 17:03:27 +02:00
1 parent 281e7dad7c
commit 494d0f47bd
19 files changed
+192 -61

No files matched your search

+115
View File
@@ -5677,6 +5677,91 @@ struct test_mul_mat_w4a8 : public test_mul_mat {
}
};
// src1 GGML_PREC_BF16: the implementation may keep src1 in F32 or BF16, but must not convert it down to F16,
// which cannot hold the src1 values that the BF16 range allows
struct test_mul_mat_prec_src1_bf16 : public test_mul_mat {
test_mul_mat_prec_src1_bf16(ggml_type type_a = GGML_TYPE_F32, int64_t m = 64, int64_t n = 32, int64_t k = 256,
std::array<int64_t, 2> bs = {1, 1},
std::array<int64_t, 2> nr = {1, 1})
: test_mul_mat(type_a, GGML_TYPE_F32, m, n, k, bs, nr) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
ggml_tensor * out = test_mul_mat::build_graph(ctx);
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
if (t->op == GGML_OP_MUL_MAT) {
ggml_prec_set_src(t, GGML_PREC_BF16, 1);
}
}
return out;
}
void initialize_tensors(ggml_context * ctx) override {
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
if (strcmp(t->name, "b") == 0) {
init_tensor_uniform(t, -1e6f, 1e6f);
} else {
init_tensor_uniform(t);
}
}
}
std::string op_desc(ggml_tensor * t) override {
GGML_UNUSED(t);
return "MUL_MAT_PREC_SRC1_BF16";
}
};
// src1 GGML_PREC_F32: the implementation can not convert src1 or the F32 src0 tiles down to F16 or TF32
struct test_mul_mat_prec_src1_f32 : public test_mul_mat {
test_mul_mat_prec_src1_f32(ggml_type type_a = GGML_TYPE_F32, int64_t m = 64, int64_t n = 32, int64_t k = 256,
std::array<int64_t, 2> bs = {1, 1},
std::array<int64_t, 2> nr = {1, 1})
: test_mul_mat(type_a, GGML_TYPE_F32, m, n, k, bs, nr) {}
double max_nmse_err() override {
return 1e-10;
}
ggml_tensor * build_graph(ggml_context * ctx) override {
ggml_tensor * out = test_mul_mat::build_graph(ctx);
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
if (t->op == GGML_OP_MUL_MAT) {
ggml_prec_set_src(t, GGML_PREC_F32, 1);
}
}
return out;
}
void initialize_tensors(ggml_context * ctx) override {
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
if (strcmp(t->name, "b") == 0) {
init_tensor_uniform(t, -1e6f, 1e6f);
} else {
init_tensor_uniform(t);
}
}
}
std::string op_desc(ggml_tensor * t) override {
GGML_UNUSED(t);
return "MUL_MAT_PREC_SRC1_F32";
}
};
// src1 GGML_PREC_BF16 with src0 values outside the F16 range: the F32 src0 tiles must not be downcast to F16
struct test_mul_mat_prec_src1_bf16_src0 : public test_mul_mat_prec_src1_bf16 {
test_mul_mat_prec_src1_bf16_src0(ggml_type type_a = GGML_TYPE_F32, int64_t m = 64, int64_t n = 32, int64_t k = 256,
std::array<int64_t, 2> bs = {1, 1},
std::array<int64_t, 2> nr = {1, 1})
: test_mul_mat_prec_src1_bf16(type_a, m, n, k, bs, nr) {}
void initialize_tensors(ggml_context * ctx) override {
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
if (strcmp(t->name, "a") == 0) {
init_tensor_uniform(t, -1e6f, 1e6f);
} else {
init_tensor_uniform(t);
}
}
}
std::string op_desc(ggml_tensor * t) override {
GGML_UNUSED(t);
return "MUL_MAT_PREC_SRC1_BF16_SRC0";
}
};
// FP4 native W4A4 path (default precision)
struct test_mul_mat_w4a4 : public test_mul_mat {
test_mul_mat_w4a4(ggml_type type_a = GGML_TYPE_NVFP4, ggml_type type_b = GGML_TYPE_F32,
@@ -11634,6 +11719,36 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 8192, 1, 8192));
test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 1024, 7, 1024)); // many rows
// src1 precision: few-row paths, aligned tiles, partial tiles (K and output), broadcast batches
for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q8_0}) {
// Q8_0 requires K to be a multiple of its block size (32)
const int64_t k_partial = type_a == GGML_TYPE_Q8_0 ? 96 : 100;
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 1, 256));
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 8, 256));
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 32, 256));
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 128, 512, 512));
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 67, 33, k_partial));
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 32, 256, {2, 3}, {2, 1}));
}
// src1 precision: same shapes with GGML_PREC_F32, which the backends must not round down to F16 or TF32
for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16}) {
test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 1, 256));
test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 8, 256));
test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 32, 256));
test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 128, 512, 512));
test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 67, 33, 100));
test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 32, 256, {2, 3}, {2, 1}));
}
// src0 range: an F32 src0 must not be downcast to F16 when src1 requests BF16
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 1, 256));
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 8, 256));
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 32, 256));
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 128, 512, 512));
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 67, 33, 100));
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 32, 256, {2, 3}, {2, 1}));
// FP4 activation precision (default = native W4A4, src1 GGML_PREC_Q8 = W4A8)
test_cases.emplace_back(new test_mul_mat_w4a8(GGML_TYPE_NVFP4, GGML_TYPE_F32, 32, 1, 256));
test_cases.emplace_back(new test_mul_mat_w4a8(GGML_TYPE_NVFP4, GGML_TYPE_F32, 32, 32, 256));