ggml : add MUL_MAT src1 precision tests and gating (#31896)
This commit is contained in:
1 parent
281e7dad7c
commit
494d0f47bd
19 files changed
+192
-61
No files matched your search
@@ -5677,6 +5677,91 @@ struct test_mul_mat_w4a8 : public test_mul_mat {
|
||||
}
|
||||
};
|
||||
|
||||
// src1 GGML_PREC_BF16: the implementation may keep src1 in F32 or BF16, but must not convert it down to F16,
|
||||
// which cannot hold the src1 values that the BF16 range allows
|
||||
struct test_mul_mat_prec_src1_bf16 : public test_mul_mat {
|
||||
test_mul_mat_prec_src1_bf16(ggml_type type_a = GGML_TYPE_F32, int64_t m = 64, int64_t n = 32, int64_t k = 256,
|
||||
std::array<int64_t, 2> bs = {1, 1},
|
||||
std::array<int64_t, 2> nr = {1, 1})
|
||||
: test_mul_mat(type_a, GGML_TYPE_F32, m, n, k, bs, nr) {}
|
||||
ggml_tensor * build_graph(ggml_context * ctx) override {
|
||||
ggml_tensor * out = test_mul_mat::build_graph(ctx);
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->op == GGML_OP_MUL_MAT) {
|
||||
ggml_prec_set_src(t, GGML_PREC_BF16, 1);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
void initialize_tensors(ggml_context * ctx) override {
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (strcmp(t->name, "b") == 0) {
|
||||
init_tensor_uniform(t, -1e6f, 1e6f);
|
||||
} else {
|
||||
init_tensor_uniform(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
std::string op_desc(ggml_tensor * t) override {
|
||||
GGML_UNUSED(t);
|
||||
return "MUL_MAT_PREC_SRC1_BF16";
|
||||
}
|
||||
};
|
||||
|
||||
// src1 GGML_PREC_F32: the implementation can not convert src1 or the F32 src0 tiles down to F16 or TF32
|
||||
struct test_mul_mat_prec_src1_f32 : public test_mul_mat {
|
||||
test_mul_mat_prec_src1_f32(ggml_type type_a = GGML_TYPE_F32, int64_t m = 64, int64_t n = 32, int64_t k = 256,
|
||||
std::array<int64_t, 2> bs = {1, 1},
|
||||
std::array<int64_t, 2> nr = {1, 1})
|
||||
: test_mul_mat(type_a, GGML_TYPE_F32, m, n, k, bs, nr) {}
|
||||
double max_nmse_err() override {
|
||||
return 1e-10;
|
||||
}
|
||||
ggml_tensor * build_graph(ggml_context * ctx) override {
|
||||
ggml_tensor * out = test_mul_mat::build_graph(ctx);
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->op == GGML_OP_MUL_MAT) {
|
||||
ggml_prec_set_src(t, GGML_PREC_F32, 1);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
void initialize_tensors(ggml_context * ctx) override {
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (strcmp(t->name, "b") == 0) {
|
||||
init_tensor_uniform(t, -1e6f, 1e6f);
|
||||
} else {
|
||||
init_tensor_uniform(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
std::string op_desc(ggml_tensor * t) override {
|
||||
GGML_UNUSED(t);
|
||||
return "MUL_MAT_PREC_SRC1_F32";
|
||||
}
|
||||
};
|
||||
|
||||
// src1 GGML_PREC_BF16 with src0 values outside the F16 range: the F32 src0 tiles must not be downcast to F16
|
||||
struct test_mul_mat_prec_src1_bf16_src0 : public test_mul_mat_prec_src1_bf16 {
|
||||
test_mul_mat_prec_src1_bf16_src0(ggml_type type_a = GGML_TYPE_F32, int64_t m = 64, int64_t n = 32, int64_t k = 256,
|
||||
std::array<int64_t, 2> bs = {1, 1},
|
||||
std::array<int64_t, 2> nr = {1, 1})
|
||||
: test_mul_mat_prec_src1_bf16(type_a, m, n, k, bs, nr) {}
|
||||
void initialize_tensors(ggml_context * ctx) override {
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (strcmp(t->name, "a") == 0) {
|
||||
init_tensor_uniform(t, -1e6f, 1e6f);
|
||||
} else {
|
||||
init_tensor_uniform(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
std::string op_desc(ggml_tensor * t) override {
|
||||
GGML_UNUSED(t);
|
||||
return "MUL_MAT_PREC_SRC1_BF16_SRC0";
|
||||
}
|
||||
};
|
||||
|
||||
// FP4 native W4A4 path (default precision)
|
||||
struct test_mul_mat_w4a4 : public test_mul_mat {
|
||||
test_mul_mat_w4a4(ggml_type type_a = GGML_TYPE_NVFP4, ggml_type type_b = GGML_TYPE_F32,
|
||||
@@ -11634,6 +11719,36 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 8192, 1, 8192));
|
||||
test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 1024, 7, 1024)); // many rows
|
||||
|
||||
// src1 precision: few-row paths, aligned tiles, partial tiles (K and output), broadcast batches
|
||||
for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q8_0}) {
|
||||
// Q8_0 requires K to be a multiple of its block size (32)
|
||||
const int64_t k_partial = type_a == GGML_TYPE_Q8_0 ? 96 : 100;
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 1, 256));
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 8, 256));
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 32, 256));
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 128, 512, 512));
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 67, 33, k_partial));
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 32, 256, {2, 3}, {2, 1}));
|
||||
}
|
||||
|
||||
// src1 precision: same shapes with GGML_PREC_F32, which the backends must not round down to F16 or TF32
|
||||
for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16}) {
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 1, 256));
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 8, 256));
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 32, 256));
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 128, 512, 512));
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 67, 33, 100));
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 32, 256, {2, 3}, {2, 1}));
|
||||
}
|
||||
|
||||
// src0 range: an F32 src0 must not be downcast to F16 when src1 requests BF16
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 1, 256));
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 8, 256));
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 32, 256));
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 128, 512, 512));
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 67, 33, 100));
|
||||
test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 32, 256, {2, 3}, {2, 1}));
|
||||
|
||||
// FP4 activation precision (default = native W4A4, src1 GGML_PREC_Q8 = W4A8)
|
||||
test_cases.emplace_back(new test_mul_mat_w4a8(GGML_TYPE_NVFP4, GGML_TYPE_F32, 32, 1, 256));
|
||||
test_cases.emplace_back(new test_mul_mat_w4a8(GGML_TYPE_NVFP4, GGML_TYPE_F32, 32, 32, 256));
|
||||
|
||||
Reference in new issue
Block a user