tests: Fail test-backend-ops and exit with error status 1 when missing and/or unsupported types and/or unsupported backends are provided in the command line (#30279)
* Fail tests on missing or unsupported type * Fix output * Remove whitespace * Fail if no tests ran regardless of backend support * Revert unnecessary cosmetic change
This commit is contained in:
1 parent
6d3568e31d
commit
5a4afd8077
1 file changed
+83
-55
+83
-55
@@ -14232,25 +14232,25 @@ static bool op_names_filter_selects(const char * op_names_filter, const char * o
|
||||
|
||||
// Covers padded rows, sinks, kvpad, multi-SIMDgroup reduction, quantized K/V, and MLA views.
|
||||
// The override is backend-global, so this runs after all parallel workers have joined.
|
||||
static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu, const char * op_names_filter) {
|
||||
static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu, const char * op_names_filter, const char * params_filter, const char * test_file_path) {
|
||||
const char * LLAMA_TEST_FA_VEC_DISABLE = getenv("LLAMA_TEST_FA_VEC_DISABLE");
|
||||
if (LLAMA_TEST_FA_VEC_DISABLE) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!op_names_filter_selects(op_names_filter, "FLASH_ATTN_EXT")) {
|
||||
if (test_file_path != nullptr || !op_names_filter_selects(op_names_filter, "FLASH_ATTN_EXT")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend));
|
||||
|
||||
auto set_ov = (set_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override");
|
||||
auto clear_ov = (clear_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override");
|
||||
if (!set_ov || !clear_ov) {
|
||||
return true; // not the Metal backend: nothing to force
|
||||
}
|
||||
|
||||
printf("Running FA vec slice tests (env LLAMA_TEST_FA_VEC_DISABLE=1 to skip)\n");
|
||||
std::regex params_filter_regex;
|
||||
if (params_filter != nullptr) { params_filter_regex = std::regex(params_filter); }
|
||||
|
||||
struct shape_t { int dk, dv; };
|
||||
const shape_t shapes[] = { { 128, 128 }, { 576, 512 } }; // mainstream head size + MLA shared K/V view
|
||||
@@ -14267,10 +14267,13 @@ static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu,
|
||||
for (bool sinks : { false, true }) {
|
||||
for (int ne01 : ne01_pts) {
|
||||
for (int ne11 : ne11_pts) {
|
||||
set_ov(Q, ne);
|
||||
test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, { 1, 1 }, /*kv=*/ne11, /*nb=*/ne01,
|
||||
/*mask=*/true, sinks, 0.0f, 0.0f, GGML_PREC_F32,
|
||||
type_kv, type_kv);
|
||||
if (params_filter != nullptr && !std::regex_search(tc.vars(), params_filter_regex)) { continue; }
|
||||
if (n_run == 0) { printf("Running FA vec slice tests (env LLAMA_TEST_FA_VEC_DISABLE=1 to skip)\n"); }
|
||||
|
||||
set_ov(Q, ne);
|
||||
auto st = tc.eval(backend, backend_cpu, "FLASH_ATTN_EXT", nullptr);
|
||||
clear_ov();
|
||||
|
||||
@@ -14288,30 +14291,12 @@ static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu,
|
||||
}
|
||||
}
|
||||
|
||||
printf(" fa_vec (Q,NE) slice: %d cases run, %d failed\n", n_run, n_fail);
|
||||
if (n_run > 0) { printf(" fa_vec (Q,NE) slice: %d cases run, %d failed\n", n_run, n_fail); }
|
||||
|
||||
return n_fail == 0;
|
||||
}
|
||||
|
||||
static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mode mode, const char * op_names_filter, const char * params_filter,
|
||||
printer * output_printer, const char * test_file_path, int parallel_workers) {
|
||||
auto filter_test_cases = [](std::vector<std::unique_ptr<test_case>> & test_cases, const char * params_filter) {
|
||||
if (params_filter == nullptr) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::regex params_filter_regex(params_filter);
|
||||
|
||||
for (auto it = test_cases.begin(); it != test_cases.end();) {
|
||||
if (!std::regex_search((*it)->vars(), params_filter_regex)) {
|
||||
it = test_cases.erase(it);
|
||||
continue;
|
||||
}
|
||||
|
||||
it++;
|
||||
}
|
||||
};
|
||||
|
||||
static std::vector<std::unique_ptr<test_case>> make_test_cases(test_mode mode, const char * params_filter, const char * test_file_path) {
|
||||
std::vector<std::unique_ptr<test_case>> test_cases;
|
||||
|
||||
if (test_file_path == nullptr) {
|
||||
@@ -14329,7 +14314,45 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo
|
||||
test_cases = make_test_cases_from_file(test_file_path);
|
||||
}
|
||||
|
||||
filter_test_cases(test_cases, params_filter);
|
||||
if (params_filter != nullptr) {
|
||||
std::regex params_filter_regex(params_filter);
|
||||
test_cases.erase( std::remove_if(test_cases.begin(), test_cases.end(),
|
||||
[&](const std::unique_ptr<test_case> & tc) { return !std::regex_search(tc->vars(), params_filter_regex); }),
|
||||
test_cases.end());
|
||||
}
|
||||
|
||||
if (mode == MODE_GRAD || mode == MODE_SUPPORT) {
|
||||
test_cases.erase(std::remove_if(test_cases.begin(), test_cases.end(),
|
||||
[](const std::unique_ptr<test_case> & tc) { return tc->run_whole_graph(); }),
|
||||
test_cases.end());
|
||||
}
|
||||
|
||||
return test_cases;
|
||||
}
|
||||
|
||||
static bool test_case_selected(const test_mode mode, const char * op_names_filter, const char * params_filter, const char * test_file_path) {
|
||||
ggml_init_params params = {
|
||||
/* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead(),
|
||||
/* .mem_base = */ NULL,
|
||||
/* .no_alloc = */ true,
|
||||
};
|
||||
|
||||
for (auto & tc : make_test_cases(mode, params_filter, test_file_path)) {
|
||||
ggml_context_ptr ctx(ggml_init(params));
|
||||
GGML_ASSERT(ctx);
|
||||
ggml_context_ptr ctx_weights(tc->use_weight_context() ? ggml_init(params) : nullptr);
|
||||
|
||||
tc->mode = mode;
|
||||
tc->gf = ggml_new_graph(ctx.get());
|
||||
if (tc->matches_filter(tc->build_graph(ctx.get(), ctx_weights.get()), op_names_filter)) { return true; }
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mode mode, const char * op_names_filter, const char * params_filter,
|
||||
printer * output_printer, const char * test_file_path, int parallel_workers, const bool must_run) {
|
||||
std::vector<std::unique_ptr<test_case>> test_cases = make_test_cases(mode, params_filter, test_file_path);
|
||||
|
||||
if (mode == MODE_TEST) {
|
||||
ggml_backend_ptr backend_cpu(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, NULL));
|
||||
@@ -14373,9 +14396,7 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo
|
||||
for (size_t i = my_begin; i < my_end; ++i) {
|
||||
auto & test = test_cases[i];
|
||||
test_status_t status = test->eval(b, b_cpu, op_names_filter, output_printer);
|
||||
if (status == test_status_t::SKIPPED || status == test_status_t::NOT_SUPPORTED) {
|
||||
continue;
|
||||
}
|
||||
if (status == test_status_t::SKIPPED || status == test_status_t::NOT_SUPPORTED) { continue; }
|
||||
tests_run++;
|
||||
if (status == test_status_t::OK) {
|
||||
n_ok++;
|
||||
@@ -14430,24 +14451,21 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo
|
||||
output_printer->print_summary(test_summary_info(n_ok, tests_run, false));
|
||||
output_printer->print_failed_tests(failed_tests);
|
||||
|
||||
const bool slice_ok = run_fa_vec_slice(backend, backend_cpu.get(), op_names_filter);
|
||||
bool ran_any = true;
|
||||
if (tests_run == 0 && must_run) {
|
||||
printf(" no test case ran: the selected test cases are not supported\n");
|
||||
ran_any = false;
|
||||
}
|
||||
|
||||
return n_ok == tests_run && slice_ok;
|
||||
const bool slice_ok = run_fa_vec_slice(backend, backend_cpu.get(), op_names_filter, params_filter, test_file_path);
|
||||
|
||||
return n_ok == tests_run && slice_ok && ran_any;
|
||||
}
|
||||
|
||||
if (mode == MODE_GRAD) {
|
||||
test_cases.erase(
|
||||
std::remove_if(test_cases.begin(), test_cases.end(), [](const std::unique_ptr<test_case> & tc) {
|
||||
return tc->run_whole_graph();
|
||||
}),
|
||||
test_cases.end()
|
||||
);
|
||||
|
||||
size_t n_ok = 0;
|
||||
for (auto & test : test_cases) {
|
||||
if (test->eval_grad(backend, op_names_filter, output_printer)) {
|
||||
n_ok++;
|
||||
}
|
||||
if (test->eval_grad(backend, op_names_filter, output_printer)) { n_ok++; }
|
||||
}
|
||||
output_printer->print_summary(test_summary_info(n_ok, test_cases.size(), false));
|
||||
|
||||
@@ -14462,14 +14480,6 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo
|
||||
}
|
||||
|
||||
if (mode == MODE_SUPPORT) {
|
||||
// Filter out fusion cases
|
||||
test_cases.erase(
|
||||
std::remove_if(test_cases.begin(), test_cases.end(), [](const std::unique_ptr<test_case> & tc) {
|
||||
return tc->run_whole_graph();
|
||||
}),
|
||||
test_cases.end()
|
||||
);
|
||||
|
||||
for (auto & test : test_cases) {
|
||||
test->eval_support(backend, op_names_filter, output_printer);
|
||||
}
|
||||
@@ -14686,6 +14696,27 @@ int main(int argc, char ** argv) {
|
||||
// load and enumerate backends
|
||||
ggml_backend_load_all();
|
||||
|
||||
if (backend_filter != NULL) {
|
||||
bool found = false;
|
||||
std::string names;
|
||||
for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
|
||||
const char * name = ggml_backend_dev_name(ggml_backend_dev_get(i));
|
||||
found = found || strcmp(backend_filter, name) == 0;
|
||||
if (!names.empty()) { names += ", "; }
|
||||
names += name;
|
||||
}
|
||||
|
||||
if (!found) {
|
||||
fprintf(stderr, "error: no device named '%s' (devices: %s)\n", backend_filter, names.c_str());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (!test_case_selected(mode, op_names_filter, params_filter, test_file_path)) {
|
||||
fprintf(stderr, "error: no test case matches the filters\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Create printer for output format
|
||||
std::unique_ptr<printer> output_printer = create_printer(output_format);
|
||||
if (output_printer) {
|
||||
@@ -14728,13 +14759,10 @@ int main(int argc, char ** argv) {
|
||||
false, "", ggml_backend_dev_description(dev),
|
||||
total / 1024 / 1024, free / 1024 / 1024, true));
|
||||
|
||||
bool ok = test_backend(backend.get(), dev, mode, op_names_filter, params_filter, output_printer.get(), test_file_path, parallel_workers);
|
||||
bool ok = test_backend(backend.get(), dev, mode, op_names_filter, params_filter, output_printer.get(), test_file_path, parallel_workers, backend_filter != NULL);
|
||||
|
||||
if (ok) {
|
||||
n_ok++;
|
||||
}
|
||||
output_printer->print_backend_status(
|
||||
backend_status_info(ggml_backend_name(backend.get()), ok ? test_status_t::OK : test_status_t::FAIL));
|
||||
if (ok) { n_ok++; }
|
||||
output_printer->print_backend_status(backend_status_info(ggml_backend_name(backend.get()), ok ? test_status_t::OK : test_status_t::FAIL));
|
||||
}
|
||||
|
||||
ggml_quantize_free();
|
||||
|
||||
Reference in new issue
Block a user