Files
llama.cpp/tests/test-sleep.cpp
T
ac830a9cbb ggml : add backend scheduler tests (#27258)
* Adds initial scheduler stress tests

* Adds new sleep op to simplify scheduler testing

* Adds testcase addressing https://github.com/ggml-org/llama.cpp/issues/23321

* Adds experimental vulkan GGML_OP_SLEEP

* Improves structure and naming of previous tests. Adds new tests,
including one which detects the nkvo-bug correctly, and validates the
proposed hotfix.

* Cleanup: Removes CUDA specific comments and nvtx-markers

* Fix: forgot to add sleep specific test case

* Fix oversights identified by CI

* Drops default prints so that it behaves like other ctests. Adds -v flag
to toggle this behavior

* Fix: Early exit in CPU-only settings (cause for graviton CI fails)

* Improves stress_test_linked_list by removing overlap with
test_chain_all_backends and test_pair_user_inputs. The latter ones test
either activations or user inputs in isolation and just once, whereas
the stress test tests all combinations multiple rounds

* Exclude ACCEL devices from new tests due to incompatibility

* Fixes vulkan test failure in llvmpipe

* Adds asserts, improves comments and some checks

* Fixes nitpicks in white-space and variable naming

* Adds comments that certain vk drivers could time OP_SLEEP incorrectly

* Drops limitiation to CPU/GPU in test-sleep

* Aligns OP_SLEEP guards in cuda with vk guards (input shape and
contiguity)

* Implements feedback for OP_SLEEP to work in place (no copy)

* Apply non-code suggestions  from JohannesGaesslers code review

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>

* Removes struct backend_supports; enables tests to return skip, instead
of only pass/fail

* Make clang-tidy happy

* Removes sched_backend_caps, improves create_test_scheduler according to
feedback (L149)

* Removes "name" field from struct sched_check in favor of
ggml_set_name/ggml_get_name

* Drops unnecessary shader_clock.comp test and guards

* Replaces sched_ prefixes with helper_ prefix

* Moves test suite to new RAII types for better memory management

* Adds sleep option to test_chain_all_backends

* Adds sleep option to first-run arm of the y-shaped test

* Fix switchcase order for legibility in stress_test_dag

* Simplfies stress_test_linked_list via double for-loop

* Adds pseudo-random input to stress_test_linked_list, and runs all
transitions both with and without CPU input (next to activation copy as
input)

* Fix RPC Proto patch due to GGML_OPS bump

* Adds ggml-cpp.h treatment to test-sleep.cpp

* Adds TODO pointers for subsequent additional tests.

* Apply suggestion to ggml.h

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>

* Apply suggestion to tests/test-backend-sched.cpp

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>

* Apply suggestion to tests/test-backend-sched.cpp

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>

* Fixes node compatibility check to not take buffer locations into account.
This led to skipped tests due to incorrectly rejected ops due to wrong buffer locations in multi-GPU (which is one of the scheduler behaviors under test)

* Apply batched suggestions from gg code review

Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>

* Completes callsite change of gg-suggestion

* Switch test-sleep from std::chrono::steady_clock::now() to ggml_time_us
to avoid mac clock drift and spurious test failures.
This makes the test a bit nonsensical on CPU (same clock checks itself),
but the test still has merit on the other backends where
independent device clocks are used.

---------

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2026-11-19 16:40:13 +02:00

155 lines
5.6 KiB
C++

// Tests for GGML_OP_SLEEP, on every backend that supports it. Three properties are checked:
// - the op only delays: input = output
// - the op does not return early: a chain of n sleeps must take at least n*us
// Only a lower bound is asserted for the duration. A busy-wait cannot finish early, so that bound is
// deterministic, whereas an upper bound would be flaky on a loaded machine.
// Known limitation: Vulkan does not specify the tick rate of the shader clock, see ggml_vk_sleep. A driver
// whose clock runs slower than assumed could undershoot the request and fail this bound.
#include "ggml.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include "ggml-cpp.h"
#include <cinttypes>
#include <cstdio>
#include <cstring>
#include <random>
#include <vector>
enum test_result {
TEST_OK,
TEST_FAIL,
TEST_SKIP,
};
static test_result test_sleep(ggml_backend_t backend, ggml_type type, int64_t ne, int32_t us, int n_nodes) {
const size_t graph_size = n_nodes + 16;
ggml_init_params params_static = {
/*.mem_size =*/ ggml_tensor_overhead(),
/*.mem_buffer =*/ nullptr,
/*.no_alloc =*/ true,
};
ggml_context_ptr ctx_static(ggml_init(params_static));
ggml_tensor * x = ggml_new_tensor_1d(ctx_static.get(), type, ne);
ggml_set_name(x, "x");
ggml_set_input(x);
ggml_init_params params_compute = {
/*.mem_size =*/ n_nodes*ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false),
/*.mem_buffer =*/ nullptr,
/*.no_alloc =*/ true,
};
ggml_context_ptr ctx_compute(ggml_init(params_compute));
ggml_cgraph * gf = ggml_new_graph_custom(ctx_compute.get(), graph_size, false);
ggml_tensor * out = x;
for (int i = 0; i < n_nodes; i++) {
out = ggml_sleep(ctx_compute.get(), out, us);
}
ggml_set_name(out, "out");
ggml_build_forward_expand(gf, out);
// the backends log to stdout while computing, so the result line is only printed once it is complete
char desc[128];
snprintf(desc, sizeof(desc), " %-6s %-4s ne=%-9" PRId64 " us=%-7d n_nodes=%d ",
ggml_backend_name(backend), ggml_type_name(type), ne, us, n_nodes);
test_result result = TEST_OK;
if (!ggml_backend_supports_op(backend, out)) {
printf("%snot supported\n", desc);
result = TEST_SKIP;
} else {
ggml_backend_buffer_ptr buf_static(ggml_backend_alloc_ctx_tensors(ctx_static.get(), backend));
// random bytes rather than random floats, so that bit patterns a value-wise copy could
// mangle, such as NaNs and denormals, are covered as well
std::vector<uint8_t> data_in(ggml_nbytes(x));
std::mt19937 rng(1234);
for (size_t i = 0; i < data_in.size(); i++) {
data_in[i] = rng() & 0xFF;
}
ggml_backend_tensor_set(x, data_in.data(), 0, data_in.size());
ggml_gallocr_ptr galloc(ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)));
ggml_gallocr_alloc_graph(galloc.get(), gf);
// warm up, so that one-time costs such as lazy CUDA module loading are not timed
ggml_backend_graph_compute(backend, gf);
const int64_t t_start_us = ggml_time_us();
const ggml_status status = ggml_backend_graph_compute(backend, gf);
const int64_t t_us = ggml_time_us() - t_start_us;
std::vector<uint8_t> data_out(ggml_nbytes(out));
ggml_backend_tensor_get(out, data_out.data(), 0, data_out.size());
const int64_t t_min_us = (int64_t) n_nodes * us;
const bool ok_status = status == GGML_STATUS_SUCCESS;
const bool ok_alias = out->data == x->data;
const bool ok_data = memcmp(data_in.data(), data_out.data(), data_in.size()) == 0;
const bool ok_time = t_us >= t_min_us;
printf("%selapsed=%7" PRId64 " us (>= %6" PRId64 ") alias: %-4s data: %-4s time: %s\n",
desc, t_us, t_min_us, ok_alias ? "OK" : "FAIL", ok_data ? "OK" : "FAIL", ok_time ? "OK" : "FAIL");
if (!ok_status) {
printf(" compute failed: %s\n", ggml_status_to_string(status));
}
result = ok_status && ok_alias && ok_data && ok_time ? TEST_OK : TEST_FAIL;
}
return result;
}
int main() {
ggml_time_init();
ggml_backend_load_all();
int n_ok = 0;
int n_fail = 0;
int n_skip = 0;
for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
ggml_backend_ptr backend(ggml_backend_dev_init(dev, nullptr));
if (backend == nullptr) {
printf(" failed to initialize %s\n", ggml_backend_dev_name(dev));
n_fail++;
continue;
}
// a zero duration must still be a valid no-op, longer ones are checked against the clock,
// and the last case verifies that a chain of sleeps accumulates instead of collapsing into one
const struct {
ggml_type type;
int64_t ne;
int32_t us;
int n_nodes;
} cases[] = {
{ GGML_TYPE_F32, 1, 0, 1 },
{ GGML_TYPE_F32, 4096, 10000, 1 },
{ GGML_TYPE_F16, 1024, 10000, 1 },
{ GGML_TYPE_F32, 1 << 20, 5000, 4 },
};
for (const auto & c : cases) {
switch (test_sleep(backend.get(), c.type, c.ne, c.us, c.n_nodes)) {
case TEST_OK: n_ok++; break;
case TEST_FAIL: n_fail++; break;
case TEST_SKIP: n_skip++; break;
}
}
}
printf("%d passed, %d failed, %d skipped\n", n_ok, n_fail, n_skip);
return n_fail > 0;
}