* Adds initial scheduler stress tests * Adds new sleep op to simplify scheduler testing * Adds testcase addressing https://github.com/ggml-org/llama.cpp/issues/23321 * Adds experimental vulkan GGML_OP_SLEEP * Improves structure and naming of previous tests. Adds new tests, including one which detects the nkvo-bug correctly, and validates the proposed hotfix. * Cleanup: Removes CUDA specific comments and nvtx-markers * Fix: forgot to add sleep specific test case * Fix oversights identified by CI * Drops default prints so that it behaves like other ctests. Adds -v flag to toggle this behavior * Fix: Early exit in CPU-only settings (cause for graviton CI fails) * Improves stress_test_linked_list by removing overlap with test_chain_all_backends and test_pair_user_inputs. The latter ones test either activations or user inputs in isolation and just once, whereas the stress test tests all combinations multiple rounds * Exclude ACCEL devices from new tests due to incompatibility * Fixes vulkan test failure in llvmpipe * Adds asserts, improves comments and some checks * Fixes nitpicks in white-space and variable naming * Adds comments that certain vk drivers could time OP_SLEEP incorrectly * Drops limitiation to CPU/GPU in test-sleep * Aligns OP_SLEEP guards in cuda with vk guards (input shape and contiguity) * Implements feedback for OP_SLEEP to work in place (no copy) * Apply non-code suggestions from JohannesGaesslers code review Co-authored-by: Johannes Gäßler <johannesg@5d6.de> * Removes struct backend_supports; enables tests to return skip, instead of only pass/fail * Make clang-tidy happy * Removes sched_backend_caps, improves create_test_scheduler according to feedback (L149) * Removes "name" field from struct sched_check in favor of ggml_set_name/ggml_get_name * Drops unnecessary shader_clock.comp test and guards * Replaces sched_ prefixes with helper_ prefix * Moves test suite to new RAII types for better memory management * Adds sleep option to test_chain_all_backends * Adds sleep option to first-run arm of the y-shaped test * Fix switchcase order for legibility in stress_test_dag * Simplfies stress_test_linked_list via double for-loop * Adds pseudo-random input to stress_test_linked_list, and runs all transitions both with and without CPU input (next to activation copy as input) * Fix RPC Proto patch due to GGML_OPS bump * Adds ggml-cpp.h treatment to test-sleep.cpp * Adds TODO pointers for subsequent additional tests. * Apply suggestion to ggml.h Co-authored-by: Johannes Gäßler <johannesg@5d6.de> * Apply suggestion to tests/test-backend-sched.cpp Co-authored-by: Johannes Gäßler <johannesg@5d6.de> * Apply suggestion to tests/test-backend-sched.cpp Co-authored-by: Johannes Gäßler <johannesg@5d6.de> * Fixes node compatibility check to not take buffer locations into account. This led to skipped tests due to incorrectly rejected ops due to wrong buffer locations in multi-GPU (which is one of the scheduler behaviors under test) * Apply batched suggestions from gg code review Co-authored-by: Georgi Gerganov <ggerganov@gmail.com> * Completes callsite change of gg-suggestion * Switch test-sleep from std::chrono::steady_clock::now() to ggml_time_us to avoid mac clock drift and spurious test failures. This makes the test a bit nonsensical on CPU (same clock checks itself), but the test still has merit on the other backends where independent device clocks are used. --------- Co-authored-by: Johannes Gäßler <johannesg@5d6.de> Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
155 lines
5.6 KiB
C++
155 lines
5.6 KiB
C++
// Tests for GGML_OP_SLEEP, on every backend that supports it. Three properties are checked:
|
|
// - the op only delays: input = output
|
|
// - the op does not return early: a chain of n sleeps must take at least n*us
|
|
// Only a lower bound is asserted for the duration. A busy-wait cannot finish early, so that bound is
|
|
// deterministic, whereas an upper bound would be flaky on a loaded machine.
|
|
// Known limitation: Vulkan does not specify the tick rate of the shader clock, see ggml_vk_sleep. A driver
|
|
// whose clock runs slower than assumed could undershoot the request and fail this bound.
|
|
|
|
#include "ggml.h"
|
|
#include "ggml-alloc.h"
|
|
#include "ggml-backend.h"
|
|
#include "ggml-cpp.h"
|
|
|
|
#include <cinttypes>
|
|
#include <cstdio>
|
|
#include <cstring>
|
|
#include <random>
|
|
#include <vector>
|
|
|
|
enum test_result {
|
|
TEST_OK,
|
|
TEST_FAIL,
|
|
TEST_SKIP,
|
|
};
|
|
|
|
static test_result test_sleep(ggml_backend_t backend, ggml_type type, int64_t ne, int32_t us, int n_nodes) {
|
|
const size_t graph_size = n_nodes + 16;
|
|
|
|
ggml_init_params params_static = {
|
|
/*.mem_size =*/ ggml_tensor_overhead(),
|
|
/*.mem_buffer =*/ nullptr,
|
|
/*.no_alloc =*/ true,
|
|
};
|
|
ggml_context_ptr ctx_static(ggml_init(params_static));
|
|
|
|
ggml_tensor * x = ggml_new_tensor_1d(ctx_static.get(), type, ne);
|
|
ggml_set_name(x, "x");
|
|
ggml_set_input(x);
|
|
|
|
ggml_init_params params_compute = {
|
|
/*.mem_size =*/ n_nodes*ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false),
|
|
/*.mem_buffer =*/ nullptr,
|
|
/*.no_alloc =*/ true,
|
|
};
|
|
ggml_context_ptr ctx_compute(ggml_init(params_compute));
|
|
ggml_cgraph * gf = ggml_new_graph_custom(ctx_compute.get(), graph_size, false);
|
|
|
|
ggml_tensor * out = x;
|
|
for (int i = 0; i < n_nodes; i++) {
|
|
out = ggml_sleep(ctx_compute.get(), out, us);
|
|
}
|
|
ggml_set_name(out, "out");
|
|
ggml_build_forward_expand(gf, out);
|
|
|
|
// the backends log to stdout while computing, so the result line is only printed once it is complete
|
|
char desc[128];
|
|
snprintf(desc, sizeof(desc), " %-6s %-4s ne=%-9" PRId64 " us=%-7d n_nodes=%d ",
|
|
ggml_backend_name(backend), ggml_type_name(type), ne, us, n_nodes);
|
|
|
|
test_result result = TEST_OK;
|
|
|
|
if (!ggml_backend_supports_op(backend, out)) {
|
|
printf("%snot supported\n", desc);
|
|
result = TEST_SKIP;
|
|
} else {
|
|
ggml_backend_buffer_ptr buf_static(ggml_backend_alloc_ctx_tensors(ctx_static.get(), backend));
|
|
|
|
// random bytes rather than random floats, so that bit patterns a value-wise copy could
|
|
// mangle, such as NaNs and denormals, are covered as well
|
|
std::vector<uint8_t> data_in(ggml_nbytes(x));
|
|
std::mt19937 rng(1234);
|
|
for (size_t i = 0; i < data_in.size(); i++) {
|
|
data_in[i] = rng() & 0xFF;
|
|
}
|
|
ggml_backend_tensor_set(x, data_in.data(), 0, data_in.size());
|
|
|
|
ggml_gallocr_ptr galloc(ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)));
|
|
ggml_gallocr_alloc_graph(galloc.get(), gf);
|
|
|
|
// warm up, so that one-time costs such as lazy CUDA module loading are not timed
|
|
ggml_backend_graph_compute(backend, gf);
|
|
|
|
const int64_t t_start_us = ggml_time_us();
|
|
const ggml_status status = ggml_backend_graph_compute(backend, gf);
|
|
const int64_t t_us = ggml_time_us() - t_start_us;
|
|
|
|
std::vector<uint8_t> data_out(ggml_nbytes(out));
|
|
ggml_backend_tensor_get(out, data_out.data(), 0, data_out.size());
|
|
|
|
const int64_t t_min_us = (int64_t) n_nodes * us;
|
|
|
|
const bool ok_status = status == GGML_STATUS_SUCCESS;
|
|
const bool ok_alias = out->data == x->data;
|
|
const bool ok_data = memcmp(data_in.data(), data_out.data(), data_in.size()) == 0;
|
|
const bool ok_time = t_us >= t_min_us;
|
|
|
|
printf("%selapsed=%7" PRId64 " us (>= %6" PRId64 ") alias: %-4s data: %-4s time: %s\n",
|
|
desc, t_us, t_min_us, ok_alias ? "OK" : "FAIL", ok_data ? "OK" : "FAIL", ok_time ? "OK" : "FAIL");
|
|
|
|
if (!ok_status) {
|
|
printf(" compute failed: %s\n", ggml_status_to_string(status));
|
|
}
|
|
|
|
result = ok_status && ok_alias && ok_data && ok_time ? TEST_OK : TEST_FAIL;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
int main() {
|
|
ggml_time_init();
|
|
ggml_backend_load_all();
|
|
|
|
int n_ok = 0;
|
|
int n_fail = 0;
|
|
int n_skip = 0;
|
|
|
|
for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
|
|
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
|
|
|
|
ggml_backend_ptr backend(ggml_backend_dev_init(dev, nullptr));
|
|
if (backend == nullptr) {
|
|
printf(" failed to initialize %s\n", ggml_backend_dev_name(dev));
|
|
n_fail++;
|
|
continue;
|
|
}
|
|
|
|
// a zero duration must still be a valid no-op, longer ones are checked against the clock,
|
|
// and the last case verifies that a chain of sleeps accumulates instead of collapsing into one
|
|
const struct {
|
|
ggml_type type;
|
|
int64_t ne;
|
|
int32_t us;
|
|
int n_nodes;
|
|
} cases[] = {
|
|
{ GGML_TYPE_F32, 1, 0, 1 },
|
|
{ GGML_TYPE_F32, 4096, 10000, 1 },
|
|
{ GGML_TYPE_F16, 1024, 10000, 1 },
|
|
{ GGML_TYPE_F32, 1 << 20, 5000, 4 },
|
|
};
|
|
|
|
for (const auto & c : cases) {
|
|
switch (test_sleep(backend.get(), c.type, c.ne, c.us, c.n_nodes)) {
|
|
case TEST_OK: n_ok++; break;
|
|
case TEST_FAIL: n_fail++; break;
|
|
case TEST_SKIP: n_skip++; break;
|
|
}
|
|
}
|
|
}
|
|
|
|
printf("%d passed, %d failed, %d skipped\n", n_ok, n_fail, n_skip);
|
|
|
|
return n_fail > 0;
|
|
}
|