* Adds initial scheduler stress tests * Adds new sleep op to simplify scheduler testing * Adds testcase addressing https://github.com/ggml-org/llama.cpp/issues/23321 * Adds experimental vulkan GGML_OP_SLEEP * Improves structure and naming of previous tests. Adds new tests, including one which detects the nkvo-bug correctly, and validates the proposed hotfix. * Cleanup: Removes CUDA specific comments and nvtx-markers * Fix: forgot to add sleep specific test case * Fix oversights identified by CI * Drops default prints so that it behaves like other ctests. Adds -v flag to toggle this behavior * Fix: Early exit in CPU-only settings (cause for graviton CI fails) * Improves stress_test_linked_list by removing overlap with test_chain_all_backends and test_pair_user_inputs. The latter ones test either activations or user inputs in isolation and just once, whereas the stress test tests all combinations multiple rounds * Exclude ACCEL devices from new tests due to incompatibility * Fixes vulkan test failure in llvmpipe * Adds asserts, improves comments and some checks * Fixes nitpicks in white-space and variable naming * Adds comments that certain vk drivers could time OP_SLEEP incorrectly * Drops limitiation to CPU/GPU in test-sleep * Aligns OP_SLEEP guards in cuda with vk guards (input shape and contiguity) * Implements feedback for OP_SLEEP to work in place (no copy) * Apply non-code suggestions from JohannesGaesslers code review Co-authored-by: Johannes Gäßler <johannesg@5d6.de> * Removes struct backend_supports; enables tests to return skip, instead of only pass/fail * Make clang-tidy happy * Removes sched_backend_caps, improves create_test_scheduler according to feedback (L149) * Removes "name" field from struct sched_check in favor of ggml_set_name/ggml_get_name * Drops unnecessary shader_clock.comp test and guards * Replaces sched_ prefixes with helper_ prefix * Moves test suite to new RAII types for better memory management * Adds sleep option to test_chain_all_backends * Adds sleep option to first-run arm of the y-shaped test * Fix switchcase order for legibility in stress_test_dag * Simplfies stress_test_linked_list via double for-loop * Adds pseudo-random input to stress_test_linked_list, and runs all transitions both with and without CPU input (next to activation copy as input) * Fix RPC Proto patch due to GGML_OPS bump * Adds ggml-cpp.h treatment to test-sleep.cpp * Adds TODO pointers for subsequent additional tests. * Apply suggestion to ggml.h Co-authored-by: Johannes Gäßler <johannesg@5d6.de> * Apply suggestion to tests/test-backend-sched.cpp Co-authored-by: Johannes Gäßler <johannesg@5d6.de> * Apply suggestion to tests/test-backend-sched.cpp Co-authored-by: Johannes Gäßler <johannesg@5d6.de> * Fixes node compatibility check to not take buffer locations into account. This led to skipped tests due to incorrectly rejected ops due to wrong buffer locations in multi-GPU (which is one of the scheduler behaviors under test) * Apply batched suggestions from gg code review Co-authored-by: Georgi Gerganov <ggerganov@gmail.com> * Completes callsite change of gg-suggestion * Switch test-sleep from std::chrono::steady_clock::now() to ggml_time_us to avoid mac clock drift and spurious test failures. This makes the test a bit nonsensical on CPU (same clock checks itself), but the test still has merit on the other backends where independent device clocks are used. --------- Co-authored-by: Johannes Gäßler <johannesg@5d6.de> Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
1085 lines
41 KiB
C++
1085 lines
41 KiB
C++
// Test suite for scheduler and backend behavior
|
|
|
|
// Towards formalized scheduling behavior:
|
|
// - a single inference pass can run on several backends. The subset of nodes running on a single backend is a split
|
|
// - synchronous backends:
|
|
// - explicit synchronization (ggml_backend_synchronize()) is required between each operation
|
|
// - asynchronous backends:
|
|
// - Activations from one split can be copied asynchronously to the next (output of split N -> input of split N+1)
|
|
// - Several scheduling patterns must be supported by async backends. The scheduler may:
|
|
// - not explicitly synchronize between CPU->backend memcpy and graph execution on backend
|
|
// - dispatch several parallel memcpys to the same backend at once
|
|
// - schedule splits without inputs at any point in the graph
|
|
|
|
#include "ggml.h"
|
|
#include "ggml-alloc.h"
|
|
#include "ggml-backend.h"
|
|
#include "ggml-cpp.h"
|
|
|
|
#include <algorithm>
|
|
#include <cinttypes>
|
|
#include <cstdarg>
|
|
#include <cstdio>
|
|
#include <cstring>
|
|
#include <random>
|
|
#include <vector>
|
|
|
|
|
|
|
|
enum test_result {
|
|
TEST_FAIL = 0,
|
|
TEST_OK = 1,
|
|
TEST_SKIP = 2, // in case a backend does not support a required OP (e.g. OP_SLEEP)
|
|
};
|
|
|
|
// global counters for pass/fail/skip
|
|
static int n_ok = 0;
|
|
static int n_skip = 0;
|
|
static int n_test = 0;
|
|
|
|
// default silent on success; --verbose / -v prints progress
|
|
static bool print_log = false;
|
|
|
|
GGML_ATTRIBUTE_FORMAT(1, 2)
|
|
static void log_maybe(const char * fmt, ...) {
|
|
if (!print_log) {
|
|
return;
|
|
}
|
|
|
|
va_list args;
|
|
va_start(args, fmt);
|
|
vprintf(fmt, args);
|
|
va_end(args);
|
|
}
|
|
|
|
// pretty-print helpers start
|
|
static constexpr int status_column = 82; // for ok/fail column alignment
|
|
static int case_len = 0;
|
|
static char case_label[256]; // max length of a case label
|
|
|
|
static void case_end(test_result res) {
|
|
n_ok += res == TEST_OK;
|
|
n_skip += res == TEST_SKIP;
|
|
n_test++;
|
|
|
|
if (res != TEST_FAIL && !print_log) {
|
|
case_len = 0;
|
|
return;
|
|
}
|
|
|
|
if (case_len == 0) {
|
|
case_len = printf(" %s", case_label);
|
|
}
|
|
|
|
const char * status = "FAIL";
|
|
if (res == TEST_OK) {
|
|
status = "OK";
|
|
} else if (res == TEST_SKIP) {
|
|
status = "SKIP";
|
|
}
|
|
|
|
printf("%*s%s\n", std::max(1, status_column - case_len), "", status);
|
|
case_len = 0;
|
|
}
|
|
|
|
static void vnote(const char * fmt, va_list args) {
|
|
if (case_len != 0) {
|
|
printf("\n");
|
|
}
|
|
printf(" ");
|
|
vprintf(fmt, args);
|
|
printf("\n");
|
|
|
|
case_len = 0;
|
|
}
|
|
|
|
GGML_ATTRIBUTE_FORMAT(1, 2)
|
|
static void note(const char * fmt, ...) {
|
|
if (!print_log) {
|
|
return;
|
|
}
|
|
|
|
va_list args;
|
|
va_start(args, fmt);
|
|
vnote(fmt, args);
|
|
va_end(args);
|
|
}
|
|
|
|
GGML_ATTRIBUTE_FORMAT(1, 2)
|
|
static test_result fail(const char * fmt, ...) {
|
|
if (case_len == 0) {
|
|
case_len = printf(" %s", case_label);
|
|
}
|
|
|
|
va_list args;
|
|
va_start(args, fmt);
|
|
vnote(fmt, args);
|
|
va_end(args);
|
|
|
|
return TEST_FAIL;
|
|
}
|
|
|
|
GGML_ATTRIBUTE_FORMAT(1, 2)
|
|
static test_result skip(const char * fmt, ...) {
|
|
if (print_log) {
|
|
va_list args;
|
|
va_start(args, fmt);
|
|
vnote(fmt, args);
|
|
va_end(args);
|
|
}
|
|
|
|
return TEST_SKIP;
|
|
}
|
|
|
|
GGML_ATTRIBUTE_FORMAT(1, 2)
|
|
static void case_begin(const char * fmt, ...) {
|
|
va_list args;
|
|
va_start(args, fmt);
|
|
vsnprintf(case_label, sizeof(case_label), fmt, args);
|
|
va_end(args);
|
|
|
|
if (print_log) {
|
|
case_len = printf(" %s", case_label);
|
|
fflush(stdout);
|
|
} else {
|
|
case_len = 0;
|
|
}
|
|
}
|
|
// pretty-print helpers end
|
|
|
|
|
|
// NB: the last backend passed has to be the CPU backend (cf. ggml_backend_sched_new)
|
|
static ggml_backend_sched_ptr create_test_scheduler(std::vector<ggml_backend_t> backends, size_t graph_size,
|
|
bool use_device_host_buft, bool parallel = false) {
|
|
|
|
std::vector<ggml_backend_buffer_type_t> bufts(backends.size());
|
|
for (size_t b = 0; b < backends.size(); b++) {
|
|
bufts[b] = ggml_backend_get_default_buffer_type(backends[b]);
|
|
}
|
|
|
|
// pinned memory for the CPU backend (always required to be the last backend) enables async copies to other devices
|
|
// the host buffer type comes from the first backend, (cf. llama_context::llama_context())
|
|
if (use_device_host_buft) {
|
|
ggml_backend_buffer_type_t host = ggml_backend_dev_host_buffer_type(ggml_backend_get_device(backends[0]));
|
|
if (host != nullptr) {
|
|
bufts.back() = host;
|
|
}
|
|
}
|
|
|
|
return ggml_backend_sched_ptr(ggml_backend_sched_new(backends.data(), bufts.data(), (int) backends.size(),
|
|
graph_size, parallel, /*op_offload =*/ false));
|
|
}
|
|
|
|
// the nodes of the graph together with the backend each of them is assigned to
|
|
struct helper_graph {
|
|
ggml_cgraph * gf;
|
|
std::vector<ggml_tensor *> nodes;
|
|
std::vector<int> backend_id;
|
|
|
|
helper_graph(ggml_context * ctx, size_t graph_size) : gf(ggml_new_graph_custom(ctx, graph_size, /*grads =*/ false)) {}
|
|
|
|
ggml_tensor * add(ggml_tensor * node, int backend) {
|
|
nodes.push_back(node);
|
|
backend_id.push_back(backend);
|
|
ggml_build_forward_expand(gf, node);
|
|
|
|
return node;
|
|
}
|
|
};
|
|
|
|
struct helper_check {
|
|
ggml_tensor * tensor; // Tensor for which to check the data.
|
|
float expected; // The expected value that the tensor data should have after graph execution.
|
|
};
|
|
|
|
// asks whether the backend implements the op, independent of where the sources of this node currently live.
|
|
static bool backend_supports_op(ggml_backend_t backend, const ggml_tensor * node) {
|
|
ggml_init_params params = {
|
|
/*.mem_size =*/ GGML_MAX_SRC*ggml_tensor_overhead(),
|
|
/*.mem_buffer =*/ nullptr,
|
|
/*.no_alloc =*/ true,
|
|
};
|
|
ggml_context_ptr ctx(ggml_init(params));
|
|
|
|
ggml_tensor probe = *node;
|
|
probe.buffer = nullptr;
|
|
probe.view_src = nullptr;
|
|
probe.data = nullptr;
|
|
|
|
for (ggml_tensor *& src : probe.src) {
|
|
if (src != nullptr) {
|
|
src = ggml_dup_tensor(ctx.get(), src);
|
|
}
|
|
}
|
|
|
|
return ggml_backend_supports_op(backend, &probe);
|
|
}
|
|
|
|
// executes the graph, then checks the number of splits and the results
|
|
static test_result run_and_check(ggml_backend_sched_t sched, const helper_graph & g,
|
|
const std::vector<ggml_backend_t> & backends, const std::vector<helper_check> & checks, int64_t ne) {
|
|
|
|
// ggml_backend_sched_set_tensor_backend does not validate the op, an unsupported one would abort inside the backend
|
|
for (size_t i = 0; i < g.nodes.size(); i++) {
|
|
ggml_backend_t backend = backends[g.backend_id[i]];
|
|
if (!backend_supports_op(backend, g.nodes[i])) {
|
|
return skip("%s does not support %s", ggml_backend_name(backend), ggml_op_name(g.nodes[i]->op));
|
|
}
|
|
}
|
|
|
|
// to later check if any further graph logic alters the number of splits
|
|
int n_splits_expected = g.nodes.empty() ? 0 : 1;
|
|
for (size_t i = 1; i < g.backend_id.size(); i++) {
|
|
n_splits_expected += g.backend_id[i] != g.backend_id[i - 1];
|
|
}
|
|
|
|
ggml_backend_sched_reset(sched);
|
|
for (size_t i = 0; i < g.nodes.size(); i++) {
|
|
ggml_backend_sched_set_tensor_backend(sched, g.nodes[i], backends[g.backend_id[i]]);
|
|
}
|
|
|
|
if (!ggml_backend_sched_alloc_graph(sched, g.gf)) {
|
|
return fail("failed to allocate the graph");
|
|
}
|
|
|
|
if (ggml_backend_sched_graph_compute(sched, g.gf) != GGML_STATUS_SUCCESS) {
|
|
return fail("failed to compute the graph");
|
|
}
|
|
|
|
const int n_splits = ggml_backend_sched_get_n_splits(sched);
|
|
if (n_splits != n_splits_expected) {
|
|
return fail("n_splits = %d, expected %d, each split consisting of just a single node", n_splits, n_splits_expected);
|
|
}
|
|
|
|
std::vector<float> data(ne);
|
|
for (const helper_check & c : checks) {
|
|
ggml_backend_tensor_get(c.tensor, data.data(), 0, ggml_nbytes(c.tensor));
|
|
for (int64_t i = 0; i < ne; i++) {
|
|
if (data[i] != c.expected) {
|
|
return fail("%s[%" PRId64 "] = %f, expected %f", ggml_get_name(c.tensor), i, data[i], c.expected);
|
|
}
|
|
}
|
|
}
|
|
|
|
return TEST_OK;
|
|
}
|
|
|
|
// helper struct for inputless splits, and constants like 0 and 1
|
|
struct backend_consts {
|
|
// the buffers are declared after the contexts, so that they are released first
|
|
std::vector<ggml_context_ptr> ctxs;
|
|
std::vector<ggml_backend_buffer_ptr> bufs;
|
|
std::vector<ggml_tensor *> zero;
|
|
std::vector<ggml_tensor *> one;
|
|
|
|
backend_consts(const std::vector<ggml_backend_t> & backends, int64_t ne) {
|
|
std::vector<float> data(ne);
|
|
|
|
for (size_t b = 0; b < backends.size(); b++) {
|
|
ggml_init_params params = {
|
|
/*.mem_size =*/ 2*ggml_tensor_overhead(),
|
|
/*.mem_buffer =*/ nullptr,
|
|
/*.no_alloc =*/ true,
|
|
};
|
|
ggml_context * ctx = ggml_init(params);
|
|
|
|
ggml_tensor * z = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne);
|
|
ggml_format_name(z, "zero_%s", ggml_backend_name(backends[b]));
|
|
|
|
ggml_tensor * o = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne);
|
|
ggml_format_name(o, "one_%s", ggml_backend_name(backends[b]));
|
|
|
|
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backends[b]);
|
|
GGML_ASSERT(buf != nullptr);
|
|
|
|
std::fill(data.begin(), data.end(), 0.0f);
|
|
ggml_backend_tensor_set(z, data.data(), 0, ggml_nbytes(z));
|
|
|
|
std::fill(data.begin(), data.end(), 1.0f);
|
|
ggml_backend_tensor_set(o, data.data(), 0, ggml_nbytes(o));
|
|
|
|
ctxs.emplace_back(ctx);
|
|
bufs.emplace_back(buf);
|
|
zero.push_back(z);
|
|
one.push_back(o);
|
|
}
|
|
}
|
|
};
|
|
|
|
// TODO add tests for the auto-assign logic of nodes to backends of the scheduler. Possible tests:
|
|
// - test of CPU backend with any other backend. Expectation is that GEMM-nodes (usually the first accelerated components) are assigned to the non-CPU backend
|
|
// - test of the property that tensor location influences backend assignment.
|
|
// - test of neighbor propagation logic? 3 nodes, with the middle one initially on CPU, moved to the other backend?
|
|
|
|
|
|
// stress test: a tensor is incremented and sent back and forth between the backends in a ping-pong pattern.
|
|
// One pass pairs every backend a with every other backend b via nested loops.
|
|
// The counter is brought to a, then bounced to b and back for every partner in turn.
|
|
// So one pass copies an activation across every ordered pair of backends.
|
|
// Every lap runs two passes: one where the nodes add the constant 1, one where they add a user input
|
|
// from the CPU backend, so that every transition is tested with and without an input next to the activation.
|
|
// The input values come from a fixed seed.
|
|
static test_result stress_test_linked_list(const std::vector<ggml_backend_t> & backends, int n_laps,
|
|
int64_t tensor_len, bool use_device_host_buft) {
|
|
const int n_backends = (int) backends.size();
|
|
|
|
// per pass and backend a: one node to bring the counter to a, then two per partner for the ping-pong
|
|
const int n_nodes_pass = n_backends*(2*n_backends - 1);
|
|
const int n_nodes = 2*n_laps*n_nodes_pass;
|
|
const int n_inputs = n_laps*n_nodes_pass; // one per node of the pass that takes user inputs
|
|
|
|
// initial pseudo-random value draw so that the expected result is known before graph execution
|
|
std::mt19937 rng(42);
|
|
std::uniform_int_distribution<int> draw_value(1, 10000);
|
|
|
|
std::vector<int> input_value(n_inputs); // random values store
|
|
int64_t sum_inputs = 0;
|
|
|
|
for (int i = 0; i < n_inputs; i++) {
|
|
input_value[i] = draw_value(rng);
|
|
sum_inputs += input_value[i];
|
|
}
|
|
|
|
// see sched->hash_set FIXME, the set has to hold the nodes and the leafs: two constants per backend
|
|
// plus the user inputs
|
|
const size_t graph_size = n_nodes + 2*n_backends + n_inputs;
|
|
|
|
backend_consts consts(backends, tensor_len);
|
|
|
|
ggml_backend_sched_ptr sched = create_test_scheduler(backends, graph_size, use_device_host_buft);
|
|
|
|
// the user inputs all live on the CPU backend, with the same buffer type the scheduler uses for it
|
|
ggml_init_params params_input = {
|
|
/*.mem_size =*/ n_inputs*ggml_tensor_overhead(),
|
|
/*.mem_buffer =*/ nullptr,
|
|
/*.no_alloc =*/ true,
|
|
};
|
|
ggml_context_ptr ctx_input(ggml_init(params_input));
|
|
ggml_backend_buffer_ptr buf_input;
|
|
|
|
std::vector<ggml_tensor *> input(n_inputs);
|
|
for (int i = 0; i < n_inputs; i++) {
|
|
input[i] = ggml_new_tensor_1d(ctx_input.get(), GGML_TYPE_F32, tensor_len);
|
|
ggml_format_name(input[i], "input%d", i);
|
|
ggml_set_input(input[i]);
|
|
}
|
|
|
|
// the inputs follow the buffer type the scheduler uses for the CPU backend (cf. create_test_scheduler)
|
|
ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backends.back());
|
|
if (use_device_host_buft) {
|
|
ggml_backend_buffer_type_t host = ggml_backend_dev_host_buffer_type(ggml_backend_get_device(backends[0]));
|
|
if (host != nullptr) {
|
|
buft = host;
|
|
}
|
|
}
|
|
|
|
buf_input.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_input.get(), buft));
|
|
GGML_ASSERT(buf_input != nullptr);
|
|
|
|
// write inputs before graph execution
|
|
std::vector<float> data(tensor_len);
|
|
for (int i = 0; i < n_inputs; i++) {
|
|
std::fill(data.begin(), data.end(), float(input_value[i]));
|
|
ggml_backend_tensor_set(input[i], data.data(), 0, ggml_nbytes(input[i]));
|
|
}
|
|
|
|
ggml_init_params params_compute = {
|
|
/*.mem_size =*/ (n_nodes + 2)*ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false),
|
|
/*.mem_buffer =*/ nullptr,
|
|
/*.no_alloc =*/ true,
|
|
};
|
|
ggml_context_ptr ctx_compute(ggml_init(params_compute));
|
|
|
|
helper_graph g(ctx_compute.get(), graph_size);
|
|
|
|
int64_t sum = 0;
|
|
ggml_tensor * out = nullptr;
|
|
|
|
int i = 0; // next user input to consume
|
|
|
|
// increments the counter on backend b, which makes the scheduler copy it there from the previous backend
|
|
auto step = [&](int b, bool with_input) {
|
|
ggml_tensor * src = out == nullptr ? consts.zero[b] : out;
|
|
|
|
if (with_input) {
|
|
out = g.add(ggml_add(ctx_compute.get(), src, input[i]), b);
|
|
i++;
|
|
} else {
|
|
out = g.add(ggml_add(ctx_compute.get(), src, consts.one[b]), b);
|
|
sum += 1;
|
|
}
|
|
};
|
|
|
|
for (int lap = 0; lap < n_laps; lap++) {
|
|
for (bool with_input : { false, true }) { // test each transition with and without input
|
|
for (int a = 0; a < n_backends; a++) {
|
|
step(a, with_input);
|
|
|
|
for (int b = 0; b < n_backends; b++) {
|
|
if (b == a) {
|
|
continue;
|
|
}
|
|
|
|
step(b, with_input);
|
|
step(a, with_input);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
ggml_set_name(out, "out");
|
|
ggml_set_output(out);
|
|
|
|
// every node adds either the constant 1 or its drawn user input
|
|
const int64_t expected = sum + sum_inputs;
|
|
|
|
note("%d of %d nodes take a user input, they contribute %" PRId64 " of %" PRId64, n_inputs, n_nodes, sum_inputs, expected);
|
|
|
|
test_result ok;
|
|
if (expected >= (1 << 24)) { // the counter has to stay exact in f32
|
|
ok = fail("expected result %" PRId64 " does not fit into the f32 mantissa, lower n_laps", expected);
|
|
} else {
|
|
ok = run_and_check(sched.get(), g, backends, {{ out, float(expected) }}, tensor_len);
|
|
}
|
|
|
|
return ok;
|
|
}
|
|
|
|
// Same idea as stress_test_linked_list, but with the compute graph being a directed acyclic graph (DAG).
|
|
// n_lanes independent counters run through rounds of four shapes, so that one graph covers:
|
|
// - lanes with different histories, so a misrouted copy shows up as a wrong count in a single lane
|
|
// - one split producing two values followed by two splits that each consume one of them and do not depend on each other
|
|
// - splits with no inputs at all, from lanes re-seeded out of constants that already live on the split's backend
|
|
// - splits taking one activation per lane from the previous split, one of them from an older split instead
|
|
static test_result stress_test_dag(const std::vector<ggml_backend_t> & backends, int n_lanes, int n_rounds,
|
|
int64_t tensor_len, bool use_device_host_buft) {
|
|
|
|
const int n_backends = (int) backends.size();
|
|
|
|
GGML_ASSERT(n_lanes % 2 == 0);
|
|
|
|
// sized for the worst case
|
|
const size_t graph_size = n_lanes*(2*n_rounds + 1) + 2*n_backends;
|
|
|
|
backend_consts consts(backends, tensor_len);
|
|
|
|
ggml_backend_sched_ptr sched = create_test_scheduler(backends, graph_size, use_device_host_buft);
|
|
|
|
ggml_init_params params_compute = {
|
|
/*.mem_size =*/ (graph_size + 8)*ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false),
|
|
/*.mem_buffer =*/ nullptr,
|
|
/*.no_alloc =*/ true,
|
|
};
|
|
ggml_context_ptr ctx_compute(ggml_init(params_compute));
|
|
|
|
helper_graph g(ctx_compute.get(), graph_size);
|
|
|
|
std::vector<ggml_tensor *> lane(n_lanes);
|
|
std::vector<int64_t> value(n_lanes);
|
|
std::vector<helper_check> checks;
|
|
|
|
// every lane starts at a different value, so that a misrouted copy changes a sum instead of going unnoticed
|
|
auto seed = [&](int b, const char * prefix) {
|
|
for (int l = 0; l < n_lanes; l++) {
|
|
lane[l] = g.add(ggml_add(ctx_compute.get(), l == 0 ? consts.zero[b] : lane[l - 1], consts.one[b]), b);
|
|
value[l] = l + 1;
|
|
ggml_format_name(lane[l], "%s%d", prefix, l);
|
|
}
|
|
};
|
|
|
|
auto check_lanes = [&]() {
|
|
for (int l = 0; l < n_lanes; l++) {
|
|
ggml_set_output(lane[l]);
|
|
checks.push_back({ lane[l], float(value[l]) });
|
|
}
|
|
};
|
|
|
|
seed(0, "seed");
|
|
|
|
std::vector<ggml_tensor *> older = lane;
|
|
std::vector<int64_t> value_older = value;
|
|
|
|
for (int round = 0; round < n_rounds; round++) {
|
|
// a backend the previous node did not use, so that the round is not absorbed into the split before it
|
|
const int b = (g.backend_id.back() + 1) % n_backends;
|
|
|
|
const std::vector<ggml_tensor *> prev = lane;
|
|
const std::vector<int64_t> value_prev = value;
|
|
|
|
switch (round % 5) {
|
|
case 0:
|
|
for (int p = 0; p < n_lanes/2; p++) {
|
|
const int la = 2*p;
|
|
const int lb = 2*p + 1;
|
|
const int prod = (b + p) % n_backends;
|
|
const int cons = (prod + 1) % n_backends;
|
|
|
|
// one split producing two values ...
|
|
for (int l : { la, lb }) {
|
|
lane[l] = g.add(ggml_add(ctx_compute.get(), lane[l], consts.one[prod]), prod);
|
|
value[l]++;
|
|
ggml_format_name(lane[l], "r%d_prod%d", round, l);
|
|
}
|
|
|
|
// ... and two splits that each consume one of them, with no dependency on each other
|
|
lane[la] = g.add(ggml_add(ctx_compute.get(), lane[la], consts.one[cons]), cons);
|
|
value[la]++;
|
|
ggml_format_name(lane[la], "r%d_cons%d", round, la);
|
|
|
|
lane[lb] = g.add(ggml_add(ctx_compute.get(), lane[lb], consts.one[prod]), prod);
|
|
value[lb]++;
|
|
ggml_format_name(lane[lb], "r%d_cons%d", round, lb);
|
|
}
|
|
break;
|
|
case 1:
|
|
check_lanes();
|
|
seed(b, "restart");
|
|
break;
|
|
case 2:
|
|
case 3:
|
|
for (int l = 0; l < n_lanes; l++) {
|
|
const int partner = (l + 1 + round) % n_lanes;
|
|
const bool reach_back = round >= 2 && l == round % n_lanes;
|
|
|
|
ggml_tensor * a = reach_back ? older[l] : prev[l];
|
|
const int64_t va = reach_back ? value_older[l] : value_prev[l];
|
|
|
|
lane[l] = g.add(ggml_add(ctx_compute.get(), a, prev[partner]), b);
|
|
value[l] = va + value_prev[partner];
|
|
ggml_format_name(lane[l], "r%d_lane%d", round, l);
|
|
}
|
|
break;
|
|
case 4:
|
|
lane[0] = g.add(ggml_add(ctx_compute.get(), prev[0], consts.one[b]), b);
|
|
value[0] = value_prev[0] + 1;
|
|
ggml_format_name(lane[0], "r%d_single", round);
|
|
break;
|
|
default:
|
|
GGML_ABORT("unexpected round %d", round);
|
|
}
|
|
|
|
older = prev;
|
|
value_older = value_prev;
|
|
}
|
|
|
|
check_lanes();
|
|
|
|
test_result ok = TEST_OK;
|
|
for (const helper_check & c : checks) {
|
|
if (c.expected >= float(1 << 24)) { // the counts have to stay exact in f32
|
|
ok = fail("expected result %.0f does not fit into the f32 mantissa, lower n_rounds", c.expected);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (ok == TEST_OK) {
|
|
ok = run_and_check(sched.get(), g, backends, checks, tensor_len);
|
|
}
|
|
|
|
return ok;
|
|
}
|
|
|
|
// Created in response to https://github.com/ggml-org/llama.cpp/issues/23321
|
|
// Currently, input copies to splits synchronize split execution.
|
|
// CPU split N+1 could overwrite output of split N if this was not copied to backend in time.
|
|
// Mechanism to test: GPU split with long sleep to clog CUDA stream / vk command queue,
|
|
// followed by CPU splits without input, provoking a race condition in the current setup.
|
|
// GPU: 100ms sleep to clog stream/queue
|
|
// CPU: produce {55}
|
|
// GPU: request copy of {55} in CUDA stream, ADD {44}; Output should be {99}
|
|
// CPU: produce {66}
|
|
// GPU: increment both {99} and {66} -> {100} and {67}
|
|
// Correct result is thus {100}, incorrect output is {111} when {55} was overwritten by {66} before the copy started.
|
|
// Note: currently only reproducible on async H2D copy (=pinned memory).
|
|
static test_result test_inputless_splits_scheduling(ggml_backend_t gpu, ggml_backend_t cpu, int64_t tensor_len, int32_t sleep_us, bool use_device_host_buft) {
|
|
|
|
const int GPU = 0;
|
|
const int CPU = 1;
|
|
|
|
const std::vector<ggml_backend_t> backends = { gpu, cpu };
|
|
|
|
const size_t graph_size = 64;
|
|
|
|
ggml_backend_sched_ptr sched = create_test_scheduler(backends, graph_size, use_device_host_buft);
|
|
|
|
ggml_init_params params_static = {
|
|
/*.mem_size =*/ 4*ggml_tensor_overhead(),
|
|
/*.mem_buffer =*/ nullptr,
|
|
/*.no_alloc =*/ true,
|
|
};
|
|
|
|
ggml_context_ptr ctx_cpu(ggml_init(params_static));
|
|
|
|
ggml_tensor * zero_cpu = ggml_new_tensor_1d(ctx_cpu.get(), GGML_TYPE_F32, tensor_len);
|
|
ggml_set_name(zero_cpu, "zero_cpu");
|
|
|
|
ggml_tensor * val55_cpu = ggml_new_tensor_1d(ctx_cpu.get(), GGML_TYPE_F32, tensor_len);
|
|
ggml_set_name(val55_cpu, "val55_cpu");
|
|
|
|
ggml_tensor * val66_cpu = ggml_new_tensor_1d(ctx_cpu.get(), GGML_TYPE_F32, tensor_len);
|
|
ggml_set_name(val66_cpu, "val66_cpu");
|
|
|
|
ggml_backend_buffer_ptr buf_cpu(ggml_backend_alloc_ctx_tensors(ctx_cpu.get(), cpu));
|
|
GGML_ASSERT(buf_cpu != nullptr);
|
|
|
|
ggml_context_ptr ctx_gpu(ggml_init(params_static));
|
|
|
|
ggml_tensor * zero_gpu = ggml_new_tensor_1d(ctx_gpu.get(), GGML_TYPE_F32, tensor_len);
|
|
ggml_set_name(zero_gpu, "zero_gpu");
|
|
|
|
ggml_tensor * one_gpu = ggml_new_tensor_1d(ctx_gpu.get(), GGML_TYPE_F32, tensor_len);
|
|
ggml_set_name(one_gpu, "one_gpu");
|
|
|
|
ggml_tensor * val44_gpu = ggml_new_tensor_1d(ctx_gpu.get(), GGML_TYPE_F32, tensor_len);
|
|
ggml_set_name(val44_gpu, "val44_gpu");
|
|
|
|
ggml_backend_buffer_ptr buf_gpu(ggml_backend_alloc_ctx_tensors(ctx_gpu.get(), gpu));
|
|
GGML_ASSERT(buf_gpu != nullptr);
|
|
|
|
std::vector<float> data(tensor_len);
|
|
|
|
std::fill(data.begin(), data.end(), 0.0f);
|
|
ggml_backend_tensor_set(zero_cpu, data.data(), 0, ggml_nbytes(zero_cpu));
|
|
ggml_backend_tensor_set(zero_gpu, data.data(), 0, ggml_nbytes(zero_gpu));
|
|
|
|
std::fill(data.begin(), data.end(), 1.0f);
|
|
ggml_backend_tensor_set(one_gpu, data.data(), 0, ggml_nbytes(one_gpu));
|
|
|
|
std::fill(data.begin(), data.end(), 44.0f);
|
|
ggml_backend_tensor_set(val44_gpu, data.data(), 0, ggml_nbytes(val44_gpu));
|
|
|
|
std::fill(data.begin(), data.end(), 55.0f);
|
|
ggml_backend_tensor_set(val55_cpu, data.data(), 0, ggml_nbytes(val55_cpu));
|
|
|
|
std::fill(data.begin(), data.end(), 66.0f);
|
|
ggml_backend_tensor_set(val66_cpu, data.data(), 0, ggml_nbytes(val66_cpu));
|
|
|
|
ggml_init_params params_compute = {
|
|
/*.mem_size =*/ (graph_size + 8)*ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false),
|
|
/*.mem_buffer =*/ nullptr,
|
|
/*.no_alloc =*/ true,
|
|
};
|
|
ggml_context_ptr ctx_compute(ggml_init(params_compute));
|
|
|
|
helper_graph g(ctx_compute.get(), graph_size);
|
|
|
|
// GPU: add then sleep - occupies the GPU while later splits run
|
|
ggml_tensor * delayed = g.add(ggml_add(ctx_compute.get(), zero_gpu, one_gpu), GPU);
|
|
ggml_set_name(delayed, "to_delay");
|
|
delayed = g.add(ggml_sleep(ctx_compute.get(), delayed, sleep_us), GPU);
|
|
ggml_set_name(delayed, "delayed");
|
|
|
|
// CPU: 0 + 55 -> 55
|
|
ggml_tensor * v55 = g.add(ggml_add(ctx_compute.get(), zero_cpu, val55_cpu), CPU);
|
|
ggml_set_name(v55, "v55");
|
|
|
|
// GPU: 55 + 44 -> 99
|
|
ggml_tensor * v99 = g.add(ggml_add(ctx_compute.get(), v55, val44_gpu), GPU);
|
|
ggml_set_name(v99, "v99");
|
|
|
|
// CPU: 0 + 66 -> 66
|
|
ggml_tensor * v66 = g.add(ggml_add(ctx_compute.get(), zero_cpu, val66_cpu), CPU);
|
|
ggml_set_name(v66, "v66");
|
|
|
|
// GPU: increment both of the previous split outputs
|
|
ggml_tensor * out99 = g.add(ggml_add(ctx_compute.get(), v99, one_gpu), GPU);
|
|
ggml_set_name(out99, "out99");
|
|
|
|
ggml_tensor * out66 = g.add(ggml_add(ctx_compute.get(), v66, one_gpu), GPU);
|
|
ggml_set_name(out66, "out66");
|
|
|
|
ggml_set_output(delayed);
|
|
ggml_set_output(out99);
|
|
ggml_set_output(out66);
|
|
|
|
const test_result ok = run_and_check(sched.get(), g, backends, {
|
|
{ delayed, 1.0f },
|
|
{ out99, 100.0f },
|
|
{ out66, 67.0f },
|
|
}, tensor_len);
|
|
|
|
// the race needs the allocator to reuse the memory of the first CPU split for the second one
|
|
note("v55 and v66 %s (v55=%p v66=%p)", v55->data == v66->data ? "alias" : "do not alias", v55->data, v66->data);
|
|
|
|
return ok;
|
|
}
|
|
|
|
// Test that all async backends transmit their activations to the following async backend correctly. No user inputs tested.
|
|
static test_result test_chain_all_backends(const std::vector<ggml_backend_t> & backends, int64_t tensor_len, int32_t sleep_us,
|
|
bool use_device_host_buft) {
|
|
|
|
const int n_backends = (int) backends.size();
|
|
|
|
std::vector<int> seq;
|
|
|
|
auto push = [&](int b) {
|
|
if (seq.empty() || seq.back() != b) {
|
|
seq.push_back(b);
|
|
}
|
|
};
|
|
|
|
for (int i = 0; i < n_backends; i++) {
|
|
for (int j = 0; j < n_backends; j++) {
|
|
if (i != j) {
|
|
push(i);
|
|
push(j);
|
|
}
|
|
}
|
|
}
|
|
|
|
const size_t n_nodes = seq.size();
|
|
const size_t n_sleeps = sleep_us > 0 ? n_nodes : 0; // one sleep per split
|
|
const size_t graph_size = n_nodes + n_sleeps + n_backends + 1; // see sched->hash_set FIXME, one "one" per backend plus the "zero" that starts the chain
|
|
|
|
backend_consts consts(backends, tensor_len);
|
|
|
|
ggml_backend_sched_ptr sched = create_test_scheduler(backends, graph_size, use_device_host_buft);
|
|
|
|
ggml_init_params params_compute = {
|
|
/*.mem_size =*/ (graph_size + 8)*ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false),
|
|
/*.mem_buffer =*/ nullptr,
|
|
/*.no_alloc =*/ true,
|
|
};
|
|
ggml_context_ptr ctx_compute(ggml_init(params_compute));
|
|
|
|
helper_graph g(ctx_compute.get(), graph_size);
|
|
|
|
ggml_tensor * out = nullptr;
|
|
for (size_t k = 0; k < n_nodes; k++) {
|
|
const int b = seq[k];
|
|
|
|
out = g.add(ggml_add(ctx_compute.get(), k == 0 ? consts.zero[b] : out, consts.one[b]), b);
|
|
|
|
// the sleep keeps the backend busy, so that the next one has to wait for this split
|
|
if (sleep_us > 0) {
|
|
out = g.add(ggml_sleep(ctx_compute.get(), out, sleep_us), b);
|
|
}
|
|
}
|
|
ggml_set_name(out, "out");
|
|
ggml_set_output(out);
|
|
|
|
const test_result ok = run_and_check(sched.get(), g, backends, {{ out, float(n_nodes) }}, tensor_len);
|
|
|
|
return ok;
|
|
}
|
|
|
|
// Tests data transfer between all combinations of backend pairs
|
|
// The receiving split takes one activation from the sender plus n_inputs user inputs
|
|
static test_result test_pair_user_inputs(const std::vector<ggml_backend_t> & backends, int b_send, int b_recv, int64_t tensor_len,
|
|
int n_inputs, bool inputs_on_sender, bool parallel, bool use_device_host_buft) {
|
|
|
|
const size_t graph_size = 64;
|
|
|
|
backend_consts consts(backends, tensor_len);
|
|
|
|
ggml_backend_sched_ptr sched = create_test_scheduler(backends, graph_size, use_device_host_buft, parallel);
|
|
|
|
// placing the inputs on the sender makes the receiving split copy all of them, placing them on the
|
|
// receiver leaves the activation as its only input
|
|
const int b_inputs = inputs_on_sender ? b_send : b_recv;
|
|
|
|
ggml_init_params params_inputs = {
|
|
/*.mem_size =*/ n_inputs*ggml_tensor_overhead(),
|
|
/*.mem_buffer =*/ nullptr,
|
|
/*.no_alloc =*/ true,
|
|
};
|
|
ggml_context_ptr ctx_inputs(ggml_init(params_inputs));
|
|
|
|
std::vector<ggml_tensor *> inputs;
|
|
for (int k = 0; k < n_inputs; k++) {
|
|
ggml_tensor * in = ggml_new_tensor_1d(ctx_inputs.get(), GGML_TYPE_F32, tensor_len);
|
|
ggml_format_name(in, "in%d", k);
|
|
ggml_set_input(in);
|
|
inputs.push_back(in);
|
|
}
|
|
|
|
ggml_backend_buffer_ptr buf_inputs(ggml_backend_alloc_ctx_tensors(ctx_inputs.get(), backends[b_inputs]));
|
|
GGML_ASSERT(buf_inputs != nullptr);
|
|
|
|
std::vector<float> data(tensor_len);
|
|
std::fill(data.begin(), data.end(), 1.0f);
|
|
for (ggml_tensor * in : inputs) {
|
|
ggml_backend_tensor_set(in, data.data(), 0, ggml_nbytes(in));
|
|
}
|
|
|
|
ggml_init_params params_compute = {
|
|
/*.mem_size =*/ (graph_size + 8)*ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false),
|
|
/*.mem_buffer =*/ nullptr,
|
|
/*.no_alloc =*/ true,
|
|
};
|
|
ggml_context_ptr ctx_compute(ggml_init(params_compute));
|
|
|
|
helper_graph g(ctx_compute.get(), graph_size);
|
|
|
|
ggml_tensor * out = g.add(ggml_add(ctx_compute.get(), consts.zero[b_send], consts.one[b_send]), b_send);
|
|
|
|
for (ggml_tensor * in : inputs) {
|
|
out = g.add(ggml_add(ctx_compute.get(), out, in), b_recv);
|
|
}
|
|
ggml_set_name(out, "out");
|
|
ggml_set_output(out);
|
|
|
|
test_result ok = run_and_check(sched.get(), g, backends, {{ out, float(1 + n_inputs) }}, tensor_len);
|
|
|
|
for (ggml_tensor * in : inputs) {
|
|
if (ok != TEST_OK) {
|
|
break;
|
|
}
|
|
ggml_backend_tensor_get(in, data.data(), 0, ggml_nbytes(in));
|
|
for (int64_t i = 0; i < tensor_len; i++) {
|
|
if (data[i] != 1.0f) {
|
|
ok = fail("%s[%" PRId64 "] = %f after compute, expected 1.000000 - the input was modified", ggml_get_name(in), i, data[i]);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
return ok;
|
|
}
|
|
|
|
// Tests Y-shaped scheduling: two parallel lanes merging into 1. Lane A and B, merging into a single split.
|
|
// The lanes join in a final split on backend_a that receives one activation from each of the two backend_b splits.
|
|
static test_result test_y_shaped_graph(const std::vector<ggml_backend_t> & backends, int backend_a, int backend_b, int64_t tensor_len,
|
|
int32_t sleep_us, bool use_device_host_buft) {
|
|
|
|
const size_t graph_size = 64;
|
|
|
|
backend_consts consts(backends, tensor_len);
|
|
|
|
ggml_backend_sched_ptr sched = create_test_scheduler(backends, graph_size, use_device_host_buft);
|
|
|
|
ggml_init_params params_compute = {
|
|
/*.mem_size =*/ (graph_size + 8)*ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false),
|
|
/*.mem_buffer =*/ nullptr,
|
|
/*.no_alloc =*/ true,
|
|
};
|
|
ggml_context_ptr ctx_compute(ggml_init(params_compute));
|
|
|
|
helper_graph g(ctx_compute.get(), graph_size);
|
|
|
|
ggml_tensor * seed_a = g.add(ggml_add(ctx_compute.get(), consts.zero[backend_a], consts.one[backend_a]), backend_a);
|
|
ggml_set_name(seed_a, "seed_a");
|
|
|
|
// only lane a sleeps, so that lane b is scheduled while it is still running
|
|
if (sleep_us > 0) {
|
|
seed_a = g.add(ggml_sleep(ctx_compute.get(), seed_a, sleep_us), backend_a);
|
|
ggml_set_name(seed_a, "seed_a_delayed");
|
|
}
|
|
|
|
ggml_tensor * lane_a = g.add(ggml_add(ctx_compute.get(), seed_a, consts.one[backend_b]), backend_b);
|
|
ggml_set_name(lane_a, "lane_a");
|
|
|
|
if (sleep_us > 0) {
|
|
lane_a = g.add(ggml_sleep(ctx_compute.get(), lane_a, sleep_us), backend_b);
|
|
ggml_set_name(lane_a, "lane_a_delayed");
|
|
}
|
|
|
|
ggml_tensor * seed_b = g.add(ggml_add(ctx_compute.get(), consts.zero[backend_a], consts.one[backend_a]), backend_a);
|
|
ggml_set_name(seed_b, "seed_b");
|
|
|
|
ggml_tensor * lane_b = g.add(ggml_add(ctx_compute.get(), seed_b, consts.one[backend_b]), backend_b);
|
|
ggml_set_name(lane_b, "lane_b");
|
|
|
|
// neither lane ends on backend_a, so both activations are inputs of the last split
|
|
ggml_tensor * out = g.add(ggml_add(ctx_compute.get(), lane_a, lane_b), backend_a);
|
|
ggml_set_name(out, "out");
|
|
ggml_set_output(out);
|
|
|
|
const test_result ok = run_and_check(sched.get(), g, backends, {{ out, 4.0f }}, tensor_len);
|
|
|
|
return ok;
|
|
}
|
|
|
|
static const char * dev_type_name(enum ggml_backend_dev_type type) {
|
|
switch (type) {
|
|
case GGML_BACKEND_DEVICE_TYPE_CPU: return "CPU";
|
|
case GGML_BACKEND_DEVICE_TYPE_GPU: return "GPU";
|
|
case GGML_BACKEND_DEVICE_TYPE_IGPU: return "IGPU";
|
|
case GGML_BACKEND_DEVICE_TYPE_ACCEL: return "ACCEL";
|
|
case GGML_BACKEND_DEVICE_TYPE_META: return "META";
|
|
}
|
|
return "UNKNOWN";
|
|
}
|
|
|
|
static bool initialize_backends(std::vector<ggml_backend_ptr> & backends) {
|
|
|
|
// cf. GGML_SCHED_MAX_BACKENDS
|
|
const size_t max_backends = 16;
|
|
|
|
for (size_t i = 0; i < ggml_backend_dev_count() && backends.size() + 1 < max_backends; i++) {
|
|
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
|
|
log_maybe("device %2zu: %-10s %-5s (%s)\n", i, ggml_backend_dev_name(dev),
|
|
dev_type_name(ggml_backend_dev_type(dev)), ggml_backend_dev_description(dev));
|
|
|
|
const enum ggml_backend_dev_type type = ggml_backend_dev_type(dev);
|
|
// Exclude ACCEL devices as they run in tandem with the CPU backend and only implement matmul, the tests here require ADD
|
|
if (type == GGML_BACKEND_DEVICE_TYPE_CPU || type == GGML_BACKEND_DEVICE_TYPE_ACCEL) {
|
|
continue;
|
|
}
|
|
|
|
ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
|
|
if (backend == nullptr) {
|
|
printf("failed to initialize %s, skipping it\n", ggml_backend_dev_name(dev));
|
|
continue;
|
|
}
|
|
|
|
backends.emplace_back(backend);
|
|
}
|
|
|
|
if (backends.empty()) {
|
|
printf("no non-CPU backend found, skipping\n");
|
|
return false;
|
|
}
|
|
|
|
// ggml_backend_sched_new requires the CPU backend to be the last one
|
|
backends.emplace_back(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr));
|
|
GGML_ASSERT(backends.back() != nullptr);
|
|
|
|
for (const ggml_backend_ptr & backend : backends) {
|
|
ggml_backend_dev_t dev = ggml_backend_get_device(backend.get());
|
|
|
|
log_maybe("backend: %-10s host_buft = %d (%s)\n", ggml_backend_name(backend.get()),
|
|
ggml_backend_dev_host_buffer_type(dev) != nullptr, ggml_backend_dev_description(dev));
|
|
}
|
|
log_maybe("\n");
|
|
return true;
|
|
}
|
|
|
|
|
|
int main(int argc, char ** argv) {
|
|
for (int i = 1; i < argc; i++) {
|
|
if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
|
|
print_log = true;
|
|
} else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
|
|
printf("Usage: %s [-v|--verbose]\n", argv[0]);
|
|
return 0;
|
|
} else {
|
|
fprintf(stderr, "unknown argument: %s\n", argv[i]);
|
|
fprintf(stderr, "Usage: %s [-v|--verbose]\n", argv[0]);
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
ggml_backend_load_all();
|
|
|
|
std::vector<ggml_backend_ptr> backends;
|
|
|
|
// nothing to test with synchronous CPU backend only
|
|
if (!initialize_backends(backends)) {
|
|
return 0;
|
|
}
|
|
|
|
// the tests only borrow the backends, the ownership stays with this vector
|
|
std::vector<ggml_backend_t> backend_ptrs;
|
|
for (const ggml_backend_ptr & backend : backends) {
|
|
backend_ptrs.push_back(backend.get());
|
|
}
|
|
|
|
ggml_backend_t cpu = backend_ptrs.back();
|
|
const size_t n_non_cpu_backends = backend_ptrs.size() - 1;
|
|
|
|
// Test cases for single backends against CPU backend
|
|
for (size_t b_id = 0; b_id < n_non_cpu_backends; b_id++) {
|
|
ggml_backend_t backend = backend_ptrs[b_id];
|
|
const char * name_backend = ggml_backend_name(backend);
|
|
|
|
for (bool use_device_host_buft : { false, true }) {
|
|
// without a host buffer type this case is a duplicate of the pageable one
|
|
if (use_device_host_buft && ggml_backend_dev_host_buffer_type(ggml_backend_get_device(backend)) == nullptr) {
|
|
continue;
|
|
}
|
|
|
|
log_maybe("=== Backend: %s, CPU sched buft: %s ===\n\n", name_backend, use_device_host_buft ? "device_host" : "pageable");
|
|
|
|
for (int32_t sleep_us : {50, 60, 70, 80, 400, 500, 600, 700, 1000, 10000}) {
|
|
for (int tensor_len : { 2, 2048, 4096, 8192, 1600}) {
|
|
case_begin("test_inputless_splits_scheduling %-8s sleep_us = %6d, tensor_len = %4d", name_backend, sleep_us, tensor_len);
|
|
case_end(test_inputless_splits_scheduling(backend, cpu, tensor_len, sleep_us, use_device_host_buft));
|
|
}
|
|
}
|
|
|
|
log_maybe("\n");
|
|
}
|
|
}
|
|
|
|
// Test cases for all present backend subsets
|
|
for (bool use_device_host_buft : { false, true }) {
|
|
// create_test_scheduler takes the host buffer type from the first backend
|
|
if (use_device_host_buft && ggml_backend_dev_host_buffer_type(ggml_backend_get_device(backend_ptrs[0])) == nullptr) {
|
|
continue;
|
|
}
|
|
|
|
log_maybe("=== all backends, CPU sched buft: %s ===\n\n", use_device_host_buft ? "device_host" : "pageable");
|
|
|
|
for (int n_lanes : { 8, 16 }) {
|
|
for (int n_rounds : { 5, 13, 64 }) {
|
|
for (int tensor_len : { 2, 4096 }) {
|
|
case_begin("stress_test_dag n_lanes = %2d, n_rounds = %2d, tensor_len = %4d", n_lanes, n_rounds, tensor_len);
|
|
case_end(stress_test_dag(backend_ptrs, n_lanes, n_rounds, tensor_len, use_device_host_buft));
|
|
}
|
|
}
|
|
}
|
|
|
|
log_maybe("\n");
|
|
|
|
for (int tensor_len : { 2, 4096 }) {
|
|
for (int32_t sleep_us : { 0, 80 }) {
|
|
case_begin("test_chain_all_backends sleep_us = %4d, tensor_len = %4d", sleep_us, tensor_len);
|
|
case_end(test_chain_all_backends(backend_ptrs, tensor_len, sleep_us, use_device_host_buft));
|
|
}
|
|
}
|
|
|
|
log_maybe("\n");
|
|
|
|
for (int n_laps : { 1, 4, 64 }) {
|
|
for (int tensor_len : { 2, 4096 }) {
|
|
case_begin("stress_test_linked_list n_laps = %3d, tensor_len = %4d", n_laps, tensor_len);
|
|
case_end(stress_test_linked_list(backend_ptrs, n_laps, tensor_len, use_device_host_buft));
|
|
}
|
|
}
|
|
|
|
log_maybe("\n");
|
|
|
|
// every ordered pair of backends is the sender and the receiver of a copy
|
|
for (size_t backend_a = 0; backend_a < backend_ptrs.size(); backend_a++) {
|
|
for (size_t backend_b = 0; backend_b < backend_ptrs.size(); backend_b++) {
|
|
if (backend_a == backend_b) {
|
|
continue;
|
|
}
|
|
|
|
const char * name_a = ggml_backend_name(backend_ptrs[backend_a]);
|
|
const char * name_b = ggml_backend_name(backend_ptrs[backend_b]);
|
|
|
|
for (bool inputs_on_sender : { false, true }) {
|
|
for (bool parallel : { false, true }) {
|
|
for (int tensor_len : { 1, 4096 }) {
|
|
case_begin("test_pair_user_inputs %-8s -> %-8s inputs on %-8s parallel = %d, tensor_len = %4d",
|
|
name_a, name_b, inputs_on_sender ? "sender" : "receiver", parallel, tensor_len);
|
|
case_end(test_pair_user_inputs(backend_ptrs, (int) backend_a, (int) backend_b, tensor_len,
|
|
/*n_inputs =*/ 4, inputs_on_sender, parallel, use_device_host_buft));
|
|
}
|
|
}
|
|
}
|
|
|
|
for (int tensor_len : { 1, 4096 }) {
|
|
for (int32_t sleep_us : { 0, 80 }) {
|
|
case_begin("test_y_shaped %-8s -> %-8s sleep_us = %4d, tensor_len = %4d", name_a, name_b, sleep_us, tensor_len);
|
|
case_end(test_y_shaped_graph(backend_ptrs, (int) backend_a, (int) backend_b, tensor_len, sleep_us, use_device_host_buft));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
log_maybe("\n");
|
|
}
|
|
|
|
if (print_log || n_ok + n_skip != n_test) {
|
|
printf("%d/%d tests passed", n_ok, n_test);
|
|
if (n_skip > 0) {
|
|
printf(", %d skipped", n_skip);
|
|
}
|
|
printf("\n");
|
|
}
|
|
|
|
return n_ok + n_skip == n_test ? 0 : 1;
|
|
}
|