ggml : add backend scheduler tests (#27258)

* Adds initial scheduler stress tests

* Adds new sleep op to simplify scheduler testing

* Adds testcase addressing https://github.com/ggml-org/llama.cpp/issues/23321

* Adds experimental vulkan GGML_OP_SLEEP

* Improves structure and naming of previous tests. Adds new tests,
including one which detects the nkvo-bug correctly, and validates the
proposed hotfix.

* Cleanup: Removes CUDA specific comments and nvtx-markers

* Fix: forgot to add sleep specific test case

* Fix oversights identified by CI

* Drops default prints so that it behaves like other ctests. Adds -v flag
to toggle this behavior

* Fix: Early exit in CPU-only settings (cause for graviton CI fails)

* Improves stress_test_linked_list by removing overlap with
test_chain_all_backends and test_pair_user_inputs. The latter ones test
either activations or user inputs in isolation and just once, whereas
the stress test tests all combinations multiple rounds

* Exclude ACCEL devices from new tests due to incompatibility

* Fixes vulkan test failure in llvmpipe

* Adds asserts, improves comments and some checks

* Fixes nitpicks in white-space and variable naming

* Adds comments that certain vk drivers could time OP_SLEEP incorrectly

* Drops limitiation to CPU/GPU in test-sleep

* Aligns OP_SLEEP guards in cuda with vk guards (input shape and
contiguity)

* Implements feedback for OP_SLEEP to work in place (no copy)

* Apply non-code suggestions  from JohannesGaesslers code review

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>

* Removes struct backend_supports; enables tests to return skip, instead
of only pass/fail

* Make clang-tidy happy

* Removes sched_backend_caps, improves create_test_scheduler according to
feedback (L149)

* Removes "name" field from struct sched_check in favor of
ggml_set_name/ggml_get_name

* Drops unnecessary shader_clock.comp test and guards

* Replaces sched_ prefixes with helper_ prefix

* Moves test suite to new RAII types for better memory management

* Adds sleep option to test_chain_all_backends

* Adds sleep option to first-run arm of the y-shaped test

* Fix switchcase order for legibility in stress_test_dag

* Simplfies stress_test_linked_list via double for-loop

* Adds pseudo-random input to stress_test_linked_list, and runs all
transitions both with and without CPU input (next to activation copy as
input)

* Fix RPC Proto patch due to GGML_OPS bump

* Adds ggml-cpp.h treatment to test-sleep.cpp

* Adds TODO pointers for subsequent additional tests.

* Apply suggestion to ggml.h

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>

* Apply suggestion to tests/test-backend-sched.cpp

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>

* Apply suggestion to tests/test-backend-sched.cpp

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>

* Fixes node compatibility check to not take buffer locations into account.
This led to skipped tests due to incorrectly rejected ops due to wrong buffer locations in multi-GPU (which is one of the scheduler behaviors under test)

* Apply batched suggestions from gg code review

Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>

* Completes callsite change of gg-suggestion

* Switch test-sleep from std::chrono::steady_clock::now() to ggml_time_us
to avoid mac clock drift and spurious test failures.
This makes the test a bit nonsensical on CPU (same clock checks itself),
but the test still has merit on the other backends where
independent device clocks are used.

---------

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
This commit is contained in:
authored and GitHub committed 2026-11-19 16:40:13 +02:00
1 parent 8af68d486a
commit ac830a9cbb
19 files changed
+1469 -4

No files matched your search

+2 -2
View File
@@ -8,10 +8,10 @@ extern "C" {
#define RPC_PROTO_MAJOR_VERSION 10
#define RPC_PROTO_MINOR_VERSION 0
#define RPC_PROTO_PATCH_VERSION 0
#define RPC_PROTO_PATCH_VERSION 1
#ifdef __cplusplus
static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
#endif
#define GGML_RPC_MAX_SERVERS 16
+8
View File
@@ -429,6 +429,8 @@ extern "C" {
GGML_OP_GLU,
GGML_OP_SLEEP,
GGML_OP_COUNT,
};
@@ -2599,6 +2601,12 @@ extern "C" {
int n_tasks,
void * userdata);
// Busy-wait operator for scheduling and backend testing.
GGML_API struct ggml_tensor * ggml_sleep(
struct ggml_context * ctx,
struct ggml_tensor * a, // wait after this tensor has executed, return value is simply a full view of this tensor
int32_t us); // amount of time to wait in microseconds
// loss function
GGML_API struct ggml_tensor * ggml_cross_entropy_loss(
+5
View File
@@ -2034,6 +2034,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
{
ggml_compute_forward_fill(params, tensor);
} break;
case GGML_OP_SLEEP:
{
ggml_compute_forward_sleep(params, tensor);
} break;
case GGML_OP_FLASH_ATTN_EXT:
{
ggml_compute_forward_flash_attn_ext(params, tensor);
@@ -2306,6 +2310,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
case GGML_OP_REPEAT:
case GGML_OP_REPEAT_BACK:
case GGML_OP_LEAKY_RELU:
case GGML_OP_SLEEP:
{
n_tasks = 1;
} break;
+13
View File
@@ -2238,6 +2238,19 @@ void ggml_compute_forward_fill(const ggml_compute_params * params, ggml_tensor *
}
}
// ggml_compute_forward_sleep
void ggml_compute_forward_sleep(const ggml_compute_params * params, ggml_tensor * dst) {
if (params->ith != 0) {
return;
}
// same clock as ggml_time_us() callers; on macOS steady_clock is CLOCK_MONOTONIC_RAW, which can run faster
const int64_t t_end = ggml_time_us() + ggml_get_op_params_i32(dst, 0);
while (ggml_time_us() < t_end) {}
}
// ggml_compute_tri
template <typename T>
+1
View File
@@ -84,6 +84,7 @@ void ggml_compute_forward_top_k(const struct ggml_compute_params * params, struc
void ggml_compute_forward_leaky_relu(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_fill(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_sleep(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_flash_attn_ext(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_flash_attn_back(
const struct ggml_compute_params * params,
+10
View File
@@ -48,6 +48,7 @@
#include "ggml-cuda/rope.cuh"
#include "ggml-cuda/roll.cuh"
#include "ggml-cuda/scale.cuh"
#include "ggml-cuda/sleep.cuh"
#include "ggml-cuda/snake.cuh"
#include "ggml-cuda/softcap.cuh"
#include "ggml-cuda/softmax.cuh"
@@ -3145,6 +3146,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
case GGML_OP_FILL:
ggml_cuda_op_fill(ctx, dst);
break;
case GGML_OP_SLEEP:
ggml_cuda_op_sleep(ctx, dst);
break;
case GGML_OP_LIGHTNING_INDEXER:
ggml_cuda_lightning_indexer(ctx, dst);
break;
@@ -6809,6 +6813,12 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
return ggml_cuda_lightning_indexer_supported(dev_ctx->device, op);
case GGML_OP_TOP_K_MASK:
return op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32;
case GGML_OP_SLEEP:
#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
return false;
#else
return true;
#endif // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
default:
return false;
+32
View File
@@ -0,0 +1,32 @@
#include "sleep.cuh"
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
// %globaltimer is a nanosecond wall clock, unlike clock64() it is unaffected by the SM clock and by frequency scaling
static __device__ __forceinline__ uint64_t globaltimer_ns() {
uint64_t t;
asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
return t;
}
// a single thread is enough, the next kernel on the same stream cannot start before this one retires
static __global__ void sleep_ns(const uint64_t ns) {
const uint64_t t0 = globaltimer_ns();
while (globaltimer_ns() - t0 < ns) {}
}
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
sleep_ns<<<1, 1, 0, ctx.stream()>>>(1000*(uint64_t) ggml_get_op_params_i32(dst, 0));
CUDA_CHECK(cudaGetLastError());
}
#else
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
GGML_UNUSED(ctx);
GGML_UNUSED(dst);
GGML_ABORT("GGML_OP_SLEEP requires the %%globaltimer register, which is only available on CUDA");
}
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
+3
View File
@@ -0,0 +1,3 @@
#include "common.cuh"
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
@@ -1043,6 +1043,8 @@ void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph
} else if (tensor->op == GGML_OP_FILL) {
const float value = ggml_get_op_params_f32(tensor, 0);
tensor_clone = ggml_fill(ggml_ctx, src_clone[0], value);
} else if (tensor->op == GGML_OP_SLEEP) {
tensor_clone = ggml_sleep(ggml_ctx, src_clone[0], ggml_get_op_params_i32(tensor, 0));
} else if (tensor->op == GGML_OP_SQR) {
tensor_clone = ggml_sqr(ggml_ctx, src_clone[0]);
} else if (tensor->op == GGML_OP_SQRT) {
@@ -189,6 +189,10 @@ struct vk_op_push_constants {
float param4;
};
struct vk_op_sleep_push_constants {
uint32_t ticks;
};
struct vk_op_fwht_push_constants {
uint32_t n_rows;
uint32_t src_offset;
+3
View File
@@ -739,6 +739,7 @@ struct vk_device_struct {
bool shader_int64;
bool buffer_device_address;
bool vulkan_memory_model;
bool shader_clock;
bool add_rms_fusion;
uint32_t partials_binding_alignment;
@@ -923,6 +924,8 @@ struct vk_device_struct {
vk_pipeline pipeline_fill_f32;
vk_pipeline pipeline_fill_f16;
vk_pipeline pipeline_sleep;
// [src/dst 0=fp32,1=fp16,2=bf16]
vk_pipeline pipeline_glu[GGML_GLU_OP_COUNT][3];
+50
View File
@@ -3892,6 +3892,10 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
ggml_vk_create_pipeline(device, device->pipeline_fill_f32, "fill_f32", fill_f32_len, fill_f32_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_fill_f16, "fill_f16", fill_f16_len, fill_f16_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
if (device->shader_clock) {
ggml_vk_create_pipeline(device, device->pipeline_sleep, "sleep", sleep_len, sleep_data, "main", 1, sizeof(vk_op_sleep_push_constants), {1, 1, 1}, {}, 1);
}
#define CREATE_GLU(name, op) \
ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \
ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \
@@ -4521,6 +4525,7 @@ vk_device ggml_vk_get_device(size_t idx) {
bool ocp_microscaling_extension = false;
bool shader_float8_extension = false;
bool workgroup_memory_explicit_layout_support = false;
bool shader_clock_support = false;
for (const auto& properties : ext_props) {
if (strcmp("VK_KHR_maintenance4", properties.extensionName) == 0) {
@@ -4571,6 +4576,9 @@ vk_device ggml_vk_get_device(size_t idx) {
} else if (strcmp(VK_EXT_SHADER_FLOAT8_EXTENSION_NAME, properties.extensionName) == 0) {
shader_float8_extension = true;
#endif
} else if (strcmp("VK_KHR_shader_clock", properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_SHADER_CLOCK")) {
shader_clock_support = true;
} else if (strcmp("VK_VALVE_shader_mixed_float_dot_product", properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_DOT2")) {
dot2_f16_support = true;
@@ -4919,6 +4927,14 @@ vk_device ggml_vk_get_device(size_t idx) {
device_extensions.push_back("VK_KHR_shader_integer_dot_product");
}
VkPhysicalDeviceShaderClockFeaturesKHR shader_clock_features {};
shader_clock_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CLOCK_FEATURES_KHR;
if (shader_clock_support) {
last_struct->pNext = (VkBaseOutStructure *)&shader_clock_features;
last_struct = (VkBaseOutStructure *)&shader_clock_features;
device_extensions.push_back("VK_KHR_shader_clock");
}
VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE dot2_features {};
dot2_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MIXED_FLOAT_DOT_PRODUCT_FEATURES_VALVE;
if (dot2_f16_support) {
@@ -5025,6 +5041,7 @@ vk_device ggml_vk_get_device(size_t idx) {
device->shader_int64 = device_features2.features.shaderInt64;
device->buffer_device_address = vk12_features.bufferDeviceAddress;
device->vulkan_memory_model = vk12_features.vulkanMemoryModel;
device->shader_clock = shader_clock_support && shader_clock_features.shaderDeviceClock;
if (device->subgroup_size_control) {
device->subgroup_min_size = subgroup_size_control_props.minSubgroupSize;
@@ -10235,6 +10252,8 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
return ctx->device->pipeline_arange_f32;
}
return nullptr;
case GGML_OP_SLEEP:
return ctx->device->pipeline_sleep;
case GGML_OP_FILL:
if (dst->type == GGML_TYPE_F32) {
return ctx->device->pipeline_fill_f32;
@@ -11471,6 +11490,31 @@ void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements);
}
static void ggml_vk_sleep(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
VK_LOG_DEBUG("ggml_vk_sleep(dst=" << dst << ", us=" << ggml_get_op_params_i32(dst, 0) << ")");
// Vulkan does not specify the period of the shader realtime clock this assumes it matches the timestamp counter.
// A driver where this does not hold could sleep too short or too long by that ratio
const float ns_per_tick = ctx->device->properties.limits.timestampPeriod;
const uint64_t ns = 1000 * (uint64_t) ggml_get_op_params_i32(dst, 0);
const uint64_t ticks = std::min<uint64_t>(ns_per_tick > 0.0f ? (uint64_t)(ns / ns_per_tick) : ns, std::numeric_limits<uint32_t>::max());
vk_op_sleep_push_constants pc = {
(uint32_t) ticks,
};
vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, nullptr, nullptr, nullptr, dst, GGML_OP_SLEEP);
GGML_ASSERT(pipeline != nullptr);
ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst);
// a single invocation spins, the buffer is bound only to keep the spin from being optimized out
std::array<uint32_t, 3> elements = { 1, 1, 1 };
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements);
}
void ggml_vk_sin(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SIN, vk_op_unary_push_constants_init(src0, dst));
}
@@ -13317,6 +13361,10 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in
case GGML_OP_FILL:
ggml_vk_fill(ctx, compute_ctx, node);
break;
case GGML_OP_SLEEP:
ggml_vk_sleep(ctx, compute_ctx, node);
break;
case GGML_OP_SCALE:
if (ctx->fused_hc_post_gate) {
@@ -16768,6 +16816,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
return op->type == GGML_TYPE_F32;
case GGML_OP_FILL:
return op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16;
case GGML_OP_SLEEP:
return device->shader_clock;
case GGML_OP_SCALE:
return ggml_is_contiguous(op->src[0]) && ggml_vk_is_float_elementwise(op);
case GGML_OP_PAD:
@@ -0,0 +1,37 @@
#version 450
#extension GL_EXT_shader_realtime_clock : require
layout(local_size_x = 1, local_size_y = 1, local_size_z = 1) in;
// never written, it only anchors the spin below so the compiler cannot drop it
layout (binding = 0) writeonly buffer D {uint data_d[];};
layout (push_constant) uniform parameter {
uint ticks;
} p;
// bounds the spin in case the clock does not advance, turning a hang into a too-short sleep
const uint MAX_ITER = 1u << 27;
void main() {
const uvec2 t0 = clockRealtime2x32EXT();
uvec2 t = t0;
for (uint iter = 0; iter < MAX_ITER; ++iter) {
t = clockRealtime2x32EXT();
const uint lo = t.x - t0.x;
const uint hi = t.y - t0.y - (t.x < t0.x ? 1u : 0u);
if (hi != 0u || lo >= p.ticks) {
break;
}
}
// the store never runs, but the compiler cannot prove it and has to keep the spin
if (t.x == 0xFFFFFFFFu && t.y == 0xFFFFFFFFu) {
data_d[0] = t.x;
}
}
@@ -1028,6 +1028,7 @@ void process_shaders() {
string_to_spv("arange_f32", "arange.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
string_to_spv("fill_f32", "fill.comp", {{"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
string_to_spv("fill_f16", "fill.comp", {{"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});
string_to_spv("sleep", "sleep.comp", {});
string_to_spv("silu_back_f32", "silu_back.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}});
+27 -2
View File
@@ -1112,9 +1112,11 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
"OPT_STEP_SGD",
"GLU",
"SLEEP",
};
static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99");
static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT != 100");
static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
"none",
@@ -1225,9 +1227,11 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
"sgd(x)",
"glu(x)",
"sleep(x)",
};
static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99");
static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT != 100");
static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");
@@ -6176,6 +6180,27 @@ struct ggml_tensor * ggml_custom_inplace(
return result;
}
// ggml_sleep
// the result aliases a, the op only delays and never touches the data
struct ggml_tensor * ggml_sleep(
struct ggml_context * ctx,
struct ggml_tensor * a,
int32_t us) {
GGML_ASSERT(us >= 0);
struct ggml_tensor * result = ggml_view_tensor(ctx, a);
ggml_format_name(result, "%s (sleep)", a->name);
ggml_set_op_params_i32(result, 0, us);
result->op = GGML_OP_SLEEP;
result->src[0] = a;
return result;
}
// ggml_cross_entropy_loss
struct ggml_tensor * ggml_cross_entropy_loss(
+2
View File
@@ -367,6 +367,8 @@ endif()
llama_build(test-backend-ops.cpp LINK ggml Threads::Threads)
llama_build_and_test(test-backend-tensor.cpp LINK ggml)
llama_build_and_test(test-sleep.cpp LINK ggml)
llama_build_and_test(test-backend-sched.cpp LINK ggml)
if (BUILD_SHARED_LIBS)
llama_build_and_test(test-llama-mmap.cpp ${PROJECT_SOURCE_DIR}/src/llama-impl.cpp ${PROJECT_SOURCE_DIR}/src/llama-mmap.cpp LABEL behavior)
+31
View File
@@ -9657,6 +9657,33 @@ struct test_fill : public test_case {
}
};
// GGML_OP_SLEEP (for scheduler testing only)
struct test_sleep : public test_case {
const ggml_type type;
const std::array<int64_t, 4> ne;
const int32_t us;
std::string vars() override { return VARS_TO_STR3(type, ne, us); }
// the op only delays, the output aliases the input and must match it exactly
double max_nmse_err() override { return 0.0; }
test_sleep(int32_t us = 100, ggml_type type = GGML_TYPE_F32,
std::array<int64_t, 4> ne = { 10, 10, 4, 3 })
: type(type), ne(ne), us(us) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]);
ggml_set_name(a, "a");
ggml_tensor * out = ggml_sleep(ctx, a, us);
ggml_set_name(out, "out");
return out;
}
};
// GGML_OP_SOLVE_TRI
struct test_solve_tri : public test_case {
const ggml_type type;
@@ -12996,6 +13023,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_fill(-152.0f, GGML_TYPE_F16, { 800, 600, 4, 4 }));
test_cases.emplace_back(new test_fill(3.5f, GGML_TYPE_F16, { 2048, 512, 2, 2 }));
test_cases.emplace_back(new test_sleep(0));
test_cases.emplace_back(new test_sleep(10, GGML_TYPE_F16, { 128, 4, 2, 2 }));
test_cases.emplace_back(new test_sleep(10, GGML_TYPE_F32, { 128, 4, 2, 2 }));
for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) {
test_cases.emplace_back(new test_diag(type));
test_cases.emplace_back(new test_diag(type, { 79, 1, 19, 13 }));
File diff suppressed because it is too large. Load diff
+154
View File
@@ -0,0 +1,154 @@
// Tests for GGML_OP_SLEEP, on every backend that supports it. Three properties are checked:
// - the op only delays: input = output
// - the op does not return early: a chain of n sleeps must take at least n*us
// Only a lower bound is asserted for the duration. A busy-wait cannot finish early, so that bound is
// deterministic, whereas an upper bound would be flaky on a loaded machine.
// Known limitation: Vulkan does not specify the tick rate of the shader clock, see ggml_vk_sleep. A driver
// whose clock runs slower than assumed could undershoot the request and fail this bound.
#include "ggml.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include "ggml-cpp.h"
#include <cinttypes>
#include <cstdio>
#include <cstring>
#include <random>
#include <vector>
enum test_result {
TEST_OK,
TEST_FAIL,
TEST_SKIP,
};
static test_result test_sleep(ggml_backend_t backend, ggml_type type, int64_t ne, int32_t us, int n_nodes) {
const size_t graph_size = n_nodes + 16;
ggml_init_params params_static = {
/*.mem_size =*/ ggml_tensor_overhead(),
/*.mem_buffer =*/ nullptr,
/*.no_alloc =*/ true,
};
ggml_context_ptr ctx_static(ggml_init(params_static));
ggml_tensor * x = ggml_new_tensor_1d(ctx_static.get(), type, ne);
ggml_set_name(x, "x");
ggml_set_input(x);
ggml_init_params params_compute = {
/*.mem_size =*/ n_nodes*ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false),
/*.mem_buffer =*/ nullptr,
/*.no_alloc =*/ true,
};
ggml_context_ptr ctx_compute(ggml_init(params_compute));
ggml_cgraph * gf = ggml_new_graph_custom(ctx_compute.get(), graph_size, false);
ggml_tensor * out = x;
for (int i = 0; i < n_nodes; i++) {
out = ggml_sleep(ctx_compute.get(), out, us);
}
ggml_set_name(out, "out");
ggml_build_forward_expand(gf, out);
// the backends log to stdout while computing, so the result line is only printed once it is complete
char desc[128];
snprintf(desc, sizeof(desc), " %-6s %-4s ne=%-9" PRId64 " us=%-7d n_nodes=%d ",
ggml_backend_name(backend), ggml_type_name(type), ne, us, n_nodes);
test_result result = TEST_OK;
if (!ggml_backend_supports_op(backend, out)) {
printf("%snot supported\n", desc);
result = TEST_SKIP;
} else {
ggml_backend_buffer_ptr buf_static(ggml_backend_alloc_ctx_tensors(ctx_static.get(), backend));
// random bytes rather than random floats, so that bit patterns a value-wise copy could
// mangle, such as NaNs and denormals, are covered as well
std::vector<uint8_t> data_in(ggml_nbytes(x));
std::mt19937 rng(1234);
for (size_t i = 0; i < data_in.size(); i++) {
data_in[i] = rng() & 0xFF;
}
ggml_backend_tensor_set(x, data_in.data(), 0, data_in.size());
ggml_gallocr_ptr galloc(ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)));
ggml_gallocr_alloc_graph(galloc.get(), gf);
// warm up, so that one-time costs such as lazy CUDA module loading are not timed
ggml_backend_graph_compute(backend, gf);
const int64_t t_start_us = ggml_time_us();
const ggml_status status = ggml_backend_graph_compute(backend, gf);
const int64_t t_us = ggml_time_us() - t_start_us;
std::vector<uint8_t> data_out(ggml_nbytes(out));
ggml_backend_tensor_get(out, data_out.data(), 0, data_out.size());
const int64_t t_min_us = (int64_t) n_nodes * us;
const bool ok_status = status == GGML_STATUS_SUCCESS;
const bool ok_alias = out->data == x->data;
const bool ok_data = memcmp(data_in.data(), data_out.data(), data_in.size()) == 0;
const bool ok_time = t_us >= t_min_us;
printf("%selapsed=%7" PRId64 " us (>= %6" PRId64 ") alias: %-4s data: %-4s time: %s\n",
desc, t_us, t_min_us, ok_alias ? "OK" : "FAIL", ok_data ? "OK" : "FAIL", ok_time ? "OK" : "FAIL");
if (!ok_status) {
printf(" compute failed: %s\n", ggml_status_to_string(status));
}
result = ok_status && ok_alias && ok_data && ok_time ? TEST_OK : TEST_FAIL;
}
return result;
}
int main() {
ggml_time_init();
ggml_backend_load_all();
int n_ok = 0;
int n_fail = 0;
int n_skip = 0;
for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
ggml_backend_ptr backend(ggml_backend_dev_init(dev, nullptr));
if (backend == nullptr) {
printf(" failed to initialize %s\n", ggml_backend_dev_name(dev));
n_fail++;
continue;
}
// a zero duration must still be a valid no-op, longer ones are checked against the clock,
// and the last case verifies that a chain of sleeps accumulates instead of collapsing into one
const struct {
ggml_type type;
int64_t ne;
int32_t us;
int n_nodes;
} cases[] = {
{ GGML_TYPE_F32, 1, 0, 1 },
{ GGML_TYPE_F32, 4096, 10000, 1 },
{ GGML_TYPE_F16, 1024, 10000, 1 },
{ GGML_TYPE_F32, 1 << 20, 5000, 4 },
};
for (const auto & c : cases) {
switch (test_sleep(backend.get(), c.type, c.ne, c.us, c.n_nodes)) {
case TEST_OK: n_ok++; break;
case TEST_FAIL: n_fail++; break;
case TEST_SKIP: n_skip++; break;
}
}
}
printf("%d passed, %d failed, %d skipped\n", n_ok, n_fail, n_skip);
return n_fail > 0;
}