cuda : add GGML_CUDA_GRAPH_DEBUG and read env vars once (#30327)
* ggml-cuda: gate CUDA graph debug logs behind GGML_CUDA_GRAPH_DEBUG * ggml-cuda: cache env var lookups in static const instead of re-reading per call * ggml-cuda: read env vars exactly once via ggml_cuda_env_flag/ggml_cuda_env_int * ggml-cuda: consolidate env flag reads into single static accessors * ggml-cuda : drop env helper comments Assisted-by: pi:llama.cpp/Qwen3.8-27B * cont : remove NDEBUG gates for CUDA graph logs
This commit is contained in:
1 parent
bfe65e1dfa
commit
f851e244d0
1 file changed
+76
-23
@@ -138,10 +138,54 @@ int ggml_cuda_get_device() {
|
||||
return id;
|
||||
}
|
||||
|
||||
static bool ggml_cuda_env_flag(const char * name) {
|
||||
const char * env = getenv(name);
|
||||
return env != nullptr && std::atoi(env) != 0;
|
||||
}
|
||||
|
||||
static int ggml_cuda_env_int(const char * name, int default_value) {
|
||||
const char * env = getenv(name);
|
||||
return env != nullptr ? std::atoi(env) : default_value;
|
||||
}
|
||||
|
||||
static bool ggml_cuda_env_set(const char * name) {
|
||||
return getenv(name) != nullptr;
|
||||
}
|
||||
|
||||
static bool ggml_cuda_unified_memory_enabled() {
|
||||
static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_ENABLE_UNIFIED_MEMORY");
|
||||
return enabled;
|
||||
}
|
||||
|
||||
static bool ggml_cuda_pinned_memory_disabled() {
|
||||
static const bool disabled = ggml_cuda_env_flag("GGML_CUDA_NO_PINNED");
|
||||
return disabled;
|
||||
}
|
||||
|
||||
static bool ggml_cuda_register_host_enabled() {
|
||||
static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_REGISTER_HOST");
|
||||
return enabled;
|
||||
}
|
||||
|
||||
static bool ggml_cuda_fusion_disabled() {
|
||||
static const bool disabled = ggml_cuda_env_flag("GGML_CUDA_DISABLE_FUSION");
|
||||
return disabled;
|
||||
}
|
||||
|
||||
static bool ggml_cuda_p2p_enabled() {
|
||||
static const bool enabled = ggml_cuda_env_set("GGML_CUDA_P2P");
|
||||
return enabled;
|
||||
}
|
||||
|
||||
static int ggml_cuda_op_offload_min_batch() {
|
||||
static const int min_batch = ggml_cuda_env_int("GGML_OP_OFFLOAD_MIN_BATCH", 32);
|
||||
return min_batch;
|
||||
}
|
||||
|
||||
static cudaError_t ggml_cuda_device_malloc(void ** ptr, size_t size, int device) {
|
||||
ggml_cuda_set_device(device);
|
||||
cudaError_t err;
|
||||
if (getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") != nullptr) {
|
||||
if (ggml_cuda_unified_memory_enabled()) {
|
||||
err = cudaMallocManaged(ptr, size);
|
||||
#if defined(GGML_USE_HIP)
|
||||
if (err == hipSuccess) {
|
||||
@@ -383,7 +427,7 @@ static ggml_cuda_device_info ggml_cuda_init() {
|
||||
// configure logging to stdout
|
||||
// CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr));
|
||||
|
||||
if (getenv("GGML_CUDA_P2P") != nullptr) {
|
||||
if (ggml_cuda_p2p_enabled()) {
|
||||
for (int id = 0; id < info.physical_device_count; ++id) {
|
||||
CUDA_CHECK(cudaSetDevice(id));
|
||||
for (int id_other = 0; id_other < info.physical_device_count; ++id_other) {
|
||||
@@ -598,7 +642,7 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool {
|
||||
CU_CHECK(cuMemRelease(handle));
|
||||
|
||||
// VMM Bug fix for P2P access if GGML_CUDA_P2P is set, or if NCCL build
|
||||
bool use_peer_access = getenv("GGML_CUDA_P2P") != nullptr;
|
||||
bool use_peer_access = ggml_cuda_p2p_enabled();
|
||||
#if defined(GGML_USE_NCCL)
|
||||
use_peer_access = true;
|
||||
#endif // defined(GGML_USE_NCCL)
|
||||
@@ -1272,7 +1316,7 @@ static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buff
|
||||
}
|
||||
|
||||
static void * ggml_cuda_host_malloc(size_t size) {
|
||||
if (getenv("GGML_CUDA_NO_PINNED") != nullptr) {
|
||||
if (ggml_cuda_pinned_memory_disabled()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
@@ -2641,6 +2685,11 @@ static bool ggml_cuda_is_view_or_noop(const ggml_tensor * t) {
|
||||
}
|
||||
|
||||
#ifdef USE_CUDA_GRAPH
|
||||
static bool ggml_cuda_graph_debug_enabled() {
|
||||
static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_GRAPH_DEBUG");
|
||||
return enabled;
|
||||
}
|
||||
|
||||
static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) {
|
||||
|
||||
bool use_cuda_graph = true;
|
||||
@@ -2660,9 +2709,9 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) {
|
||||
// the mul_mat_id fallback path synchronizes the stream, so we cannot use CUDA graphs
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/18958
|
||||
use_cuda_graph = false;
|
||||
#ifndef NDEBUG
|
||||
GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__);
|
||||
#endif
|
||||
if (ggml_cuda_graph_debug_enabled()) {
|
||||
GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2686,7 +2735,9 @@ static bool ggml_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx
|
||||
|
||||
if (cgraph->uid != 0 &&
|
||||
cgraph->uid == graph->uid) {
|
||||
GGML_LOG_DEBUG("CUDA Graph id %zu reused\n", cgraph->uid);
|
||||
if (ggml_cuda_graph_debug_enabled()) {
|
||||
GGML_LOG_DEBUG("CUDA Graph id %zu reused\n", cgraph->uid);
|
||||
}
|
||||
GGML_ASSERT((int)graph->node_props.size() == cgraph->n_nodes);
|
||||
return false;
|
||||
}
|
||||
@@ -2733,9 +2784,9 @@ static void ggml_cuda_graph_update_executable(ggml_backend_cuda_context * cuda_c
|
||||
#endif // CUDART_VERSION >= 12000
|
||||
|
||||
if (stat == cudaErrorGraphExecUpdateFailure) {
|
||||
#ifndef NDEBUG
|
||||
GGML_LOG_DEBUG("%s: CUDA graph update failed\n", __func__);
|
||||
#endif
|
||||
if (ggml_cuda_graph_debug_enabled()) {
|
||||
GGML_LOG_DEBUG("%s: CUDA graph update failed\n", __func__);
|
||||
}
|
||||
|
||||
// The pre-existing graph exec cannot be updated due to violated constraints
|
||||
// so instead clear error and re-instantiate
|
||||
@@ -3605,8 +3656,7 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph,
|
||||
// try and fuse nodes and return the number of nodes to skip
|
||||
static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) {
|
||||
|
||||
static bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION"));
|
||||
if (disable_fusion) {
|
||||
if (ggml_cuda_fusion_disabled()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -4620,7 +4670,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co
|
||||
|
||||
if (graph->graph == nullptr) {
|
||||
if (ggml_cuda_info().devices[cuda_ctx->device].cc < GGML_CUDA_CC_VOLTA) {
|
||||
if (!graph->disable_due_to_gpu_arch) {
|
||||
if (!graph->disable_due_to_gpu_arch && ggml_cuda_graph_debug_enabled()) {
|
||||
GGML_LOG_DEBUG("%s: disabling CUDA graphs due to GPU architecture\n", __func__);
|
||||
}
|
||||
graph->disable_due_to_gpu_arch = true;
|
||||
@@ -4655,7 +4705,9 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,
|
||||
// Warmup: need at least 2 calls with no property change on the 2nd call
|
||||
if (!properties_changed) {
|
||||
graph->warmup_complete = true;
|
||||
GGML_LOG_DEBUG("%s: CUDA graph warmup complete\n", __func__);
|
||||
if (ggml_cuda_graph_debug_enabled()) {
|
||||
GGML_LOG_DEBUG("%s: CUDA graph warmup complete\n", __func__);
|
||||
}
|
||||
use_cuda_graph = true;
|
||||
cuda_graph_update_required = true;
|
||||
}
|
||||
@@ -4665,7 +4717,9 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,
|
||||
if (properties_changed) {
|
||||
// Properties changed - reset warmup, execute directly until stable again
|
||||
graph->warmup_complete = false;
|
||||
GGML_LOG_DEBUG("%s: CUDA graph warmup reset\n", __func__);
|
||||
if (ggml_cuda_graph_debug_enabled()) {
|
||||
GGML_LOG_DEBUG("%s: CUDA graph warmup reset\n", __func__);
|
||||
}
|
||||
} else {
|
||||
use_cuda_graph = true;
|
||||
cuda_graph_update_required = graph->instance == nullptr;
|
||||
@@ -4718,7 +4772,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev
|
||||
static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) {
|
||||
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
|
||||
|
||||
static const bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION"));
|
||||
const bool disable_fusion = ggml_cuda_fusion_disabled();
|
||||
|
||||
auto add_alloc_deps = [&](size_t start, size_t last_node) {
|
||||
|
||||
@@ -5138,7 +5192,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot
|
||||
}
|
||||
|
||||
bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
|
||||
if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
|
||||
if (!ggml_cuda_register_host_enabled()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -5161,7 +5215,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
|
||||
}
|
||||
|
||||
void ggml_backend_cuda_unregister_host_buffer(void * buffer) {
|
||||
if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
|
||||
if (!ggml_cuda_register_host_enabled()) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -5288,8 +5342,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *
|
||||
CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device)));
|
||||
|
||||
// Check if UMA is explicitly enabled via environment variable
|
||||
bool uma_env = getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") != nullptr;
|
||||
bool is_uma = prop.integrated > 0 || uma_env;
|
||||
bool is_uma = prop.integrated > 0 || ggml_cuda_unified_memory_enabled();
|
||||
|
||||
if (is_uma) {
|
||||
// For UMA systems (like DGX Spark), use system memory info
|
||||
@@ -5330,7 +5383,7 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back
|
||||
props->device_id = ctx->pci_bus_id.empty() ? nullptr : ctx->pci_bus_id.c_str();
|
||||
ggml_backend_cuda_device_get_memory(dev, &props->memory_free, &props->memory_total);
|
||||
|
||||
bool host_buffer = getenv("GGML_CUDA_NO_PINNED") == nullptr;
|
||||
const bool host_buffer = !ggml_cuda_pinned_memory_disabled();
|
||||
#ifdef GGML_CUDA_NO_PEER_COPY
|
||||
bool events = false;
|
||||
#else
|
||||
@@ -6026,7 +6079,7 @@ ggml_backend_reg_t ggml_backend_cuda_reg() {
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
if (!initialized) {
|
||||
ggml_backend_cuda_reg_context * ctx = new ggml_backend_cuda_reg_context;
|
||||
const int min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32;
|
||||
const int min_batch_size = ggml_cuda_op_offload_min_batch();
|
||||
|
||||
const ggml_cuda_device_info & info = ggml_cuda_info();
|
||||
const bool virtual_devices = info.device_count > info.physical_device_count;
|
||||
|
||||
Reference in new issue
Block a user