From f851e244d0dc56074ab701c4467edb116840cb88 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 13 Oct 2026 15:24:34 +0300 Subject: [PATCH] cuda : add GGML_CUDA_GRAPH_DEBUG and read env vars once (#30327) * ggml-cuda: gate CUDA graph debug logs behind GGML_CUDA_GRAPH_DEBUG * ggml-cuda: cache env var lookups in static const instead of re-reading per call * ggml-cuda: read env vars exactly once via ggml_cuda_env_flag/ggml_cuda_env_int * ggml-cuda: consolidate env flag reads into single static accessors * ggml-cuda : drop env helper comments Assisted-by: pi:llama.cpp/Qwen3.8-27B * cont : remove NDEBUG gates for CUDA graph logs --- ggml/src/ggml-cuda/ggml-cuda.cu | 99 +++++++++++++++++++++++++-------- 1 file changed, 76 insertions(+), 23 deletions(-) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index f4537d447..9fe618f85 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -138,10 +138,54 @@ int ggml_cuda_get_device() { return id; } +static bool ggml_cuda_env_flag(const char * name) { + const char * env = getenv(name); + return env != nullptr && std::atoi(env) != 0; +} + +static int ggml_cuda_env_int(const char * name, int default_value) { + const char * env = getenv(name); + return env != nullptr ? std::atoi(env) : default_value; +} + +static bool ggml_cuda_env_set(const char * name) { + return getenv(name) != nullptr; +} + +static bool ggml_cuda_unified_memory_enabled() { + static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_ENABLE_UNIFIED_MEMORY"); + return enabled; +} + +static bool ggml_cuda_pinned_memory_disabled() { + static const bool disabled = ggml_cuda_env_flag("GGML_CUDA_NO_PINNED"); + return disabled; +} + +static bool ggml_cuda_register_host_enabled() { + static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_REGISTER_HOST"); + return enabled; +} + +static bool ggml_cuda_fusion_disabled() { + static const bool disabled = ggml_cuda_env_flag("GGML_CUDA_DISABLE_FUSION"); + return disabled; +} + +static bool ggml_cuda_p2p_enabled() { + static const bool enabled = ggml_cuda_env_set("GGML_CUDA_P2P"); + return enabled; +} + +static int ggml_cuda_op_offload_min_batch() { + static const int min_batch = ggml_cuda_env_int("GGML_OP_OFFLOAD_MIN_BATCH", 32); + return min_batch; +} + static cudaError_t ggml_cuda_device_malloc(void ** ptr, size_t size, int device) { ggml_cuda_set_device(device); cudaError_t err; - if (getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") != nullptr) { + if (ggml_cuda_unified_memory_enabled()) { err = cudaMallocManaged(ptr, size); #if defined(GGML_USE_HIP) if (err == hipSuccess) { @@ -383,7 +427,7 @@ static ggml_cuda_device_info ggml_cuda_init() { // configure logging to stdout // CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr)); - if (getenv("GGML_CUDA_P2P") != nullptr) { + if (ggml_cuda_p2p_enabled()) { for (int id = 0; id < info.physical_device_count; ++id) { CUDA_CHECK(cudaSetDevice(id)); for (int id_other = 0; id_other < info.physical_device_count; ++id_other) { @@ -598,7 +642,7 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { CU_CHECK(cuMemRelease(handle)); // VMM Bug fix for P2P access if GGML_CUDA_P2P is set, or if NCCL build - bool use_peer_access = getenv("GGML_CUDA_P2P") != nullptr; + bool use_peer_access = ggml_cuda_p2p_enabled(); #if defined(GGML_USE_NCCL) use_peer_access = true; #endif // defined(GGML_USE_NCCL) @@ -1272,7 +1316,7 @@ static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buff } static void * ggml_cuda_host_malloc(size_t size) { - if (getenv("GGML_CUDA_NO_PINNED") != nullptr) { + if (ggml_cuda_pinned_memory_disabled()) { return nullptr; } @@ -2641,6 +2685,11 @@ static bool ggml_cuda_is_view_or_noop(const ggml_tensor * t) { } #ifdef USE_CUDA_GRAPH +static bool ggml_cuda_graph_debug_enabled() { + static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_GRAPH_DEBUG"); + return enabled; +} + static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { bool use_cuda_graph = true; @@ -2660,9 +2709,9 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { // the mul_mat_id fallback path synchronizes the stream, so we cannot use CUDA graphs // ref: https://github.com/ggml-org/llama.cpp/pull/18958 use_cuda_graph = false; -#ifndef NDEBUG - GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__); -#endif + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__); + } } } @@ -2686,7 +2735,9 @@ static bool ggml_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx if (cgraph->uid != 0 && cgraph->uid == graph->uid) { - GGML_LOG_DEBUG("CUDA Graph id %zu reused\n", cgraph->uid); + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("CUDA Graph id %zu reused\n", cgraph->uid); + } GGML_ASSERT((int)graph->node_props.size() == cgraph->n_nodes); return false; } @@ -2733,9 +2784,9 @@ static void ggml_cuda_graph_update_executable(ggml_backend_cuda_context * cuda_c #endif // CUDART_VERSION >= 12000 if (stat == cudaErrorGraphExecUpdateFailure) { -#ifndef NDEBUG - GGML_LOG_DEBUG("%s: CUDA graph update failed\n", __func__); -#endif + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("%s: CUDA graph update failed\n", __func__); + } // The pre-existing graph exec cannot be updated due to violated constraints // so instead clear error and re-instantiate @@ -3605,8 +3656,7 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, // try and fuse nodes and return the number of nodes to skip static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { - static bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION")); - if (disable_fusion) { + if (ggml_cuda_fusion_disabled()) { return 0; } @@ -4620,7 +4670,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co if (graph->graph == nullptr) { if (ggml_cuda_info().devices[cuda_ctx->device].cc < GGML_CUDA_CC_VOLTA) { - if (!graph->disable_due_to_gpu_arch) { + if (!graph->disable_due_to_gpu_arch && ggml_cuda_graph_debug_enabled()) { GGML_LOG_DEBUG("%s: disabling CUDA graphs due to GPU architecture\n", __func__); } graph->disable_due_to_gpu_arch = true; @@ -4655,7 +4705,9 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, // Warmup: need at least 2 calls with no property change on the 2nd call if (!properties_changed) { graph->warmup_complete = true; - GGML_LOG_DEBUG("%s: CUDA graph warmup complete\n", __func__); + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("%s: CUDA graph warmup complete\n", __func__); + } use_cuda_graph = true; cuda_graph_update_required = true; } @@ -4665,7 +4717,9 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, if (properties_changed) { // Properties changed - reset warmup, execute directly until stable again graph->warmup_complete = false; - GGML_LOG_DEBUG("%s: CUDA graph warmup reset\n", __func__); + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("%s: CUDA graph warmup reset\n", __func__); + } } else { use_cuda_graph = true; cuda_graph_update_required = graph->instance == nullptr; @@ -4718,7 +4772,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context; - static const bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION")); + const bool disable_fusion = ggml_cuda_fusion_disabled(); auto add_alloc_deps = [&](size_t start, size_t last_node) { @@ -5138,7 +5192,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot } bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) { - if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) { + if (!ggml_cuda_register_host_enabled()) { return false; } @@ -5161,7 +5215,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) { } void ggml_backend_cuda_unregister_host_buffer(void * buffer) { - if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) { + if (!ggml_cuda_register_host_enabled()) { return; } @@ -5288,8 +5342,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device))); // Check if UMA is explicitly enabled via environment variable - bool uma_env = getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") != nullptr; - bool is_uma = prop.integrated > 0 || uma_env; + bool is_uma = prop.integrated > 0 || ggml_cuda_unified_memory_enabled(); if (is_uma) { // For UMA systems (like DGX Spark), use system memory info @@ -5330,7 +5383,7 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back props->device_id = ctx->pci_bus_id.empty() ? nullptr : ctx->pci_bus_id.c_str(); ggml_backend_cuda_device_get_memory(dev, &props->memory_free, &props->memory_total); - bool host_buffer = getenv("GGML_CUDA_NO_PINNED") == nullptr; + const bool host_buffer = !ggml_cuda_pinned_memory_disabled(); #ifdef GGML_CUDA_NO_PEER_COPY bool events = false; #else @@ -6026,7 +6079,7 @@ ggml_backend_reg_t ggml_backend_cuda_reg() { std::lock_guard lock(mutex); if (!initialized) { ggml_backend_cuda_reg_context * ctx = new ggml_backend_cuda_reg_context; - const int min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32; + const int min_batch_size = ggml_cuda_op_offload_min_batch(); const ggml_cuda_device_info & info = ggml_cuda_info(); const bool virtual_devices = info.device_count > info.physical_device_count;