From 2cbfc3cf34f94c8ada67eb8766efe7374a3d2db6 Mon Sep 17 00:00:00 2001 From: mmalpe-nv Date: Wed, 18 Nov 2026 22:59:56 +0530 Subject: [PATCH] [CUDA] : Add SKU-aware FP16/BF16 accumulation precision gating for NVIDIA GPUs (#30672) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cuda: select cuBLAS precision by GPU SKU Assisted-by: Codex * cuda: fix remaining GB10 macro reference * cuda: review comment addressed, store accumulation preference in device info * Apply batched suggestions from code review Co-authored-by: Johannes Gäßler * addressed Oliver's comment regarding initialize NVML once and install headers in Windows CI * move precision selection helper into init --------- Co-authored-by: Johannes Gäßler --- .github/actions/windows-setup-cuda/action.yml | 10 +++ ggml/src/ggml-cuda/CMakeLists.txt | 2 + ggml/src/ggml-cuda/common.cuh | 3 +- ggml/src/ggml-cuda/fattn-common.cuh | 2 +- ggml/src/ggml-cuda/fattn.cu | 2 +- ggml/src/ggml-cuda/gated-delta-net-mma.cu | 2 +- ggml/src/ggml-cuda/ggml-cuda.cu | 80 ++++++++++++++++++- ggml/src/ggml-cuda/mmq-few.cu | 8 +- ggml/src/ggml-cuda/mmvq.cu | 10 +-- 9 files changed, 105 insertions(+), 14 deletions(-) diff --git a/.github/actions/windows-setup-cuda/action.yml b/.github/actions/windows-setup-cuda/action.yml index 51f1147e1..7548719bc 100644 --- a/.github/actions/windows-setup-cuda/action.yml +++ b/.github/actions/windows-setup-cuda/action.yml @@ -25,7 +25,9 @@ runs: curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-11.7.91-archive.zip" curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-11.7.101-archive.zip" curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-11.7.91-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-11.7.91-archive.zip" unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvml_dev-windows-x86_64-11.7.91-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_cudart-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvcc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvrtc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y @@ -54,7 +56,9 @@ runs: curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-12.4.127-archive.zip" curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-12.4.127-archive.zip" curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-12.4.127-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-12.4.127-archive.zip" unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvml_dev-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_cudart-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvcc-windows-x86_64-12.4.131-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvrtc-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y @@ -85,7 +89,9 @@ runs: curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.1.80-archive.zip" curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.1.68-archive.zip" curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-13.1.78-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.1.68-archive.zip" unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvml_dev-windows-x86_64-13.1.68-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_crt-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_cudart-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvcc-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y @@ -116,7 +122,9 @@ runs: curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.5.25-archive.zip" curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.5.25-archive.zip" curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.5.0.6-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.5.37-archive.zip" unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_cudart-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y @@ -143,7 +151,9 @@ runs: curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.5.37-archive.zip" curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.5.37-archive.zip" curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.10.0.20-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-arm64/cuda_nvml_dev-windows-arm64-13.5.37-archive.zip" unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-arm64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cccl-windows-x86_64-13.3.5.0.6-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y diff --git a/ggml/src/ggml-cuda/CMakeLists.txt b/ggml/src/ggml-cuda/CMakeLists.txt index 211a8dac7..dcb380535 100644 --- a/ggml/src/ggml-cuda/CMakeLists.txt +++ b/ggml/src/ggml-cuda/CMakeLists.txt @@ -133,6 +133,8 @@ if (CUDAToolkit_FOUND) ${GGML_SOURCES_CUDA} ) + target_link_libraries(ggml-cuda PRIVATE ${CMAKE_DL_LIBS}) + if (GGML_CUDA_GRAPHS) add_compile_definitions(GGML_CUDA_USE_GRAPHS) endif() diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index bfbd6b621..fd047d7c3 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -59,7 +59,7 @@ // While BW spans CC 1000, 1100 & 1200, we are integrating Tensor Core instructions available to 1200 family, see // https://docs.nvidia.com/cutlass/media/docs/cpp/blackwell_functionality.html#blackwell-sm120-gemms #define GGML_CUDA_CC_BLACKWELL 1200 -#define GGML_CUDA_CC_DGX_SPARK 1210 +#define GGML_CUDA_CC_GB10 1210 #define GGML_CUDA_CC_RUBIN 1300 #define GGML_CUDA_CC_OFFSET_AMD 0x1000000 #define GGML_CUDA_CC_OFFSET_MTHREADS 0x0100000 @@ -1293,6 +1293,7 @@ struct ggml_cuda_device_info { size_t total_vram; int warp_size; // Number of threads in a dispatch bool supports_cooperative_launch; // whether cooperative launch is supported + bool prefer_fp32_acc; // prefer BF16 inputs with FP32 accumulation for quantized cuBLAS int physical_device; // backing physical CUDA device for this (virtual) device int physical_share_count; // number of (virtual) devices sharing this device's physical GPU int virtual_index; // index of this (virtual) device among those sharing its physical GPU diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index 666542c80..fc0cc7403 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -1206,7 +1206,7 @@ void launch_fattn( dim3 blocks_num; if (stream_k) { // Stream-K splits the work before the mask scan is applied, so skipped KV tiles make the blocks uneven. - const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK && async_kv_preload && scan_mask; + const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10 && async_kv_preload && scan_mask; auto should_use_stream_k = [prefer_whole_tiles](const int cc, const int ntiles_dst, const int max_blocks, const int DKQ) { const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks; diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index fb542d41c..2e3276084 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -397,7 +397,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg if (amd_mfma_available(cc) && ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, 16)) { ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst); } else if (gqa_ratio == 20 && GGML_CUDA_CC_IS_NVIDIA(cc)) { // GLM 4.7 Flash - if (cc >= GGML_CUDA_CC_DGX_SPARK) { + if (cc >= GGML_CUDA_CC_GB10) { if (Q->ne[1] <= 8) { ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst); break; diff --git a/ggml/src/ggml-cuda/gated-delta-net-mma.cu b/ggml/src/ggml-cuda/gated-delta-net-mma.cu index d3b8e575b..427a33474 100644 --- a/ggml/src/ggml-cuda/gated-delta-net-mma.cu +++ b/ggml/src/ggml-cuda/gated-delta-net-mma.cu @@ -468,7 +468,7 @@ static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a // Enable only architectures benchmarked for this schedule. MMA support alone does not imply a speedup. if (!ampere_mma_available(info.cc) || (info.cc != GGML_CUDA_CC_AMPERE_GA10X && info.cc != GGML_CUDA_CC_ADA_LOVELACE && - info.cc != GGML_CUDA_CC_BLACKWELL && info.cc != GGML_CUDA_CC_DGX_SPARK)) { + info.cc != GGML_CUDA_CC_BLACKWELL && info.cc != GGML_CUDA_CC_GB10)) { return gdn_path::ar; } // Value splitting pays off when whole heads leave at least half the SMs idle. diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 3d8d30d81..40e83750e 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -96,6 +96,11 @@ #include #include +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#include "ggml-backend-dl.h" +#include +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size"); #define GGML_LOG_WARN_ONCE(str) \ @@ -309,6 +314,64 @@ static ggml_cuda_device_info ggml_cuda_init() { } total_vram = 0; +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#if defined(_WIN32) + // NVML is installed by the driver; its NVSMI directory is not added to PATH on Windows. + dl_handle_ptr library(LoadLibraryExW(L"nvml.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32)); + if (!library) { + const DWORD size = GetEnvironmentVariableW(L"ProgramW6432", nullptr, 0); + if (size > 0) { + std::wstring path(size, L'\0'); + const DWORD length = GetEnvironmentVariableW(L"ProgramW6432", &path[0], size); + if (length > 0 && length < size) { + path.resize(length); + path += L"\\NVIDIA Corporation\\NVSMI\\nvml.dll"; + if (fs::path(path).is_absolute()) { + library.reset(LoadLibraryExW(path.c_str(), nullptr, LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS)); + } + } + } + } +#else + dl_handle_ptr library(dlopen("libnvidia-ml.so.1", RTLD_NOW | RTLD_LOCAL)); +#endif // defined(_WIN32) + const auto get_symbol = [&](const char * name) { +#if defined(_WIN32) + return library ? GetProcAddress(library.get(), name) : nullptr; +#else + return library ? dlsym(library.get(), name) : nullptr; +#endif // defined(_WIN32) + }; + const auto init = reinterpret_cast(get_symbol("nvmlInit_v2")); + const auto shutdown = reinterpret_cast(get_symbol("nvmlShutdown")); + const auto get_device = reinterpret_cast(get_symbol("nvmlDeviceGetHandleByPciBusId_v2")); + const auto get_brand = reinterpret_cast(get_symbol("nvmlDeviceGetBrand")); + const bool nvml_initialized = init && shutdown && get_device && get_brand && init() == NVML_SUCCESS; + + const auto should_use_fp32_acc = [&](const cudaDeviceProp & prop, const int cc) { + if (cc == GGML_CUDA_CC_GB10) { + return true; + } + if (!fast_bf16_hardware_available(cc) || !nvml_initialized) { + return false; + } + + // CUDA and NVML device indices can differ, e.g. with CUDA_VISIBLE_DEVICES. + char pci_bus_id[32]; + snprintf(pci_bus_id, sizeof(pci_bus_id), "%04x:%02x:%02x.0", + (unsigned int) prop.pciDomainID, (unsigned int) prop.pciBusID, (unsigned int) prop.pciDeviceID); + nvmlDevice_t device = nullptr; + nvmlBrandType_t brand = NVML_BRAND_UNKNOWN; + bool prefer_fp32_acc = false; + if (get_device(pci_bus_id, &device) == NVML_SUCCESS && get_brand(device, &brand) == NVML_SUCCESS) { + prefer_fp32_acc = brand == NVML_BRAND_QUADRO || + brand == NVML_BRAND_QUADRO_RTX || + brand == NVML_BRAND_NVIDIA_RTX; + } + return prefer_fp32_acc; + }; +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + std::vector> turing_devices_without_mma; for (int id = 0; id < info.device_count; ++id) { const int physical_id = info.devices[id].physical_device; @@ -379,6 +442,11 @@ static ggml_cuda_device_info ggml_cuda_init() { #else info.devices[id].smpbo = prop.sharedMemPerBlockOptin; info.devices[id].cc = 100*prop.major + 10*prop.minor; + if (info.devices[id].virtual_index == 0) { + info.devices[id].prefer_fp32_acc = should_use_fp32_acc(prop, info.devices[id].cc); + } else { + info.devices[id].prefer_fp32_acc = info.devices[physical_id].prefer_fp32_acc; + } GGML_LOG_INFO(" Device %d: %s, compute capability %d.%d, VMM: %s, VRAM: %zu MiB\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no", device_vram_mib); @@ -403,6 +471,13 @@ static ggml_cuda_device_info ggml_cuda_init() { #endif // defined(GGML_USE_HIP) } +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + if (nvml_initialized) { + (void) shutdown(); + } + library.reset(); +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + if (ggml_cuda_highest_compiled_arch(GGML_CUDA_CC_TURING) >= GGML_CUDA_CC_TURING && !turing_devices_without_mma.empty()) { GGML_LOG_INFO("The following devices will have suboptimal performance due to a lack of tensor cores:\n"); for (size_t device_pos = 0; device_pos < turing_devices_without_mma.size(); device_pos++) { @@ -1641,7 +1716,7 @@ static void * ggml_backend_cuda_comm_init_rank(ggml_backend_t backend, int rank, // NCCL_MAX_CTAS still overrides this. ncclConfig_t config = NCCL_CONFIG_INITIALIZER; const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc; - if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) { + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10) { config.maxCTAs = 8; } @@ -2047,6 +2122,9 @@ static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml ggml_type compute_type = src0->type; if (ggml_is_quantized(compute_type) || compute_type == GGML_TYPE_F8_E4M3) { compute_type = fast_fp16_hardware_available(cc) ? GGML_TYPE_F16 : GGML_TYPE_F32; + if (ggml_cuda_info().devices[ctx.device].prefer_fp32_acc) { + compute_type = GGML_TYPE_BF16; + } } else if (compute_type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc)) { compute_type = GGML_TYPE_F32; } else if (compute_type == GGML_TYPE_BF16 && !fast_bf16_hardware_available(cc)) { diff --git a/ggml/src/ggml-cuda/mmq-few.cu b/ggml/src/ggml-cuda/mmq-few.cu index a141cc718..bc7cbc6e5 100644 --- a/ggml/src/ggml-cuda/mmq-few.cu +++ b/ggml/src/ggml-cuda/mmq-few.cu @@ -584,7 +584,7 @@ static __global__ void mul_mat_q_few( ggml_cuda_pdl_sync(); } -#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK +#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10 // DGX Spark DRAM serves the short row pieces of the stage copies slowly, so the rows of the block are first prefetched into L2 in address order. { const int tid = threadIdx.y*warp_size + threadIdx.x; @@ -599,7 +599,7 @@ static __global__ void mul_mat_q_few( } } } -#endif // __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK +#endif // __CUDA_ARCH__ == GGML_CUDA_CC_GB10 #pragma unroll for (int stage = 0; stage < nstages - 1; ++stage) { @@ -880,14 +880,14 @@ static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const return 0; } // on DGX Spark the kernel needs its L2 prefetch, which is only in the DGX Spark device code - if (cc == GGML_CUDA_CC_DGX_SPARK && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_DGX_SPARK) { + if (cc == GGML_CUDA_CC_GB10 && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_GB10) { return 0; } // with fewer than 8 units of 256 values per row the warps idle, MMVQ and MMQ are better there (MUL_MAT_ID splits the rows instead) if (mmq_few_unit_size(type) == 0 || (!has_ids && ne00 < 8*QK_K)) { return 0; } - if (cc == GGML_CUDA_CC_DGX_SPARK) { // tuned on DGX Spark GB10, also for MUL_MAT_ID + if (cc == GGML_CUDA_CC_GB10) { // tuned on DGX Spark GB10, also for MUL_MAT_ID return type == GGML_TYPE_IQ4_XS || type == GGML_TYPE_IQ4_NL ? 8 : 2; } if (has_ids) { // after MMVQ, which is faster up to its max. batch size diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 9118fbd55..612d3e313 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -9,7 +9,7 @@ // only enabled on DGX Spark, where it is a gain on every type below. On the higher-bandwidth parts the kernel // has little exposed latency left to hide and the extra requests cost more than they save. // For perf data, see https://github.com/ggml-org/llama.cpp/pull/26705#issuecomment-5569335031 -#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK +#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10 // returns true only for those quants that benefit from prefetch and false otherwise static constexpr __host__ __device__ bool mmvq_should_prefetch(ggml_type type) { switch (type) { @@ -110,7 +110,7 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() { return MMVQ_PARAMETERS_GCN; #elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA && __CUDA_ARCH__ < GGML_CUDA_CC_AMPERE return MMVQ_PARAMETERS_TURING; -#elif __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK +#elif __CUDA_ARCH__ == GGML_CUDA_CC_GB10 return MMVQ_PARAMETERS_GB10; #else return MMVQ_PARAMETERS_GENERIC; @@ -133,7 +133,7 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) { if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA && ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_AMPERE) { return MMVQ_PARAMETERS_TURING; } - if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_DGX_SPARK) { + if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_GB10) { return MMVQ_PARAMETERS_GB10; } return MMVQ_PARAMETERS_GENERIC; @@ -342,7 +342,7 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) { return ne11 <= MMVQ_MAX_BATCH_SIZE; } } - if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) { + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10) { switch (type) { // tuned on DGX Spark GB10 case GGML_TYPE_Q2_K: return ne11 <= 6; @@ -751,7 +751,7 @@ static __global__ void mul_mat_vec_q( // x block quant index when casting the quants to int const int kqs = vdr * (tid % (qi/vdr)); -#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK +#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10 // start the next iterations' weight loads early if constexpr (mmvq_should_prefetch(type)) { // loop iterations, not blocks; q3_K and q6_K keep the distance in blocks they had with vdr 1