[CUDA] : Add SKU-aware FP16/BF16 accumulation precision gating for NVIDIA GPUs (#30672)

* cuda: select cuBLAS precision by GPU SKU

Assisted-by: Codex

* cuda: fix remaining GB10 macro reference

* cuda: review comment addressed, store accumulation preference in device info

* Apply batched suggestions from code review

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>

* addressed Oliver's comment regarding initialize NVML once and install headers in Windows CI

* move precision selection helper into init

---------

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
This commit is contained in:
mmalpe-nvandJohannes Gäßler authored and GitHub committed 2026-11-18 18:29:56 +01:00
1 parent 6f311d7ad6
commit 2cbfc3cf34
9 files changed
+105 -14

No files matched your search

@@ -25,7 +25,9 @@ runs:
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-11.7.91-archive.zip"
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-11.7.101-archive.zip"
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-11.7.91-archive.zip"
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-11.7.91-archive.zip"
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7"
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvml_dev-windows-x86_64-11.7.91-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_cudart-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvcc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvrtc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
@@ -54,7 +56,9 @@ runs:
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-12.4.127-archive.zip"
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-12.4.127-archive.zip"
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-12.4.127-archive.zip"
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-12.4.127-archive.zip"
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4"
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvml_dev-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_cudart-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvcc-windows-x86_64-12.4.131-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvrtc-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
@@ -85,7 +89,9 @@ runs:
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.1.80-archive.zip"
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.1.68-archive.zip"
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-13.1.78-archive.zip"
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.1.68-archive.zip"
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1"
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvml_dev-windows-x86_64-13.1.68-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_crt-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_cudart-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvcc-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
@@ -116,7 +122,9 @@ runs:
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.5.25-archive.zip"
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.5.25-archive.zip"
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.5.0.6-archive.zip"
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.5.37-archive.zip"
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_cudart-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
@@ -143,7 +151,9 @@ runs:
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.5.37-archive.zip"
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.5.37-archive.zip"
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.10.0.20-archive.zip"
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-arm64/cuda_nvml_dev-windows-arm64-13.5.37-archive.zip"
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-arm64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cccl-windows-x86_64-13.3.5.0.6-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
+2
View File
@@ -133,6 +133,8 @@ if (CUDAToolkit_FOUND)
${GGML_SOURCES_CUDA}
)
target_link_libraries(ggml-cuda PRIVATE ${CMAKE_DL_LIBS})
if (GGML_CUDA_GRAPHS)
add_compile_definitions(GGML_CUDA_USE_GRAPHS)
endif()
+2 -1
View File
@@ -59,7 +59,7 @@
// While BW spans CC 1000, 1100 & 1200, we are integrating Tensor Core instructions available to 1200 family, see
// https://docs.nvidia.com/cutlass/media/docs/cpp/blackwell_functionality.html#blackwell-sm120-gemms
#define GGML_CUDA_CC_BLACKWELL 1200
#define GGML_CUDA_CC_DGX_SPARK 1210
#define GGML_CUDA_CC_GB10 1210
#define GGML_CUDA_CC_RUBIN 1300
#define GGML_CUDA_CC_OFFSET_AMD 0x1000000
#define GGML_CUDA_CC_OFFSET_MTHREADS 0x0100000
@@ -1293,6 +1293,7 @@ struct ggml_cuda_device_info {
size_t total_vram;
int warp_size; // Number of threads in a dispatch
bool supports_cooperative_launch; // whether cooperative launch is supported
bool prefer_fp32_acc; // prefer BF16 inputs with FP32 accumulation for quantized cuBLAS
int physical_device; // backing physical CUDA device for this (virtual) device
int physical_share_count; // number of (virtual) devices sharing this device's physical GPU
int virtual_index; // index of this (virtual) device among those sharing its physical GPU
+1 -1
View File
@@ -1206,7 +1206,7 @@ void launch_fattn(
dim3 blocks_num;
if (stream_k) {
// Stream-K splits the work before the mask scan is applied, so skipped KV tiles make the blocks uneven.
const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK && async_kv_preload && scan_mask;
const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10 && async_kv_preload && scan_mask;
auto should_use_stream_k = [prefer_whole_tiles](const int cc, const int ntiles_dst, const int max_blocks, const int DKQ) {
const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks;
+1 -1
View File
@@ -397,7 +397,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
if (amd_mfma_available(cc) && ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, 16)) {
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
} else if (gqa_ratio == 20 && GGML_CUDA_CC_IS_NVIDIA(cc)) { // GLM 4.7 Flash
if (cc >= GGML_CUDA_CC_DGX_SPARK) {
if (cc >= GGML_CUDA_CC_GB10) {
if (Q->ne[1] <= 8) {
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
break;
+1 -1
View File
@@ -468,7 +468,7 @@ static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a
// Enable only architectures benchmarked for this schedule. MMA support alone does not imply a speedup.
if (!ampere_mma_available(info.cc) ||
(info.cc != GGML_CUDA_CC_AMPERE_GA10X && info.cc != GGML_CUDA_CC_ADA_LOVELACE &&
info.cc != GGML_CUDA_CC_BLACKWELL && info.cc != GGML_CUDA_CC_DGX_SPARK)) {
info.cc != GGML_CUDA_CC_BLACKWELL && info.cc != GGML_CUDA_CC_GB10)) {
return gdn_path::ar;
}
// Value splitting pays off when whole heads leave at least half the SMs idle.
+79 -1
View File
@@ -96,6 +96,11 @@
#include <string>
#include <vector>
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
#include "ggml-backend-dl.h"
#include <nvml.h>
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");
#define GGML_LOG_WARN_ONCE(str) \
@@ -309,6 +314,64 @@ static ggml_cuda_device_info ggml_cuda_init() {
}
total_vram = 0;
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
#if defined(_WIN32)
// NVML is installed by the driver; its NVSMI directory is not added to PATH on Windows.
dl_handle_ptr library(LoadLibraryExW(L"nvml.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32));
if (!library) {
const DWORD size = GetEnvironmentVariableW(L"ProgramW6432", nullptr, 0);
if (size > 0) {
std::wstring path(size, L'\0');
const DWORD length = GetEnvironmentVariableW(L"ProgramW6432", &path[0], size);
if (length > 0 && length < size) {
path.resize(length);
path += L"\\NVIDIA Corporation\\NVSMI\\nvml.dll";
if (fs::path(path).is_absolute()) {
library.reset(LoadLibraryExW(path.c_str(), nullptr, LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS));
}
}
}
}
#else
dl_handle_ptr library(dlopen("libnvidia-ml.so.1", RTLD_NOW | RTLD_LOCAL));
#endif // defined(_WIN32)
const auto get_symbol = [&](const char * name) {
#if defined(_WIN32)
return library ? GetProcAddress(library.get(), name) : nullptr;
#else
return library ? dlsym(library.get(), name) : nullptr;
#endif // defined(_WIN32)
};
const auto init = reinterpret_cast<decltype(&nvmlInit_v2)>(get_symbol("nvmlInit_v2"));
const auto shutdown = reinterpret_cast<decltype(&nvmlShutdown)>(get_symbol("nvmlShutdown"));
const auto get_device = reinterpret_cast<decltype(&nvmlDeviceGetHandleByPciBusId_v2)>(get_symbol("nvmlDeviceGetHandleByPciBusId_v2"));
const auto get_brand = reinterpret_cast<decltype(&nvmlDeviceGetBrand)>(get_symbol("nvmlDeviceGetBrand"));
const bool nvml_initialized = init && shutdown && get_device && get_brand && init() == NVML_SUCCESS;
const auto should_use_fp32_acc = [&](const cudaDeviceProp & prop, const int cc) {
if (cc == GGML_CUDA_CC_GB10) {
return true;
}
if (!fast_bf16_hardware_available(cc) || !nvml_initialized) {
return false;
}
// CUDA and NVML device indices can differ, e.g. with CUDA_VISIBLE_DEVICES.
char pci_bus_id[32];
snprintf(pci_bus_id, sizeof(pci_bus_id), "%04x:%02x:%02x.0",
(unsigned int) prop.pciDomainID, (unsigned int) prop.pciBusID, (unsigned int) prop.pciDeviceID);
nvmlDevice_t device = nullptr;
nvmlBrandType_t brand = NVML_BRAND_UNKNOWN;
bool prefer_fp32_acc = false;
if (get_device(pci_bus_id, &device) == NVML_SUCCESS && get_brand(device, &brand) == NVML_SUCCESS) {
prefer_fp32_acc = brand == NVML_BRAND_QUADRO ||
brand == NVML_BRAND_QUADRO_RTX ||
brand == NVML_BRAND_NVIDIA_RTX;
}
return prefer_fp32_acc;
};
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
std::vector<std::pair<int, std::string>> turing_devices_without_mma;
for (int id = 0; id < info.device_count; ++id) {
const int physical_id = info.devices[id].physical_device;
@@ -379,6 +442,11 @@ static ggml_cuda_device_info ggml_cuda_init() {
#else
info.devices[id].smpbo = prop.sharedMemPerBlockOptin;
info.devices[id].cc = 100*prop.major + 10*prop.minor;
if (info.devices[id].virtual_index == 0) {
info.devices[id].prefer_fp32_acc = should_use_fp32_acc(prop, info.devices[id].cc);
} else {
info.devices[id].prefer_fp32_acc = info.devices[physical_id].prefer_fp32_acc;
}
GGML_LOG_INFO(" Device %d: %s, compute capability %d.%d, VMM: %s, VRAM: %zu MiB\n",
id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no",
device_vram_mib);
@@ -403,6 +471,13 @@ static ggml_cuda_device_info ggml_cuda_init() {
#endif // defined(GGML_USE_HIP)
}
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
if (nvml_initialized) {
(void) shutdown();
}
library.reset();
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
if (ggml_cuda_highest_compiled_arch(GGML_CUDA_CC_TURING) >= GGML_CUDA_CC_TURING && !turing_devices_without_mma.empty()) {
GGML_LOG_INFO("The following devices will have suboptimal performance due to a lack of tensor cores:\n");
for (size_t device_pos = 0; device_pos < turing_devices_without_mma.size(); device_pos++) {
@@ -1641,7 +1716,7 @@ static void * ggml_backend_cuda_comm_init_rank(ggml_backend_t backend, int rank,
// NCCL_MAX_CTAS still overrides this.
ncclConfig_t config = NCCL_CONFIG_INITIALIZER;
const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc;
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10) {
config.maxCTAs = 8;
}
@@ -2047,6 +2122,9 @@ static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml
ggml_type compute_type = src0->type;
if (ggml_is_quantized(compute_type) || compute_type == GGML_TYPE_F8_E4M3) {
compute_type = fast_fp16_hardware_available(cc) ? GGML_TYPE_F16 : GGML_TYPE_F32;
if (ggml_cuda_info().devices[ctx.device].prefer_fp32_acc) {
compute_type = GGML_TYPE_BF16;
}
} else if (compute_type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc)) {
compute_type = GGML_TYPE_F32;
} else if (compute_type == GGML_TYPE_BF16 && !fast_bf16_hardware_available(cc)) {
+4 -4
View File
@@ -584,7 +584,7 @@ static __global__ void mul_mat_q_few(
ggml_cuda_pdl_sync();
}
#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10
// DGX Spark DRAM serves the short row pieces of the stage copies slowly, so the rows of the block are first prefetched into L2 in address order.
{
const int tid = threadIdx.y*warp_size + threadIdx.x;
@@ -599,7 +599,7 @@ static __global__ void mul_mat_q_few(
}
}
}
#endif // __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
#endif // __CUDA_ARCH__ == GGML_CUDA_CC_GB10
#pragma unroll
for (int stage = 0; stage < nstages - 1; ++stage) {
@@ -880,14 +880,14 @@ static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const
return 0;
}
// on DGX Spark the kernel needs its L2 prefetch, which is only in the DGX Spark device code
if (cc == GGML_CUDA_CC_DGX_SPARK && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_DGX_SPARK) {
if (cc == GGML_CUDA_CC_GB10 && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_GB10) {
return 0;
}
// with fewer than 8 units of 256 values per row the warps idle, MMVQ and MMQ are better there (MUL_MAT_ID splits the rows instead)
if (mmq_few_unit_size(type) == 0 || (!has_ids && ne00 < 8*QK_K)) {
return 0;
}
if (cc == GGML_CUDA_CC_DGX_SPARK) { // tuned on DGX Spark GB10, also for MUL_MAT_ID
if (cc == GGML_CUDA_CC_GB10) { // tuned on DGX Spark GB10, also for MUL_MAT_ID
return type == GGML_TYPE_IQ4_XS || type == GGML_TYPE_IQ4_NL ? 8 : 2;
}
if (has_ids) { // after MMVQ, which is faster up to its max. batch size
+5 -5
View File
@@ -9,7 +9,7 @@
// only enabled on DGX Spark, where it is a gain on every type below. On the higher-bandwidth parts the kernel
// has little exposed latency left to hide and the extra requests cost more than they save.
// For perf data, see https://github.com/ggml-org/llama.cpp/pull/26705#issuecomment-5569335031
#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10
// returns true only for those quants that benefit from prefetch and false otherwise
static constexpr __host__ __device__ bool mmvq_should_prefetch(ggml_type type) {
switch (type) {
@@ -110,7 +110,7 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
return MMVQ_PARAMETERS_GCN;
#elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA && __CUDA_ARCH__ < GGML_CUDA_CC_AMPERE
return MMVQ_PARAMETERS_TURING;
#elif __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
#elif __CUDA_ARCH__ == GGML_CUDA_CC_GB10
return MMVQ_PARAMETERS_GB10;
#else
return MMVQ_PARAMETERS_GENERIC;
@@ -133,7 +133,7 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) {
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA && ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_AMPERE) {
return MMVQ_PARAMETERS_TURING;
}
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_DGX_SPARK) {
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_GB10) {
return MMVQ_PARAMETERS_GB10;
}
return MMVQ_PARAMETERS_GENERIC;
@@ -342,7 +342,7 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
return ne11 <= MMVQ_MAX_BATCH_SIZE;
}
}
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10) {
switch (type) { // tuned on DGX Spark GB10
case GGML_TYPE_Q2_K:
return ne11 <= 6;
@@ -751,7 +751,7 @@ static __global__ void mul_mat_vec_q(
// x block quant index when casting the quants to int
const int kqs = vdr * (tid % (qi/vdr));
#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10
// start the next iterations' weight loads early
if constexpr (mmvq_should_prefetch(type)) {
// loop iterations, not blocks; q3_K and q6_K keep the distance in blocks they had with vdr 1