[CUDA] : Add SKU-aware FP16/BF16 accumulation precision gating for NVIDIA GPUs (#30672)
* cuda: select cuBLAS precision by GPU SKU Assisted-by: Codex * cuda: fix remaining GB10 macro reference * cuda: review comment addressed, store accumulation preference in device info * Apply batched suggestions from code review Co-authored-by: Johannes Gäßler <johannesg@5d6.de> * addressed Oliver's comment regarding initialize NVML once and install headers in Windows CI * move precision selection helper into init --------- Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
This commit is contained in:
1 parent
6f311d7ad6
commit
2cbfc3cf34
9 files changed
+105
-14
No files matched your search
@@ -25,7 +25,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-11.7.91-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-11.7.101-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-11.7.91-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-11.7.91-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvml_dev-windows-x86_64-11.7.91-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_cudart-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvcc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvrtc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
@@ -54,7 +56,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-12.4.127-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvml_dev-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_cudart-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvcc-windows-x86_64-12.4.131-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvrtc-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
@@ -85,7 +89,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.1.80-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.1.68-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-13.1.78-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.1.68-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvml_dev-windows-x86_64-13.1.68-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_crt-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_cudart-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvcc-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
@@ -116,7 +122,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.5.25-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.5.25-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.5.0.6-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.5.37-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_cudart-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
@@ -143,7 +151,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.10.0.20-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-arm64/cuda_nvml_dev-windows-arm64-13.5.37-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-arm64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cccl-windows-x86_64-13.3.5.0.6-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
|
||||
@@ -133,6 +133,8 @@ if (CUDAToolkit_FOUND)
|
||||
${GGML_SOURCES_CUDA}
|
||||
)
|
||||
|
||||
target_link_libraries(ggml-cuda PRIVATE ${CMAKE_DL_LIBS})
|
||||
|
||||
if (GGML_CUDA_GRAPHS)
|
||||
add_compile_definitions(GGML_CUDA_USE_GRAPHS)
|
||||
endif()
|
||||
|
||||
@@ -59,7 +59,7 @@
|
||||
// While BW spans CC 1000, 1100 & 1200, we are integrating Tensor Core instructions available to 1200 family, see
|
||||
// https://docs.nvidia.com/cutlass/media/docs/cpp/blackwell_functionality.html#blackwell-sm120-gemms
|
||||
#define GGML_CUDA_CC_BLACKWELL 1200
|
||||
#define GGML_CUDA_CC_DGX_SPARK 1210
|
||||
#define GGML_CUDA_CC_GB10 1210
|
||||
#define GGML_CUDA_CC_RUBIN 1300
|
||||
#define GGML_CUDA_CC_OFFSET_AMD 0x1000000
|
||||
#define GGML_CUDA_CC_OFFSET_MTHREADS 0x0100000
|
||||
@@ -1293,6 +1293,7 @@ struct ggml_cuda_device_info {
|
||||
size_t total_vram;
|
||||
int warp_size; // Number of threads in a dispatch
|
||||
bool supports_cooperative_launch; // whether cooperative launch is supported
|
||||
bool prefer_fp32_acc; // prefer BF16 inputs with FP32 accumulation for quantized cuBLAS
|
||||
int physical_device; // backing physical CUDA device for this (virtual) device
|
||||
int physical_share_count; // number of (virtual) devices sharing this device's physical GPU
|
||||
int virtual_index; // index of this (virtual) device among those sharing its physical GPU
|
||||
|
||||
@@ -1206,7 +1206,7 @@ void launch_fattn(
|
||||
dim3 blocks_num;
|
||||
if (stream_k) {
|
||||
// Stream-K splits the work before the mask scan is applied, so skipped KV tiles make the blocks uneven.
|
||||
const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK && async_kv_preload && scan_mask;
|
||||
const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10 && async_kv_preload && scan_mask;
|
||||
|
||||
auto should_use_stream_k = [prefer_whole_tiles](const int cc, const int ntiles_dst, const int max_blocks, const int DKQ) {
|
||||
const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks;
|
||||
|
||||
@@ -397,7 +397,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
|
||||
if (amd_mfma_available(cc) && ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, 16)) {
|
||||
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
|
||||
} else if (gqa_ratio == 20 && GGML_CUDA_CC_IS_NVIDIA(cc)) { // GLM 4.7 Flash
|
||||
if (cc >= GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (cc >= GGML_CUDA_CC_GB10) {
|
||||
if (Q->ne[1] <= 8) {
|
||||
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
|
||||
break;
|
||||
|
||||
@@ -468,7 +468,7 @@ static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a
|
||||
// Enable only architectures benchmarked for this schedule. MMA support alone does not imply a speedup.
|
||||
if (!ampere_mma_available(info.cc) ||
|
||||
(info.cc != GGML_CUDA_CC_AMPERE_GA10X && info.cc != GGML_CUDA_CC_ADA_LOVELACE &&
|
||||
info.cc != GGML_CUDA_CC_BLACKWELL && info.cc != GGML_CUDA_CC_DGX_SPARK)) {
|
||||
info.cc != GGML_CUDA_CC_BLACKWELL && info.cc != GGML_CUDA_CC_GB10)) {
|
||||
return gdn_path::ar;
|
||||
}
|
||||
// Value splitting pays off when whole heads leave at least half the SMs idle.
|
||||
|
||||
@@ -96,6 +96,11 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
#include "ggml-backend-dl.h"
|
||||
#include <nvml.h>
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");
|
||||
|
||||
#define GGML_LOG_WARN_ONCE(str) \
|
||||
@@ -309,6 +314,64 @@ static ggml_cuda_device_info ggml_cuda_init() {
|
||||
}
|
||||
total_vram = 0;
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
#if defined(_WIN32)
|
||||
// NVML is installed by the driver; its NVSMI directory is not added to PATH on Windows.
|
||||
dl_handle_ptr library(LoadLibraryExW(L"nvml.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32));
|
||||
if (!library) {
|
||||
const DWORD size = GetEnvironmentVariableW(L"ProgramW6432", nullptr, 0);
|
||||
if (size > 0) {
|
||||
std::wstring path(size, L'\0');
|
||||
const DWORD length = GetEnvironmentVariableW(L"ProgramW6432", &path[0], size);
|
||||
if (length > 0 && length < size) {
|
||||
path.resize(length);
|
||||
path += L"\\NVIDIA Corporation\\NVSMI\\nvml.dll";
|
||||
if (fs::path(path).is_absolute()) {
|
||||
library.reset(LoadLibraryExW(path.c_str(), nullptr, LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#else
|
||||
dl_handle_ptr library(dlopen("libnvidia-ml.so.1", RTLD_NOW | RTLD_LOCAL));
|
||||
#endif // defined(_WIN32)
|
||||
const auto get_symbol = [&](const char * name) {
|
||||
#if defined(_WIN32)
|
||||
return library ? GetProcAddress(library.get(), name) : nullptr;
|
||||
#else
|
||||
return library ? dlsym(library.get(), name) : nullptr;
|
||||
#endif // defined(_WIN32)
|
||||
};
|
||||
const auto init = reinterpret_cast<decltype(&nvmlInit_v2)>(get_symbol("nvmlInit_v2"));
|
||||
const auto shutdown = reinterpret_cast<decltype(&nvmlShutdown)>(get_symbol("nvmlShutdown"));
|
||||
const auto get_device = reinterpret_cast<decltype(&nvmlDeviceGetHandleByPciBusId_v2)>(get_symbol("nvmlDeviceGetHandleByPciBusId_v2"));
|
||||
const auto get_brand = reinterpret_cast<decltype(&nvmlDeviceGetBrand)>(get_symbol("nvmlDeviceGetBrand"));
|
||||
const bool nvml_initialized = init && shutdown && get_device && get_brand && init() == NVML_SUCCESS;
|
||||
|
||||
const auto should_use_fp32_acc = [&](const cudaDeviceProp & prop, const int cc) {
|
||||
if (cc == GGML_CUDA_CC_GB10) {
|
||||
return true;
|
||||
}
|
||||
if (!fast_bf16_hardware_available(cc) || !nvml_initialized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// CUDA and NVML device indices can differ, e.g. with CUDA_VISIBLE_DEVICES.
|
||||
char pci_bus_id[32];
|
||||
snprintf(pci_bus_id, sizeof(pci_bus_id), "%04x:%02x:%02x.0",
|
||||
(unsigned int) prop.pciDomainID, (unsigned int) prop.pciBusID, (unsigned int) prop.pciDeviceID);
|
||||
nvmlDevice_t device = nullptr;
|
||||
nvmlBrandType_t brand = NVML_BRAND_UNKNOWN;
|
||||
bool prefer_fp32_acc = false;
|
||||
if (get_device(pci_bus_id, &device) == NVML_SUCCESS && get_brand(device, &brand) == NVML_SUCCESS) {
|
||||
prefer_fp32_acc = brand == NVML_BRAND_QUADRO ||
|
||||
brand == NVML_BRAND_QUADRO_RTX ||
|
||||
brand == NVML_BRAND_NVIDIA_RTX;
|
||||
}
|
||||
return prefer_fp32_acc;
|
||||
};
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
std::vector<std::pair<int, std::string>> turing_devices_without_mma;
|
||||
for (int id = 0; id < info.device_count; ++id) {
|
||||
const int physical_id = info.devices[id].physical_device;
|
||||
@@ -379,6 +442,11 @@ static ggml_cuda_device_info ggml_cuda_init() {
|
||||
#else
|
||||
info.devices[id].smpbo = prop.sharedMemPerBlockOptin;
|
||||
info.devices[id].cc = 100*prop.major + 10*prop.minor;
|
||||
if (info.devices[id].virtual_index == 0) {
|
||||
info.devices[id].prefer_fp32_acc = should_use_fp32_acc(prop, info.devices[id].cc);
|
||||
} else {
|
||||
info.devices[id].prefer_fp32_acc = info.devices[physical_id].prefer_fp32_acc;
|
||||
}
|
||||
GGML_LOG_INFO(" Device %d: %s, compute capability %d.%d, VMM: %s, VRAM: %zu MiB\n",
|
||||
id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no",
|
||||
device_vram_mib);
|
||||
@@ -403,6 +471,13 @@ static ggml_cuda_device_info ggml_cuda_init() {
|
||||
#endif // defined(GGML_USE_HIP)
|
||||
}
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
if (nvml_initialized) {
|
||||
(void) shutdown();
|
||||
}
|
||||
library.reset();
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
if (ggml_cuda_highest_compiled_arch(GGML_CUDA_CC_TURING) >= GGML_CUDA_CC_TURING && !turing_devices_without_mma.empty()) {
|
||||
GGML_LOG_INFO("The following devices will have suboptimal performance due to a lack of tensor cores:\n");
|
||||
for (size_t device_pos = 0; device_pos < turing_devices_without_mma.size(); device_pos++) {
|
||||
@@ -1641,7 +1716,7 @@ static void * ggml_backend_cuda_comm_init_rank(ggml_backend_t backend, int rank,
|
||||
// NCCL_MAX_CTAS still overrides this.
|
||||
ncclConfig_t config = NCCL_CONFIG_INITIALIZER;
|
||||
const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc;
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10) {
|
||||
config.maxCTAs = 8;
|
||||
}
|
||||
|
||||
@@ -2047,6 +2122,9 @@ static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml
|
||||
ggml_type compute_type = src0->type;
|
||||
if (ggml_is_quantized(compute_type) || compute_type == GGML_TYPE_F8_E4M3) {
|
||||
compute_type = fast_fp16_hardware_available(cc) ? GGML_TYPE_F16 : GGML_TYPE_F32;
|
||||
if (ggml_cuda_info().devices[ctx.device].prefer_fp32_acc) {
|
||||
compute_type = GGML_TYPE_BF16;
|
||||
}
|
||||
} else if (compute_type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc)) {
|
||||
compute_type = GGML_TYPE_F32;
|
||||
} else if (compute_type == GGML_TYPE_BF16 && !fast_bf16_hardware_available(cc)) {
|
||||
|
||||
@@ -584,7 +584,7 @@ static __global__ void mul_mat_q_few(
|
||||
ggml_cuda_pdl_sync();
|
||||
}
|
||||
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
// DGX Spark DRAM serves the short row pieces of the stage copies slowly, so the rows of the block are first prefetched into L2 in address order.
|
||||
{
|
||||
const int tid = threadIdx.y*warp_size + threadIdx.x;
|
||||
@@ -599,7 +599,7 @@ static __global__ void mul_mat_q_few(
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif // __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#endif // __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
|
||||
#pragma unroll
|
||||
for (int stage = 0; stage < nstages - 1; ++stage) {
|
||||
@@ -880,14 +880,14 @@ static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const
|
||||
return 0;
|
||||
}
|
||||
// on DGX Spark the kernel needs its L2 prefetch, which is only in the DGX Spark device code
|
||||
if (cc == GGML_CUDA_CC_DGX_SPARK && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (cc == GGML_CUDA_CC_GB10 && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_GB10) {
|
||||
return 0;
|
||||
}
|
||||
// with fewer than 8 units of 256 values per row the warps idle, MMVQ and MMQ are better there (MUL_MAT_ID splits the rows instead)
|
||||
if (mmq_few_unit_size(type) == 0 || (!has_ids && ne00 < 8*QK_K)) {
|
||||
return 0;
|
||||
}
|
||||
if (cc == GGML_CUDA_CC_DGX_SPARK) { // tuned on DGX Spark GB10, also for MUL_MAT_ID
|
||||
if (cc == GGML_CUDA_CC_GB10) { // tuned on DGX Spark GB10, also for MUL_MAT_ID
|
||||
return type == GGML_TYPE_IQ4_XS || type == GGML_TYPE_IQ4_NL ? 8 : 2;
|
||||
}
|
||||
if (has_ids) { // after MMVQ, which is faster up to its max. batch size
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
// only enabled on DGX Spark, where it is a gain on every type below. On the higher-bandwidth parts the kernel
|
||||
// has little exposed latency left to hide and the extra requests cost more than they save.
|
||||
// For perf data, see https://github.com/ggml-org/llama.cpp/pull/26705#issuecomment-5569335031
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
// returns true only for those quants that benefit from prefetch and false otherwise
|
||||
static constexpr __host__ __device__ bool mmvq_should_prefetch(ggml_type type) {
|
||||
switch (type) {
|
||||
@@ -110,7 +110,7 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
|
||||
return MMVQ_PARAMETERS_GCN;
|
||||
#elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA && __CUDA_ARCH__ < GGML_CUDA_CC_AMPERE
|
||||
return MMVQ_PARAMETERS_TURING;
|
||||
#elif __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#elif __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
return MMVQ_PARAMETERS_GB10;
|
||||
#else
|
||||
return MMVQ_PARAMETERS_GENERIC;
|
||||
@@ -133,7 +133,7 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA && ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_AMPERE) {
|
||||
return MMVQ_PARAMETERS_TURING;
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_GB10) {
|
||||
return MMVQ_PARAMETERS_GB10;
|
||||
}
|
||||
return MMVQ_PARAMETERS_GENERIC;
|
||||
@@ -342,7 +342,7 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
|
||||
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
||||
}
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10) {
|
||||
switch (type) { // tuned on DGX Spark GB10
|
||||
case GGML_TYPE_Q2_K:
|
||||
return ne11 <= 6;
|
||||
@@ -751,7 +751,7 @@ static __global__ void mul_mat_vec_q(
|
||||
// x block quant index when casting the quants to int
|
||||
const int kqs = vdr * (tid % (qi/vdr));
|
||||
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
// start the next iterations' weight loads early
|
||||
if constexpr (mmvq_should_prefetch(type)) {
|
||||
// loop iterations, not blocks; q3_K and q6_K keep the distance in blocks they had with vdr 1
|
||||
|
||||
Reference in new issue
Block a user