Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cuda: unblock mmq for MoE on sm_60 (#26264)

Browse Source 
* cuda: unblock mmq for MoE on sm_60

* cuda: duplicate mmq-config-pascal for dp4a and older

* cuda: reduce occupancy on non-dp4a pascal for Q2_K, Q4_K, Q5_K, Q6_K
This commit is contained in:
David Friehs
2026-08-26 18:35:54 +08:00
committed by GitHub
parent da9b5d68c3
commit fc35562ba4
4 changed files with 286 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-cuda/mmq.cuh
+9 -3
View File
@@ -213,7 +213,8 @@ struct ggml_cuda_mmq_config {
return ggml_cuda_mmq_config((type_), (nthreads_), (occupancy_), (I_), (J_), (sram_layout_), (K_vram_), (stream_k_), (fallback_)); \
} \
#include "mmq-config-pascal.cuh"
#include "mmq-config-pascal-older.cuh"
#include "mmq-config-pascal-dp4a.cuh"
#include "mmq-config-ampere.cuh"
#include "mmq-config-blackwell.cuh"
@@ -247,7 +248,10 @@ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type ty
if (ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) {
return ggml_cuda_mmq_get_config_ampere(type, J, fallback);
}
return ggml_cuda_mmq_get_config_pascal(type, J, fallback);
if (ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_DP4A) {
return ggml_cuda_mmq_get_config_pascal_dp4a(type, J, fallback);
}
return ggml_cuda_mmq_get_config_pascal_older(type, J, fallback);
}
static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback) {
@@ -268,8 +272,10 @@ static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_t
return ggml_cuda_mmq_get_config_blackwell(type, J, fallback);
#elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA
return ggml_cuda_mmq_get_config_ampere(type, J, fallback);
#elif __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A
return ggml_cuda_mmq_get_config_pascal_dp4a(type, J, fallback);
#else
return ggml_cuda_mmq_get_config_pascal(type, J, fallback);
return ggml_cuda_mmq_get_config_pascal_older(type, J, fallback);
#endif // BLACKWELL_MMA_AVAILABLE
#endif // GGML_USE_HIP
GGML_UNUSED_VARS(type, J, fallback);