CUDA: optimize Orin single-column MMVQ (#31268)

* CUDA: optimize Orin single-column MMVQ

* Create a separate Orin branch in `calc_rows_per_block`
This commit is contained in:
Teng Mei authored and GitHub committed 2026-11-20 09:44:04 +02:00
1 parent 9fbe30c2a0
commit ea84d70903
1 file changed
+45 -1
+45 -1
View File
@@ -96,7 +96,8 @@ enum mmvq_parameter_table_id {
MMVQ_PARAMETERS_RDNA2,
MMVQ_PARAMETERS_RDNA3_0,
MMVQ_PARAMETERS_RDNA4,
MMVQ_PARAMETERS_GB10
MMVQ_PARAMETERS_GB10,
MMVQ_PARAMETERS_ORIN
};
static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
@@ -112,6 +113,8 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
return MMVQ_PARAMETERS_TURING;
#elif __CUDA_ARCH__ == GGML_CUDA_CC_GB10
return MMVQ_PARAMETERS_GB10;
#elif __CUDA_ARCH__ == GGML_CUDA_CC_ORIN
return MMVQ_PARAMETERS_ORIN;
#else
return MMVQ_PARAMETERS_GENERIC;
#endif
@@ -136,6 +139,9 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) {
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_GB10) {
return MMVQ_PARAMETERS_GB10;
}
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_ORIN) {
return MMVQ_PARAMETERS_ORIN;
}
return MMVQ_PARAMETERS_GENERIC;
}
@@ -580,6 +586,9 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d
}
return generic;
}
if (table_id == MMVQ_PARAMETERS_ORIN) {
return calc_nwarps(type, ncols_dst, MMVQ_PARAMETERS_GENERIC);
}
return 1;
}
@@ -620,6 +629,41 @@ static constexpr __host__ __device__ int calc_rows_per_block(ggml_type type, int
return 1;
}
}
if (table_id == MMVQ_PARAMETERS_ORIN) {
switch (ncols_dst) {
case 1:
if (small_k) {
return nwarps;
}
// One row per warp (more rows per block, more loads in flight) for these types, measured on Jetson Orin Nano.
// Other types are faster in isolated mat-vec too, but gained at most 2 % end to end.
switch (type) {
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q5_K:
case GGML_TYPE_Q6_K:
case GGML_TYPE_IQ4_NL:
return nwarps;
default:
break;
}
return mmvq_two_rows_ncols1(type) ? 2 : 1;
case 2:
case 3:
case 4:
case 5:
case 6:
case 7:
case 8:
return 2;
default:
return 1;
}
}
return 1;
}