CUDA: optimize Orin single-column MMVQ (#31268)
* CUDA: optimize Orin single-column MMVQ * Create a separate Orin branch in `calc_rows_per_block`
This commit is contained in:
1 parent
9fbe30c2a0
commit
ea84d70903
1 file changed
+45
-1
@@ -96,7 +96,8 @@ enum mmvq_parameter_table_id {
|
||||
MMVQ_PARAMETERS_RDNA2,
|
||||
MMVQ_PARAMETERS_RDNA3_0,
|
||||
MMVQ_PARAMETERS_RDNA4,
|
||||
MMVQ_PARAMETERS_GB10
|
||||
MMVQ_PARAMETERS_GB10,
|
||||
MMVQ_PARAMETERS_ORIN
|
||||
};
|
||||
|
||||
static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
|
||||
@@ -112,6 +113,8 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
|
||||
return MMVQ_PARAMETERS_TURING;
|
||||
#elif __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
return MMVQ_PARAMETERS_GB10;
|
||||
#elif __CUDA_ARCH__ == GGML_CUDA_CC_ORIN
|
||||
return MMVQ_PARAMETERS_ORIN;
|
||||
#else
|
||||
return MMVQ_PARAMETERS_GENERIC;
|
||||
#endif
|
||||
@@ -136,6 +139,9 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_GB10) {
|
||||
return MMVQ_PARAMETERS_GB10;
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_ORIN) {
|
||||
return MMVQ_PARAMETERS_ORIN;
|
||||
}
|
||||
return MMVQ_PARAMETERS_GENERIC;
|
||||
}
|
||||
|
||||
@@ -580,6 +586,9 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d
|
||||
}
|
||||
return generic;
|
||||
}
|
||||
if (table_id == MMVQ_PARAMETERS_ORIN) {
|
||||
return calc_nwarps(type, ncols_dst, MMVQ_PARAMETERS_GENERIC);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -620,6 +629,41 @@ static constexpr __host__ __device__ int calc_rows_per_block(ggml_type type, int
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
if (table_id == MMVQ_PARAMETERS_ORIN) {
|
||||
switch (ncols_dst) {
|
||||
case 1:
|
||||
if (small_k) {
|
||||
return nwarps;
|
||||
}
|
||||
// One row per warp (more rows per block, more loads in flight) for these types, measured on Jetson Orin Nano.
|
||||
// Other types are faster in isolated mat-vec too, but gained at most 2 % end to end.
|
||||
switch (type) {
|
||||
case GGML_TYPE_Q4_0:
|
||||
case GGML_TYPE_Q4_1:
|
||||
case GGML_TYPE_Q5_0:
|
||||
case GGML_TYPE_Q5_1:
|
||||
case GGML_TYPE_Q8_0:
|
||||
case GGML_TYPE_Q4_K:
|
||||
case GGML_TYPE_Q5_K:
|
||||
case GGML_TYPE_Q6_K:
|
||||
case GGML_TYPE_IQ4_NL:
|
||||
return nwarps;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return mmvq_two_rows_ncols1(type) ? 2 : 1;
|
||||
case 2:
|
||||
case 3:
|
||||
case 4:
|
||||
case 5:
|
||||
case 6:
|
||||
case 7:
|
||||
case 8:
|
||||
return 2;
|
||||
default:
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user