diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 612d3e313..091fc3071 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -96,7 +96,8 @@ enum mmvq_parameter_table_id { MMVQ_PARAMETERS_RDNA2, MMVQ_PARAMETERS_RDNA3_0, MMVQ_PARAMETERS_RDNA4, - MMVQ_PARAMETERS_GB10 + MMVQ_PARAMETERS_GB10, + MMVQ_PARAMETERS_ORIN }; static constexpr __device__ mmvq_parameter_table_id get_device_table_id() { @@ -112,6 +113,8 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() { return MMVQ_PARAMETERS_TURING; #elif __CUDA_ARCH__ == GGML_CUDA_CC_GB10 return MMVQ_PARAMETERS_GB10; +#elif __CUDA_ARCH__ == GGML_CUDA_CC_ORIN + return MMVQ_PARAMETERS_ORIN; #else return MMVQ_PARAMETERS_GENERIC; #endif @@ -136,6 +139,9 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) { if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_GB10) { return MMVQ_PARAMETERS_GB10; } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_ORIN) { + return MMVQ_PARAMETERS_ORIN; + } return MMVQ_PARAMETERS_GENERIC; } @@ -580,6 +586,9 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d } return generic; } + if (table_id == MMVQ_PARAMETERS_ORIN) { + return calc_nwarps(type, ncols_dst, MMVQ_PARAMETERS_GENERIC); + } return 1; } @@ -620,6 +629,41 @@ static constexpr __host__ __device__ int calc_rows_per_block(ggml_type type, int return 1; } } + if (table_id == MMVQ_PARAMETERS_ORIN) { + switch (ncols_dst) { + case 1: + if (small_k) { + return nwarps; + } + // One row per warp (more rows per block, more loads in flight) for these types, measured on Jetson Orin Nano. + // Other types are faster in isolated mat-vec too, but gained at most 2 % end to end. + switch (type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ4_NL: + return nwarps; + default: + break; + } + return mmvq_two_rows_ncols1(type) ? 2 : 1; + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + return 2; + default: + return 1; + } + } return 1; }