CUDA: Improve NVFP4 W4A4 activation quantization (#25730)
* Squash history before conflict-resolution during rebase on master WIP commit Add 32-byte loads, restore per-block amax Use nvfp4x4 intrinsic when available Fuse per-channel amax and quantization kernels Do pointer arithmetic only once on x Remove unnecessary ternary in the load We assert on host side that ne00 is 64-aligned Add back scale-search, but optimize it with intrinsics Code cleanup Make scale in MMQ-epilogue NVFP4-specific/restrictive for now Remove unneeded include, add comment Fix trailing whitespace Guard __builtin_align__(32) struct to NVIDIA Seems like HIP doesn't have this available, see https://github.com/ggml-org/llama.cpp/actions/runs/29438651734/job/87431623001 * compiler massaging to avoid unnecessary LDCs * kvalues_mxfp4 -> kvalues_nvfp4 in quantize_mmq_nvfp4 * Always pass in src1_scale.ptr * Extract ggml_cuda_is_aligned helper
This commit is contained in:
@@ -362,6 +362,15 @@ static bool blackwell_mma_available(const int cc) {
|
||||
ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_RUBIN;
|
||||
}
|
||||
|
||||
// Checks whether the tensor's base data pointer and higher-dimensional strides are byte-aligned to `alignment` bytes.
|
||||
static bool ggml_cuda_is_aligned(const ggml_tensor * tensor, const size_t alignment) {
|
||||
GGML_ASSERT(tensor != nullptr);
|
||||
return (reinterpret_cast<uintptr_t>(tensor->data) % alignment) == 0 &&
|
||||
tensor->nb[1] % alignment == 0 &&
|
||||
tensor->nb[2] % alignment == 0 &&
|
||||
tensor->nb[3] % alignment == 0;
|
||||
}
|
||||
|
||||
static constexpr __device__ int ggml_cuda_get_physical_warp_size() {
|
||||
#if defined(GGML_USE_HIP) && (defined(__GFX9__) || defined(__GFX8__))
|
||||
return 64;
|
||||
|
||||
Reference in New Issue
Block a user