* Initial CUDA WIP version * Use fp8 intrinsics * Do W8A16 for FP8 in GEMV case (base) osimons@ub-osimons:~/lcpp_fp8.git$ ./build-x64-linux-gcc-reldbg/bin/llama-bench -m /mnt/share/gguf/nvidia/Qwen3.6-27B-NVFP4/Qwen3.6-27B-NVFP4_fp8.gguf -p 8196 -ub 512,2048 -d 0,32000 -n 256 --load-mode dio ggml_cuda_init: found 1 CUDA devices (Total VRAM: 32165 MiB): Device 0: NVIDIA RTX PRO 4500 Blackwell, compute capability 12.0, VMM: yes, VRAM: 32165 MiB | model | size | params | backend | ngl | n_ubatch | lm | test | t/s | | ------------------------------ | ---------: | ---------: | ---------- | --: | -------: | ---------: | --------------: | -------------------: | | qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 512 | dio | pp8196 | 2781.53 ± 8.48 | | qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 512 | dio | tg256 | 37.56 ± 0.02 | | qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 512 | dio | pp8196 @ d32000 | 1616.31 ± 2.61 | | qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 512 | dio | tg256 @ d32000 | 34.11 ± 0.03 | | qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 2048 | dio | pp8196 | 2542.00 ± 0.51 | | qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 2048 | dio | tg256 | 37.36 ± 0.02 | | qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 2048 | dio | pp8196 @ d32000 | 1762.86 ± 1.33 | | qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 2048 | dio | tg256 @ d32000 | 34.09 ± 0.02 | * Optimize GEMV kernel for FP8 * Guard __nv_cvt_fp8x2_to_bf162raw on CTK 13.2 Fixes https://github.com/ggml-org/llama.cpp/pull/28413#discussion_r4044500099 * Remove trailing whitespace * Add supported for strided copies to/from FP8 in CUDA * Prepare Q_reg for FP8 FlashAttention in fattn-vec.cuh * Fix odd-rows for convert/getrows, possible with FP8 * Fix FP8 fallback in cuBlas * Gate mmvf_f8x2_e4m3_to_bf162 on FP8 availability, i.e. CTK >= 11.8 * Restrict (fused) MMVF to 2*ts alignment, and cublas FP8 fo 16-byte * Add todo about accumulator/precision honoring This is beyond the scope of the stacked PR * Avoid races in fp8_fallback * Allow Hopper and other BW SKUs for FP8 * Use intrinsic over explicit manual conversions * Resolve TODO As per spec, we don't have to preserve signs * Add commend on cublaslt workspaces * Use convert_unary_cont_cuda over block dequant in convert.cu * Use get_rows_cuda_float for fp8 kv cache * Make FP8 copies work on all IHV's * More cleanup * Simplify by falling back to shared cuBLAS helper * Fuse weight_scale multiplication for FP8 also * More code-cleanup * Guard __nv_cvt_fp8x2_to_bf162raw on CTK 13.4 Only cvt.rn.bf16x2.e4m3x2 is exposed from 13.2 onwards, so 93e98b9203dfacb85f9ac665f5f0482a8c6b4192 was incorrect fix * Use F32 activations in fallback mmvf path * Explicitly link against cublaslt * Move FP8 conversion intrinsics into ggml_cuda_cast Cleaner this way * WIP src1/prec for fp8 in MMVF * Remove comment * Keep only f8 x f8 KV cache for now * Just do FP8 x FP32 x FP32 in MMVF as its fast enough * Update ggml/src/ggml-cuda/ggml-cuda.cu Co-authored-by: Aman Gupta <amangupta052@gmail.com> * Add batched FP8 GEMMS in cublas as suggested by code review https://github.com/ggml-org/llama.cpp/pull/28413#issuecomment-6676015644 * Extract fp8x2 -> float2 load into helper * Searapte copy checks to/from fp8 * Compiler massaging to avoid repeated address offsets --------- Co-authored-by: Aman Gupta <amangupta052@gmail.com>
129 lines
5.3 KiB
CMake
129 lines
5.3 KiB
CMake
function(ggml_get_flags CCID CCVER)
|
|
set(C_FLAGS "")
|
|
set(CXX_FLAGS "")
|
|
|
|
if (CCID MATCHES "Clang" OR CCID MATCHES "IntelLLVM")
|
|
set(C_FLAGS -Wunreachable-code-break -Wunreachable-code-return)
|
|
set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi)
|
|
|
|
if (CCID STREQUAL "IntelLLVM")
|
|
# for Intel DPC++ and LLVM SYCL, using debug does not disable optimization
|
|
list(APPEND C_FLAGS -Rno-debug-disables-optimization)
|
|
list(APPEND CXX_FLAGS -Rno-debug-disables-optimization)
|
|
endif()
|
|
|
|
if (
|
|
(CCID STREQUAL "Clang" AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR
|
|
(CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0)
|
|
)
|
|
list(APPEND C_FLAGS -Wdouble-promotion)
|
|
endif()
|
|
elseif (CCID STREQUAL "GNU")
|
|
set(C_FLAGS -Wdouble-promotion)
|
|
set(CXX_FLAGS -Wno-array-bounds)
|
|
|
|
if (CCVER VERSION_GREATER_EQUAL 8.1.0)
|
|
list(APPEND CXX_FLAGS -Wextra-semi)
|
|
endif()
|
|
endif()
|
|
|
|
set(GF_C_FLAGS ${C_FLAGS} PARENT_SCOPE)
|
|
set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE)
|
|
endfunction()
|
|
|
|
function(ggml_get_system_arch)
|
|
if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR
|
|
CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR
|
|
(CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64ec" AND MSVC AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang") OR
|
|
(NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
|
|
CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$"))
|
|
set(GGML_SYSTEM_ARCH "ARM" PARENT_SCOPE)
|
|
elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
|
|
CMAKE_GENERATOR_PLATFORM_LWR MATCHES "^(x86_64|i686|amd64|x64|win32)$" OR
|
|
(NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
|
|
CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64|amd64)$"))
|
|
set(GGML_SYSTEM_ARCH "x86" PARENT_SCOPE)
|
|
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc|power")
|
|
set(GGML_SYSTEM_ARCH "PowerPC" PARENT_SCOPE)
|
|
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64")
|
|
set(GGML_SYSTEM_ARCH "loongarch64" PARENT_SCOPE)
|
|
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64")
|
|
set(GGML_SYSTEM_ARCH "riscv64" PARENT_SCOPE)
|
|
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x")
|
|
set(GGML_SYSTEM_ARCH "s390x" PARENT_SCOPE)
|
|
else()
|
|
set(GGML_SYSTEM_ARCH "UNKNOWN" PARENT_SCOPE)
|
|
endif()
|
|
endfunction()
|
|
|
|
# Determines which FlashAttention vector kernel template instances to compile, returns them in OUT_SRCS.
|
|
function(ggml_cuda_fattn_vec_instances DIR OUT_SRCS)
|
|
set(FA_TYPES q4_0 q4_1 q5_0 q5_1 q8_0 bf16 f16 f8_e4m3)
|
|
|
|
string(TOLOWER "${GGML_CUDA_FA_QUANTS}" FA_QUANTS)
|
|
string(STRIP "${FA_QUANTS}" FA_QUANTS)
|
|
if (GGML_CUDA_FA_ALL_QUANTS)
|
|
message(WARNING "GGML_CUDA_FA_ALL_QUANTS is deprecated, use GGML_CUDA_FA_QUANTS=all instead")
|
|
set(FA_QUANTS all)
|
|
endif()
|
|
if (NOT FA_QUANTS)
|
|
message(FATAL_ERROR "GGML_CUDA_FA_QUANTS must not be empty")
|
|
endif()
|
|
|
|
if (FA_QUANTS STREQUAL "all")
|
|
set(FA_COMBINATIONS "")
|
|
foreach (TYPE_V IN LISTS FA_TYPES)
|
|
foreach (TYPE_K IN LISTS FA_TYPES)
|
|
list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V})
|
|
endforeach()
|
|
endforeach()
|
|
else()
|
|
set(FA_COMBINATIONS f16-f16)
|
|
|
|
string(REPLACE "," ";" FA_SELECTED "${FA_QUANTS}")
|
|
foreach (COMBINATION IN LISTS FA_SELECTED)
|
|
string(STRIP "${COMBINATION}" COMBINATION)
|
|
if (NOT COMBINATION MATCHES "^([a-z0-9_]+)-([a-z0-9_]+)$")
|
|
message(FATAL_ERROR "GGML_CUDA_FA_QUANTS: \"${COMBINATION}\" is not \"all\" or a <type_K>-<type_V> combination")
|
|
endif()
|
|
set(TYPE_K ${CMAKE_MATCH_1})
|
|
set(TYPE_V ${CMAKE_MATCH_2})
|
|
foreach (TYPE ${TYPE_K} ${TYPE_V})
|
|
if (NOT TYPE IN_LIST FA_TYPES)
|
|
message(FATAL_ERROR
|
|
"GGML_CUDA_FA_QUANTS: unknown type \"${TYPE}\" in \"${COMBINATION}\", must be one of: ${FA_TYPES}")
|
|
endif()
|
|
endforeach()
|
|
list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V})
|
|
endforeach()
|
|
endif()
|
|
list(REMOVE_DUPLICATES FA_COMBINATIONS)
|
|
|
|
string(REPLACE ";" "," FA_QUANTS_DEFINE "${FA_QUANTS}")
|
|
add_compile_definitions(GGML_CUDA_FA_QUANTS="${FA_QUANTS_DEFINE}")
|
|
foreach (TYPE_V IN LISTS FA_TYPES)
|
|
foreach (TYPE_K IN LISTS FA_TYPES)
|
|
if ("${TYPE_K}-${TYPE_V}" IN_LIST FA_COMBINATIONS)
|
|
set(COMPILED 1)
|
|
else()
|
|
set(COMPILED 0)
|
|
endif()
|
|
string(TOUPPER "GGML_CUDA_FA_${TYPE_K}_${TYPE_V}" COMBINATION_DEF)
|
|
add_compile_definitions(${COMBINATION_DEF}=${COMPILED})
|
|
endforeach()
|
|
endforeach()
|
|
|
|
message(STATUS "FlashAttention K-V type combinations: ${FA_COMBINATIONS}")
|
|
|
|
set(SRCS "")
|
|
foreach (COMBINATION IN LISTS FA_COMBINATIONS)
|
|
set(SRC "${DIR}/template-instances/fattn-vec-instance-${COMBINATION}.cu")
|
|
if (NOT EXISTS "${SRC}")
|
|
message(FATAL_ERROR "FlashAttention template instance \"${SRC}\" does not exist")
|
|
endif()
|
|
list(APPEND SRCS "${SRC}")
|
|
endforeach()
|
|
|
|
set(${OUT_SRCS} ${SRCS} PARENT_SCOPE)
|
|
endfunction()
|