Files
llama.cpp/ggml/cmake/common.cmake
T
Oliver SimonsandAman Gupta 9e9619d030 CUDA: Add OCP FP8 support (#28413)
* Initial CUDA WIP version

* Use fp8 intrinsics

* Do W8A16 for FP8 in GEMV case

(base) osimons@ub-osimons:~/lcpp_fp8.git$ ./build-x64-linux-gcc-reldbg/bin/llama-bench -m /mnt/share/gguf/nvidia/Qwen3.6-27B-NVFP4/Qwen3.6-27B-NVFP4_fp8.gguf -p 8196 -ub 512,2048 -d 0,32000 -n 256 --load-mode dio
ggml_cuda_init: found 1 CUDA devices (Total VRAM: 32165 MiB):
  Device 0: NVIDIA RTX PRO 4500 Blackwell, compute capability 12.0, VMM: yes, VRAM: 32165 MiB
| model                          |       size |     params | backend    | ngl | n_ubatch |         lm |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -------: | ---------: | --------------: | -------------------: |
| qwen35 27B NVFP4               |  19.56 GiB |    27.32 B | CUDA       |  -1 |      512 |        dio |          pp8196 |       2781.53 ± 8.48 |
| qwen35 27B NVFP4               |  19.56 GiB |    27.32 B | CUDA       |  -1 |      512 |        dio |           tg256 |         37.56 ± 0.02 |
| qwen35 27B NVFP4               |  19.56 GiB |    27.32 B | CUDA       |  -1 |      512 |        dio | pp8196 @ d32000 |       1616.31 ± 2.61 |
| qwen35 27B NVFP4               |  19.56 GiB |    27.32 B | CUDA       |  -1 |      512 |        dio |  tg256 @ d32000 |         34.11 ± 0.03 |
| qwen35 27B NVFP4               |  19.56 GiB |    27.32 B | CUDA       |  -1 |     2048 |        dio |          pp8196 |       2542.00 ± 0.51 |
| qwen35 27B NVFP4               |  19.56 GiB |    27.32 B | CUDA       |  -1 |     2048 |        dio |           tg256 |         37.36 ± 0.02 |
| qwen35 27B NVFP4               |  19.56 GiB |    27.32 B | CUDA       |  -1 |     2048 |        dio | pp8196 @ d32000 |       1762.86 ± 1.33 |
| qwen35 27B NVFP4               |  19.56 GiB |    27.32 B | CUDA       |  -1 |     2048 |        dio |  tg256 @ d32000 |         34.09 ± 0.02 |

* Optimize GEMV kernel for FP8

* Guard __nv_cvt_fp8x2_to_bf162raw on CTK 13.2

Fixes https://github.com/ggml-org/llama.cpp/pull/28413#discussion_r4044500099

* Remove trailing whitespace

* Add supported for strided copies to/from FP8 in CUDA

* Prepare Q_reg for FP8 FlashAttention in fattn-vec.cuh

* Fix odd-rows for convert/getrows, possible with FP8

* Fix FP8 fallback in cuBlas

* Gate mmvf_f8x2_e4m3_to_bf162 on FP8 availability, i.e. CTK >= 11.8

* Restrict (fused) MMVF to 2*ts alignment, and cublas FP8 fo 16-byte

* Add todo about accumulator/precision honoring

This is beyond the scope of the stacked PR

* Avoid races in fp8_fallback

* Allow Hopper and other BW SKUs for FP8

* Use intrinsic over explicit manual conversions

* Resolve TODO

As per spec, we don't have to preserve signs

* Add commend on cublaslt workspaces

* Use convert_unary_cont_cuda over block dequant in convert.cu

* Use get_rows_cuda_float for fp8 kv cache

* Make FP8 copies work on all IHV's

* More cleanup

* Simplify by falling back to shared cuBLAS helper

* Fuse weight_scale multiplication for FP8 also

* More code-cleanup

* Guard __nv_cvt_fp8x2_to_bf162raw on CTK 13.4

Only cvt.rn.bf16x2.e4m3x2 is exposed from 13.2 onwards, so
93e98b9203dfacb85f9ac665f5f0482a8c6b4192 was incorrect fix

* Use F32 activations in fallback mmvf path

* Explicitly link against cublaslt

* Move FP8 conversion intrinsics into ggml_cuda_cast

Cleaner this way

* WIP src1/prec for fp8 in MMVF

* Remove comment

* Keep only f8 x f8 KV cache for now

* Just do FP8 x FP32 x FP32 in MMVF as its fast enough

* Update ggml/src/ggml-cuda/ggml-cuda.cu

Co-authored-by: Aman Gupta <amangupta052@gmail.com>

* Add batched FP8 GEMMS in cublas as suggested by code review

https://github.com/ggml-org/llama.cpp/pull/28413#issuecomment-6676015644

* Extract fp8x2 -> float2 load into helper

* Searapte copy checks to/from fp8

* Compiler massaging to avoid repeated address offsets

---------

Co-authored-by: Aman Gupta <amangupta052@gmail.com>
2026-11-11 10:44:30 +02:00

129 lines
5.3 KiB
CMake

function(ggml_get_flags CCID CCVER)
set(C_FLAGS "")
set(CXX_FLAGS "")
if (CCID MATCHES "Clang" OR CCID MATCHES "IntelLLVM")
set(C_FLAGS -Wunreachable-code-break -Wunreachable-code-return)
set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi)
if (CCID STREQUAL "IntelLLVM")
# for Intel DPC++ and LLVM SYCL, using debug does not disable optimization
list(APPEND C_FLAGS -Rno-debug-disables-optimization)
list(APPEND CXX_FLAGS -Rno-debug-disables-optimization)
endif()
if (
(CCID STREQUAL "Clang" AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR
(CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0)
)
list(APPEND C_FLAGS -Wdouble-promotion)
endif()
elseif (CCID STREQUAL "GNU")
set(C_FLAGS -Wdouble-promotion)
set(CXX_FLAGS -Wno-array-bounds)
if (CCVER VERSION_GREATER_EQUAL 8.1.0)
list(APPEND CXX_FLAGS -Wextra-semi)
endif()
endif()
set(GF_C_FLAGS ${C_FLAGS} PARENT_SCOPE)
set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE)
endfunction()
function(ggml_get_system_arch)
if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR
CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR
(CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64ec" AND MSVC AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang") OR
(NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$"))
set(GGML_SYSTEM_ARCH "ARM" PARENT_SCOPE)
elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
CMAKE_GENERATOR_PLATFORM_LWR MATCHES "^(x86_64|i686|amd64|x64|win32)$" OR
(NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64|amd64)$"))
set(GGML_SYSTEM_ARCH "x86" PARENT_SCOPE)
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc|power")
set(GGML_SYSTEM_ARCH "PowerPC" PARENT_SCOPE)
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64")
set(GGML_SYSTEM_ARCH "loongarch64" PARENT_SCOPE)
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64")
set(GGML_SYSTEM_ARCH "riscv64" PARENT_SCOPE)
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x")
set(GGML_SYSTEM_ARCH "s390x" PARENT_SCOPE)
else()
set(GGML_SYSTEM_ARCH "UNKNOWN" PARENT_SCOPE)
endif()
endfunction()
# Determines which FlashAttention vector kernel template instances to compile, returns them in OUT_SRCS.
function(ggml_cuda_fattn_vec_instances DIR OUT_SRCS)
set(FA_TYPES q4_0 q4_1 q5_0 q5_1 q8_0 bf16 f16 f8_e4m3)
string(TOLOWER "${GGML_CUDA_FA_QUANTS}" FA_QUANTS)
string(STRIP "${FA_QUANTS}" FA_QUANTS)
if (GGML_CUDA_FA_ALL_QUANTS)
message(WARNING "GGML_CUDA_FA_ALL_QUANTS is deprecated, use GGML_CUDA_FA_QUANTS=all instead")
set(FA_QUANTS all)
endif()
if (NOT FA_QUANTS)
message(FATAL_ERROR "GGML_CUDA_FA_QUANTS must not be empty")
endif()
if (FA_QUANTS STREQUAL "all")
set(FA_COMBINATIONS "")
foreach (TYPE_V IN LISTS FA_TYPES)
foreach (TYPE_K IN LISTS FA_TYPES)
list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V})
endforeach()
endforeach()
else()
set(FA_COMBINATIONS f16-f16)
string(REPLACE "," ";" FA_SELECTED "${FA_QUANTS}")
foreach (COMBINATION IN LISTS FA_SELECTED)
string(STRIP "${COMBINATION}" COMBINATION)
if (NOT COMBINATION MATCHES "^([a-z0-9_]+)-([a-z0-9_]+)$")
message(FATAL_ERROR "GGML_CUDA_FA_QUANTS: \"${COMBINATION}\" is not \"all\" or a <type_K>-<type_V> combination")
endif()
set(TYPE_K ${CMAKE_MATCH_1})
set(TYPE_V ${CMAKE_MATCH_2})
foreach (TYPE ${TYPE_K} ${TYPE_V})
if (NOT TYPE IN_LIST FA_TYPES)
message(FATAL_ERROR
"GGML_CUDA_FA_QUANTS: unknown type \"${TYPE}\" in \"${COMBINATION}\", must be one of: ${FA_TYPES}")
endif()
endforeach()
list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V})
endforeach()
endif()
list(REMOVE_DUPLICATES FA_COMBINATIONS)
string(REPLACE ";" "," FA_QUANTS_DEFINE "${FA_QUANTS}")
add_compile_definitions(GGML_CUDA_FA_QUANTS="${FA_QUANTS_DEFINE}")
foreach (TYPE_V IN LISTS FA_TYPES)
foreach (TYPE_K IN LISTS FA_TYPES)
if ("${TYPE_K}-${TYPE_V}" IN_LIST FA_COMBINATIONS)
set(COMPILED 1)
else()
set(COMPILED 0)
endif()
string(TOUPPER "GGML_CUDA_FA_${TYPE_K}_${TYPE_V}" COMBINATION_DEF)
add_compile_definitions(${COMBINATION_DEF}=${COMPILED})
endforeach()
endforeach()
message(STATUS "FlashAttention K-V type combinations: ${FA_COMBINATIONS}")
set(SRCS "")
foreach (COMBINATION IN LISTS FA_COMBINATIONS)
set(SRC "${DIR}/template-instances/fattn-vec-instance-${COMBINATION}.cu")
if (NOT EXISTS "${SRC}")
message(FATAL_ERROR "FlashAttention template instance \"${SRC}\" does not exist")
endif()
list(APPEND SRCS "${SRC}")
endforeach()
set(${OUT_SRCS} ${SRCS} PARENT_SCOPE)
endfunction()