* model: add support for extra bufs for all devices * hexagon: add experimental ggml-hexagon backend for the Hexagon NPU This commit introduces a new experimental backend `ggml-hexagon` with support for the Hexagon NPU. Highlights: - Supports Hexagon versions: v73, v75, v79, and v81 - Targets Android devices based on Snapdragon SoCs: Gen3, 8-Elite, and 8-Elite Gen5 - Supports Q4_0, Q8_0, MXFP4, and FP32 data types - Implements core LLM ops: MUL_MAT/MUL_MAT_ID, ADD/SUB/MUL/ADD_ID, RMS_NORM, ROPE, GLU/SWIGLU, SOFTMAX **Note:** This backend is experimental and may exhibit instability or limited performance across supported devices. It is intended for early testing and feedback from llama.cpp/ggml developer and user community. Co-Authored-By: Rajdeep Ganguly <rganguly@qti.qualcomm.com> Co-Authored-By: Todor Boinovski <todorb@qti.qualcomm.com> * hexagon: fix format checker errors * hexagon: update readme and cmake presets * ci: add android-ndk-build jobs that build plain ARM64 and Snapdragon versions * hexagon: add simple graph optimizer for stacking MUL_MAT ops with the same input * hexagon: move ADB helper scripts into scripts/snapdragon/adb * hexagon: replace all f/printfs with GGML_LOG_... * readme: add hexagon to the list supported backends * hexagon: stack malmuts with quantized inputs only * hexagon: add TODO for fixing issues in hexagon_graph_optimize * hexagon: update to hex-sdk 6.4.0 and add scripts for running on QDC * scripts: fix lint errors * scripts: update qdc pytest script to make linter happy * hexagon: add reduce sum in fp32 * hexagon: reduce number of vector stores in matmul output * hexagon: remove the need for vdelta in reduce-multiply-x8 * hexagon: consistent use of reduce_sum_fp32 for row_sums * hexagon: some more matmul optimizations and comments Optimize cases where tensor dims are not multiple of 1024 (e.g in Qwen models). We've handled those cases already but at a higher overhead. * hexagon: update cmake presets * hexagon: add OPMASK support for run-bench.sh wrapper * hexagon: update to use GGML_BACKEND_API * hexagon: remove unused logic for setting tensor flags for the views * hexagon: add asserts to set/get_tensor to make sure we handle complete tensors Same asserts as the CPU backend. * hexagon: use cpy_tensor slow path for non-host buffers * hexagon: error checks in the buffer allocator * cmake: move include(extProj) under ggml-hexagon * hexagon: don't forget to delete the backend on free * hexagon: set/get_tensor size assert apply only to quantized tensors * hexagon: reintroduce HEX_VERBOSE wrapper for GGML_LOG_DEBUG for now GGML_LOG_DEBUG is always enabled for test-backend-ops and the output gets in the way. Ideally we need a bit more finer log levels. * docs: typos in hexagon developer docs (libggm-...) * hexagon: overhaul error handling in the session/device allocation this should handle all failure paths in the session allocation. * hexagon: update cmake presets to enable fp16 vectors * hexagon: remove unused time_usec function * hexagon: don't forget to release buffer contexts * hexagon: fixed indents in hvx-utils (missed clang-format auto-format failure) * hexagon: remove custom can_repeat function and use ggml_can_repeat --------- Co-authored-by: Rajdeep Ganguly <rganguly@qti.qualcomm.com> Co-authored-by: Todor Boinovski <todorb@qti.qualcomm.com>
158 lines
5.4 KiB
CMake
158 lines
5.4 KiB
CMake
if (HEXAGON_TOOLCHAIN_INCLUDED)
|
|
return()
|
|
endif()
|
|
set(HEXAGON_TOOLCHAIN_INCLUDED true)
|
|
|
|
#Cross Compiling for Hexagon
|
|
set(HEXAGON TRUE)
|
|
set(CMAKE_SYSTEM_NAME QURT)
|
|
set(CMAKE_SYSTEM_PROCESSOR Hexagon)
|
|
set(CMAKE_SYSTEM_VERSION "1") #${HEXAGON_PLATFORM_LEVEL})
|
|
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
|
|
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
|
|
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
|
|
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
|
|
set(CUSTOM_RUNELF_PATH "")
|
|
|
|
#To fix backward compatibility with EAI addon.
|
|
if (NOT HEXAGON_SDK_ROOT)
|
|
set(HEXAGON_SDK_ROOT $ENV{HEXAGON_SDK_ROOT})
|
|
endif()
|
|
|
|
if (NOT HEXAGON_TOOLS_ROOT)
|
|
if (DEFINED ENV{HEXAGON_TOOLS_ROOT})
|
|
set(HEXAGON_TOOLS_ROOT $ENV{HEXAGON_TOOLS_ROOT})
|
|
endif()
|
|
if(NOT HEXAGON_TOOLS_ROOT)
|
|
set(HEXAGON_TOOLS_ROOT $ENV{DEFAULT_HEXAGON_TOOLS_ROOT})
|
|
endif()
|
|
endif()
|
|
|
|
file(TO_CMAKE_PATH "${HEXAGON_TOOLS_ROOT}" HEXAGON_TOOLS_ROOT)
|
|
file(TO_CMAKE_PATH "${HEXAGON_SDK_ROOT}" HEXAGON_SDK_ROOT)
|
|
|
|
#Get the Binary extension of the Hexagon Toolchain
|
|
if(CMAKE_HOST_SYSTEM_NAME STREQUAL Windows)
|
|
set(HEXAGON_TOOLCHAIN_SUFFIX .exe)
|
|
endif()
|
|
message(DEBUG "CMAKE_HOST_SYSTEM_NAME:${CMAKE_HOST_SYSTEM_NAME}")
|
|
|
|
include(${HEXAGON_SDK_ROOT}/build/cmake/hexagon_arch.cmake)
|
|
|
|
set(HEXAGON_TOOLCHAIN ${HEXAGON_TOOLS_ROOT})
|
|
set(HEXAGON_LIB_DIR "${HEXAGON_TOOLCHAIN}/Tools/target/hexagon/lib")
|
|
set(HEXAGON_ISS_DIR ${HEXAGON_TOOLCHAIN}/Tools/lib/iss)
|
|
|
|
set(CMAKE_TRY_COMPILE_PLATFORM_VARIABLES
|
|
HEXAGON_SDK_ROOT
|
|
HEXAGON_TOOLS_ROOT
|
|
)
|
|
|
|
#QURT Related includes and linker flags
|
|
set(V_ARCH ${HEXAGON_ARCH})
|
|
set(_QURT_INSTALL_DIR "${HEXAGON_SDK_ROOT}/rtos/qurt/ADSP${V_ARCH}MP${V_ARCH_EXTN}")
|
|
set(_QURT_INSTALL_DIR "${HEXAGON_SDK_ROOT}/rtos/qurt/compute${V_ARCH}${V_ARCH_EXTN}")
|
|
|
|
if( ${TREE} MATCHES PAKMAN )
|
|
set(_QURT_INSTALL_DIR "${QURT_IMAGE_DIR}/compute${V_ARCH}${V_ARCH_EXTN}")
|
|
endif()
|
|
message(DEBUG "_QURT_INSTALL_DIR:${_QURT_INSTALL_DIR}")
|
|
set(RTOS_DIR ${_QURT_INSTALL_DIR})
|
|
set(QCC_DIR "${HEXAGON_QCC_DIR}/${V_ARCH}/G0")
|
|
set(TARGET_DIR "${HEXAGON_LIB_DIR}/${V_ARCH}/G0")
|
|
|
|
include_directories(
|
|
${_QURT_INSTALL_DIR}/include
|
|
${_QURT_INSTALL_DIR}/include/qurt
|
|
${_QURT_INSTALL_DIR}/include/posix
|
|
)
|
|
|
|
set(QURT_START_LINK_LIBS)
|
|
set(QURT_START_LINK_LIBS
|
|
"${TARGET_DIR}/init.o"
|
|
"${RTOS_DIR}/lib/crt1.o"
|
|
"${RTOS_DIR}/lib/debugmon.o"
|
|
"${RTOS_DIR}/lib/libqurt.a"
|
|
"${TARGET_DIR}/libc.a"
|
|
"${TARGET_DIR}/libqcc.a"
|
|
"${TARGET_DIR}/libhexagon.a"
|
|
"${RTOS_DIR}/lib/libqurtcfs.a"
|
|
"${RTOS_DIR}/lib/libtimer_island.a"
|
|
"${RTOS_DIR}/lib/libtimer_main.a"
|
|
"${RTOS_DIR}/lib/libposix.a"
|
|
)
|
|
STRING(REPLACE ";" " " QURT_START_LINK_LIBS "${QURT_START_LINK_LIBS}")
|
|
|
|
set(QURT_END_LINK_LIBS
|
|
${TARGET_DIR}/fini.o
|
|
)
|
|
|
|
#Non QURT related includes and linker flags
|
|
|
|
set(TARGET_DIR_NOOS "${HEXAGON_TOOLCHAIN}/Tools/target/hexagon/lib/${HEXAGON_ARCH}")
|
|
|
|
if (NOT NO_WRAP_MEM_API)
|
|
set(WRAP_MALLOC -Wl,--wrap=malloc)
|
|
set(WRAP_CALLOC -Wl,--wrap=calloc)
|
|
set(WRAP_FREE -Wl,--wrap=free)
|
|
set(WRAP_REALLOC -Wl,--wrap=realloc)
|
|
set(WRAP_MEMALIGN -Wl,--wrap=memalign)
|
|
endif()
|
|
|
|
set(PIC_SHARED_LD_FLAGS
|
|
-mcpu=${V_ARCH} -m${V_ARCH} -mhvx=${V_ARCH}
|
|
-G0
|
|
-fpic
|
|
-Wl,-Bsymbolic
|
|
-Wl,-L${TARGET_DIR_NOOS}/G0/pic
|
|
-Wl,-L${HEXAGON_TOOLCHAIN}/Tools/target/hexagon/lib/
|
|
-Wl,--no-threads ${WRAP_MALLOC} ${WRAP_CALLOC} ${WRAP_FREE} ${WRAP_REALLOC} ${WRAP_MEMALIGN}
|
|
-shared
|
|
"-o <TARGET> <SONAME_FLAG><TARGET_SONAME>"
|
|
"<LINK_FLAGS>"
|
|
-Wl,--start-group
|
|
"<OBJECTS>"
|
|
"<LINK_LIBRARIES>"
|
|
-Wl,--end-group
|
|
-lc
|
|
)
|
|
STRING(REPLACE ";" " " PIC_SHARED_LD_FLAGS "${PIC_SHARED_LD_FLAGS}")
|
|
|
|
set(HEXAGON_PIC_SHARED_LINK_OPTIONS "${PIC_SHARED_LD_FLAGS}")
|
|
|
|
#System include paths
|
|
include_directories(SYSTEM ${HEXAGON_SDK_ROOT}/incs)
|
|
include_directories(SYSTEM ${HEXAGON_SDK_ROOT}/incs/stddef)
|
|
include_directories(SYSTEM ${HEXAGON_SDK_ROOT}/ipc/fastrpc/incs)
|
|
|
|
#LLVM toolchain setup
|
|
#Compiler paths, options and architecture
|
|
set(CMAKE_C_COMPILER ${HEXAGON_TOOLCHAIN}/Tools/bin/hexagon-clang${HEXAGON_TOOLCHAIN_SUFFIX})
|
|
set(CMAKE_CXX_COMPILER ${HEXAGON_TOOLCHAIN}/Tools/bin/hexagon-clang++${HEXAGON_TOOLCHAIN_SUFFIX})
|
|
set(CMAKE_AR ${HEXAGON_TOOLCHAIN}/Tools/bin/hexagon-ar${HEXAGON_TOOLCHAIN_SUFFIX})
|
|
set(CMAKE_ASM_COMPILER ${HEXAGON_TOOLCHAIN}/Tools/bin/hexagon-clang++${HEXAGON_TOOLCHAIN_SUFFIX})
|
|
set(HEXAGON_LINKER ${CMAKE_C_COMPILER})
|
|
set(CMAKE_PREFIX_PATH ${HEXAGON_TOOLCHAIN}/Tools/target/hexagon)
|
|
|
|
set(CMAKE_SHARED_LIBRARY_SONAME_C_FLAG "-Wl,-soname,")
|
|
set(CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG "-Wl,-soname,")
|
|
|
|
#Compiler Options
|
|
set(COMMON_FLAGS "-mcpu=hexagon${V_ARCH} -m${V_ARCH} -mhvx=${V_ARCH} -fvectorize -Wall -Werror -fno-zero-initialized-in-bss -G0 -fdata-sections -fpic ${XQF_ARGS}")
|
|
|
|
set(CMAKE_CXX_FLAGS_DEBUG "${COMMON_FLAGS} -O0 -D_DEBUG -g")
|
|
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${COMMON_FLAGS} -O3 -g")
|
|
set(CMAKE_CXX_FLAGS_RELEASE "${COMMON_FLAGS} -O3")
|
|
|
|
set(CMAKE_C_FLAGS_DEBUG "${COMMON_FLAGS} -O0 -D_DEBUG -g")
|
|
set(CMAKE_C_FLAGS_RELWITHDEBINFO "${COMMON_FLAGS} -O3 -g")
|
|
set(CMAKE_C_FLAGS_RELEASE "${COMMON_FLAGS} -O3")
|
|
|
|
set(CMAKE_ASM_FLAGS_DEBUG "${COMMON_FLAGS} ${CMAKE_CXX_FLAGS_DEBUG}")
|
|
set(CMAKE_ASM_FLAGS_RELEASE "${COMMON_FLAGS} ${CMAKE_CXX_FLAGS_RELEASE}")
|
|
set(CMAKE_ASM_FLAGS_RELWITHDEBINFO "${COMMON_FLAGS} ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}" )
|
|
|
|
#Linker Options
|
|
set(CMAKE_C_CREATE_SHARED_LIBRARY "${HEXAGON_LINKER} ${HEXAGON_PIC_SHARED_LINK_OPTIONS}")
|
|
set(CMAKE_CXX_CREATE_SHARED_LIBRARY "${HEXAGON_LINKER} ${HEXAGON_PIC_SHARED_LINK_OPTIONS}")
|