diff --git a/docs/backend/snapdragon/CMakeUserPresets.json b/docs/backend/snapdragon/CMakeUserPresets.json index 848d735f1..afc739232 100644 --- a/docs/backend/snapdragon/CMakeUserPresets.json +++ b/docs/backend/snapdragon/CMakeUserPresets.json @@ -8,7 +8,7 @@ "toolset": { "value": "host=x86_64", "strategy": "external" }, "cacheVariables": { "ANDROID_ABI": "arm64-v8a", - "ANDROID_PLATFORM": "android-31", + "ANDROID_PLATFORM": "android-34", "CMAKE_TOOLCHAIN_FILE": "$env{ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake", "CMAKE_C_FLAGS": "-march=armv8.7a+fp16+dotprod+i8mm -fvectorize -ffp-model=fast -fno-finite-math-only -flto -D_GNU_SOURCE", "CMAKE_CXX_FLAGS": "-march=armv8.7a+fp16+dotprod+i8mm -fvectorize -ffp-model=fast -fno-finite-math-only -flto -D_GNU_SOURCE", diff --git a/docs/backend/snapdragon/README.md b/docs/backend/snapdragon/README.md index e9f0e2158..391c8bf23 100644 --- a/docs/backend/snapdragon/README.md +++ b/docs/backend/snapdragon/README.md @@ -2,39 +2,47 @@ ## Setup -### Android +The cross-compilation toolchain images are provided by the +[Qualcomm Snapdragon Toolchain registry](https://github.com/snapdragon-toolchain). +These Docker images include the Android NDK, OpenCL SDK, Hexagon SDK, CMake, and the necessary cross-compilers: -The easiest way to build llama.cpp for a Snapdragon-based Android device is using the toolchain Docker image (see github.com/snapdragon-toolchain). -This image includes Android NDK, OpenCL SDK, Hexagon SDK, CMake, etc. +* **Android toolchain**: `ghcr.io/snapdragon-toolchain/arm64-android:v0.7` +* **Linux toolchain**: `ghcr.io/snapdragon-toolchain/arm64-linux:v0.7` -This method works on Linux, macOS, and Windows. macOS and Windows users should install Docker Desktop. - -``` -~/src/llama.cpp$ docker run -it -u $(id -u):$(id -g) --volume $(pwd):/workspace --platform linux/amd64 ghcr.io/snapdragon-toolchain/arm64-android:v0.7 -[d]/> cd /workspace -``` - -Note: The rest of the **Android** build process assumes that you're running inside the toolchain container. - -### Windows On Snapdragon - -Native Windows 11 arm64 builds has the following tools dependencies: -- MS Visual Studio 2026 (Community Edition or Pro) - - MSVC arm64 standard and runtime libraries - - UCRT and Driver Kit -- LLVM core libraries and Clang compiler (winget) -- CMake, Git, Python (winget) -- Hexagon SDK Community Edition 6.6 or later (see windows.md) -- OpenCL SDK 2.3 or later (see windows.md) - -Note: The rest of the **Windows** build process assumes that you're running natively in Powershell. -Adapt below build commands accordingly. +The unified build utility (`scripts/snapdragon/build.py`) automatically pulls +and orchestrates these containers to perform target compilation. +You only need to ensure that Docker (or Docker Desktop on macOS/Windows) is running on your host machine. +Specific setup, build, and installation details for Linux and Windows on Snapdragon platforms are documented in: +* [Linux on Snapdragon guide](linux.md) +* [Windows on Snapdragon guide](windows.md) ## How to Build -Let's build llama.cpp with CPU, OpenCL, and Hexagon backends via CMake presets: +### Using build.py script (Recommended) +The easiest way to build llama.cpp is by using the `scripts/snapdragon/build.py` script. It automatically copies the CMake presets, +launches the correct compilation Docker container, builds the libraries and tools, +installs them, and optionally pushes them to your ADB device. + +Build and deploy for Android target (accepts `android` or `adb` alias): ``` +$ ./scripts/snapdragon/build.py --target adb --push +``` + +Build and deploy for Linux target (accepts `linux` or `lnx` alias): +``` +$ ./scripts/snapdragon/build.py --target linux:user@host --push +``` + +### Manual CMake Build + +Alternatively, you can build llama.cpp manually by entering the cross-compilation Docker container and running the CMake commands: + +```bash +# Start the cross-compilation container manually: +~/src/llama.cpp$ docker run -it --rm -u $(id -u):$(id -g) --volume $(pwd):/workspace --platform linux/amd64 ghcr.io/snapdragon-toolchain/arm64-android:v0.7 + +# Inside the container, build the project using presets: [d]/workspace> cp docs/backend/snapdragon/CMakeUserPresets.json . [d]/workspace> cmake --preset arm64-android-snapdragon-release -B build-snapdragon @@ -68,19 +76,19 @@ Preset CMake variables: To generate an installable "package" simply use cmake --install: ``` -[d]/workspace> cmake --install build-snapdragon --prefix pkg-snapdragon/llama.cpp +[d]/workspace> cmake --install build-snapdragon --prefix pkg-android/llama.cpp -- Install configuration: "Release" --- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml-cpu.so --- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml-opencl.so --- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml-hexagon.so --- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml-htp-v73.so --- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml-htp-v75.so --- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml-htp-v79.so --- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml-htp-v81.so --- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml.so +-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml-cpu.so +-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml-opencl.so +-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml-hexagon.so +-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml-htp-v73.so +-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml-htp-v75.so +-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml-htp-v79.so +-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml-htp-v81.so +-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml.so ... --- Installing: /workspace/pkg-snapdragon/llama.cpp/bin/llama-bench --- Installing: /workspace/pkg-snapdragon/llama.cpp/bin/llama-cli +-- Installing: /workspace/pkg-android/llama.cpp/bin/llama-bench +-- Installing: /workspace/pkg-android/llama.cpp/bin/llama-cli ... ``` @@ -91,14 +99,14 @@ To generate an installable "package" simply use cmake --install: For this step, your device needs to be configured for on-device development. Please see https://developer.android.com/studio/debug/dev-options for details. -Once ADB is enabled, use `adb push` to install `pkg-snapdragon` on the device. +Once ADB is enabled, use `adb push` to install `pkg-android` on the device. **Note that the toolchain Docker image doesn't have ADB and doesn't set up the ADB bridge. Please use native ADB on the host.** ``` -~/src/llama.cpp$ adb push pkg-snapdragon/llama.cpp /data/local/tmp/ -pkg-snapdragon/llama.cpp/bin/: 67 files pushed, 0 skipped. 190.2 MB/s (919095042 bytes in 4.607s) -pkg-snapdragon/llama.cpp/include/: 19 files pushed, 0 skipped. 20.5 MB/s (255173 bytes in 0.012s) -pkg-snapdragon/llama.cpp/lib/: 16 files pushed, 0 skipped. 144.4 MB/s (43801382 bytes in 0.289s) +~/src/llama.cpp$ adb push pkg-android/llama.cpp /data/local/tmp/ +pkg-android/llama.cpp/bin/: 67 files pushed, 0 skipped. 190.2 MB/s (919095042 bytes in 4.607s) +pkg-android/llama.cpp/include/: 19 files pushed, 0 skipped. 20.5 MB/s (255173 bytes in 0.012s) +pkg-android/llama.cpp/lib/: 16 files pushed, 0 skipped. 144.4 MB/s (43801382 bytes in 0.289s) 102 files pushed, 0 skipped. 186.9 MB/s (963151597 bytes in 4.914s) ``` @@ -115,24 +123,44 @@ Llama-3.2-1B-Instruct-Q4_0.gguf: 1 file pushed, 0 skipped. 38.3 MB/s (773025920 ### Windows -All artifacts are already installed in the `pkg-snapdragon` folder. -To run, adapt below instructions to use Powershell scripts in `scripts/snapdragon/windows`. +All artifacts are already installed in the `pkg-wos` folder. +To run, you can use the `scripts/snapdragon/run.py` runner script (see details below). ## How to Run -The easiest way to run llama.cpp cli tools is using provided wrapper scripts that properly set up all required environment variables. +The easiest way to run llama.cpp cli tools is using the provided `scripts/snapdragon/run.py` wrapper script. This script automatically +maps CLI options to environment variables, resolves executable paths, and runs the command locally, via ADB, or remotely via SSH on the +target device. -llama.cpp supports three backends on Snapdragon-based devices: CPU, Adreno GPU (GPUOpenCL), and Hexagon NPU (HTP0-4). -You can select which backend to run the model on using the `D=` variable, which maps to the `--device` option. +llama.cpp supports three backends on Snapdragon-based devices: CPU, Adreno GPU (GPUOpenCL), and Hexagon NPU. +You can select which backend(s) to run the model on using the `--device` option of the tool (or `--devices` option in `run.py`). Hexagon NPU behaves as a "GPU" device when it comes to `-ngl` and other offload-related options. -Here are some examples of running various llama.cpp tools via ADB. +Here are some examples of running various llama.cpp tools. -Simple question for Llama-3.2-1B +Generating a completion with Gemma on Android (relying on default `HTP0:0` device and default thread count `-t 6`): ``` -~/src/llama.cpp$ M=Llama-3.2-1B-Instruct-Q4_0.gguf D=HTP0 ./scripts/snapdragon/adb/run-completion.sh -p "what is the most popular cookie in the world?" +~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb -- llama-completion -m models/gemma-2-2b-it-Q4_0.gguf -f prompts/sample_prompt_1024.txt --jinja -st +... +ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev 1 +ggml-hex: Hexagon Arch version v79 +ggml-hex: allocating new session: HTP0:0 +... +load_tensors: offloading output layer to GPU +load_tensors: offloaded 27/27 layers to GPU +load_tensors: CPU model buffer size = 300.00 MiB +load_tensors: HTP0:0 model buffer size = 1400.26 MiB +... +llama_perf_context_print: prompt eval time = 320.00 ms / 1024 tokens ( 0.31 ms per token, 3200.00 tokens per second) +llama_perf_context_print: eval time = 2100.00 ms / 100 runs ( 21.00 ms per token, 47.62 tokens per second) +``` + +Simple question for Llama-3.2-1B: + +``` +~/src/llama.cpp$ ./scripts/snapdragon/run.py --target android --devices HTP0 -- llama-cli -m Llama-3.2-1B-Instruct-Q4_0.gguf -p "what is the most popular cookie in the world?" ... ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev 1 ggml-hex: Hexagon Arch version v79 @@ -142,8 +170,7 @@ ggml-hex: new session: HTP0 : session-id 0 domain-id 3 uri file:///libggml-htp-v load_tensors: offloading output layer to GPU load_tensors: offloaded 17/17 layers to GPU load_tensors: CPU model buffer size = 225.49 MiB -load_tensors: HTP0 model buffer size = 0.26 MiB -load_tensors: HTP0-REPACK model buffer size = 504.00 MiB +load_tensors: HTP0 model buffer size = 504.26 MiB ... I hope this helps you understand the world's most popular cookies! [end of text] ... @@ -156,60 +183,25 @@ llama_perf_context_print: graphs reused = 473 llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | llama_memory_breakdown_print: | - HTP0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | llama_memory_breakdown_print: | - Host | 439 = 225 + 136 + 77 | -llama_memory_breakdown_print: | - HTP0-REPACK | 504 = 504 + 0 + 0 | ``` -Summary request for OLMoE-1B-7B. This is a large model that requires two HTP sessions/devices +Op test for MUL_MAT: ``` -~/src/llama.cpp$ M=OLMoE-1B-7B-0125-Instruct-Q4_0.gguf NDEV=2 D=HTP0,HTP1 ./scripts/snapdragon/adb/run-completion.sh -f surfing.txt +~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb --hex-hostbuf 0 --devices HTP0:0 -- test-backend-ops -b HTP0:0 -o MUL_MAT ... -ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev 1 -ggml-hex: Hexagon Arch version v81 -ggml-hex: allocating new session: HTP0 -ggml-hex: allocating new session: HTP1 -... -load_tensors: offloading output layer to GPU -load_tensors: offloaded 17/17 layers to GPU -load_tensors: CPU model buffer size = 143.86 MiB -load_tensors: HTP1 model buffer size = 0.23 MiB -load_tensors: HTP1-REPACK model buffer size = 1575.00 MiB -load_tensors: HTP0 model buffer size = 0.28 MiB -load_tensors: HTP0-REPACK model buffer size = 2025.00 MiB -... -llama_context: CPU output buffer size = 0.19 MiB -llama_kv_cache: HTP1 KV buffer size = 238.00 MiB -llama_kv_cache: HTP0 KV buffer size = 306.00 MiB -llama_kv_cache: size = 544.00 MiB ( 8192 cells, 16 layers, 1/1 seqs), K (q8_0): 272.00 MiB, V (q8_0): 272.00 MiB -llama_context: HTP0 compute buffer size = 15.00 MiB -llama_context: HTP1 compute buffer size = 15.00 MiB -llama_context: CPU compute buffer size = 24.56 MiB -... -llama_perf_context_print: prompt eval time = 1730.57 ms / 212 tokens ( 8.16 ms per token, 122.50 tokens per second) -llama_perf_context_print: eval time = 5624.75 ms / 257 runs ( 21.89 ms per token, 45.69 tokens per second) -llama_perf_context_print: total time = 7377.33 ms / 469 tokens -llama_perf_context_print: graphs reused = 255 -llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | -llama_memory_breakdown_print: | - HTP0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - HTP1 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - Host | 742 = 144 + 544 + 54 | -llama_memory_breakdown_print: | - HTP1-REPACK | 1575 = 1575 + 0 + 0 | -llama_memory_breakdown_print: | - HTP0-REPACK | 2025 = 2025 + 0 + 0 | -``` - -Op test for MUL_MAT - -``` -~/src/llama.cpp$ HB=0 ./scripts/snapdragon/adb/run-tool.sh test-backend-ops -b HTP0 -o MUL_MAT -... -Backend 2/3: HTP0 +Backend 2/3: HTP0:0 Device description: Hexagon Device memory: 2048 MB (2048 MB free) MUL_MAT(type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],v=0,o=1): OK MUL_MAT(type_a=q4_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],v=0,o=1): OK MUL_MAT(type_a=q4_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],v=0,o=1): OK +``` -~/src/llama.cpp-hexagon$ M=Llama-3.2-1B-Instruct-Q4_0.gguf ./scripts/snapdragon/adb/run-bench.sh -p 128 -n 64 +Llama benchmark: + +``` +~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb --devices HTP0 -- llama-bench -p 128 -n 64 -m Llama-3.2-1B-Instruct-Q4_0.gguf ... ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev 1 ggml-hex: Hexagon Arch version v79 @@ -219,15 +211,20 @@ ggml-hex: new session: HTP0 : session-id 0 domain-id 3 uri file:///libggml-htp-v | ---------------| ---------: | -----: | ---------- | --: | ------: | ------: | ---: | ----: | ------------: | | llama 1B Q4_0 | 729.75 MiB | 1.24 B | HTP | 99 | 4 | 128 | 0 | pp128 | 169.42 ± 1.75 | | llama 1B Q4_0 | 729.75 MiB | 1.24 B | HTP | 99 | 4 | 128 | 0 | tg64 | 51.54 ± 1.13 | - -build: 6a8cf8914 (6733) ``` ## Environment variables -- `GGML_HEXAGON_NDEV=1` - Controls the number of devices/sessions to allocate. The default is 1. - Most quantized models under 4B fit into a single session; an 8B model needs two, and a 20B model needs four. +- `GGML_HEXAGON_DEVICES` (default: not set, defaults to HTP0 session) + Controls which NPU devices and sessions to allocate. Can be configured as: + - A single integer `N`: Allocates `N` sessions named `HTP0`, `HTP1`, ..., `HTP` (behaves identically to `GGML_HEXAGON_NDEV=N`). + - A comma-separated list of device names in `HTP:` format (or legacy `HTP` format). For example, `HTP0:0,HTP0:1` creates two virtual + sessions on the first physical NPU (useful for memory limits). `HTP0:0,HTP1:0` allocates one session on each of the two physical NPUs + on a dual-NPU device. + +- `GGML_HEXAGON_NDEV` (deprecated) + Replaced by `GGML_HEXAGON_DEVICES`. Controls the number of virtual sessions to allocate on physical NPU `0`. + Allocates sessions named `HTP0`, `HTP1`, etc. - `GGML_HEXAGON_NHVX=0` Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version). @@ -255,26 +252,17 @@ build: 6a8cf8914 (6733) - `2` Extended profile with per-op `usecs`, `cycles` and default PMU counter data - `0x1,...,0x8` Extended profile with per-op `usecs`, `cycles` and custom PMU counter data - The logging output can be either saved into a file for post-processing or it can be piped directly into the post-processing tool to generate the report. + The logging output can be either saved into a file for post-processing or it can be piped directly into the post-processing tool + to generate the report. Examples: - `GGML_HEXAGON_PROFILE=1 llama-completion ... |& ./scripts/snapdragon/ggml-hexagon-profile.py -` - -- `GGML_HEXAGON_OPSTAGE=0x0` - Allows enabling specific stages of the Op processing pipeline: - - - `0x1` Enable Op Queue (i.e., queuing Ops into NPU) - - `0x2` Enable Op Compute (MUL_MAT, etc.) - - Examples: - - `GGML_HEXAGON_OPSTAGE=0x1 llama-completion ...` - Ops are enqueued to the NPU but dma & compute are disabled - `GGML_HEXAGON_OPSTAGE=0x3 llama-completion ...` - Full queuing and processing of Ops (default) + `GGML_HEXAGON_PROFILE=1 ./scripts/snapdragon/run.py --target adb -- llama-cli ... |& ./scripts/snapdragon/ggml-hexagon-profile.py -` - `GGML_HEXAGON_OPFILTER=regex` Allows filtering (disabling) Ops that match the regex pattern: Examples: - `GGML_HEXAGON_OPFILTER="FLASH_ATTN_EXT" llama-completion ...` - Disable Flash Attention on Hexagon (falls back to CPU or GPU) - `GGML_HEXAGON_OPFILTER="ADD\|SUB" llama-completion ...` - Disable ADD and SUB on Hexagon (fall back to CPU or GPU) + `GGML_HEXAGON_OPFILTER="FLASH_ATTN_EXT" ./scripts/snapdragon/run.py --target adb -- llama-cli ...` - Disable Flash Attention on Hexagon (falls back to CPU or GPU) + `GGML_HEXAGON_OPFILTER="ADD\|SUB" ./scripts/snapdragon/run.py --target adb -- llama-cli ...` - Disable ADD and SUB on Hexagon (fall back to CPU or GPU) + diff --git a/docs/backend/snapdragon/developer.md b/docs/backend/snapdragon/developer.md index 9d56638e3..d7d9f2a27 100644 --- a/docs/backend/snapdragon/developer.md +++ b/docs/backend/snapdragon/developer.md @@ -39,22 +39,21 @@ the repacking. ## Large model handling -Hexagon NPU session (aka Process Domain (PD) in the Hexagon docs) is limited to a memory mapping of around 3.5GB. -In llama.cpp/GGML the Hexagon session is mapped to a single GGML backend device (HTP0, HTP1, etc). +Hexagon NPU sessions (aka Process Domains (PD) in the Hexagon SDK) are limited to a maximum memory mapping window of around 3.5GB. +In llama.cpp/GGML, each Hexagon session is mapped to a single GGML backend device (e.g., `HTP0:0`, `HTP0:1`, etc. when using +`GGML_HEXAGON_DEVICES`, or `HTP0`, `HTP1` in legacy mode). -In order to map models larger than 3.5GB we need to allocate multiple devices and split the model. -For this we're taking advantage of the llama.cpp/GGML multi-GPU layer-splitting support. -Each Hexagon device behaves like a GPU from the offload and model splitting perspective. +To support running models larger than 3.5GB on a single device, the Hexagon backend dynamically maps and unmaps execution buffers +during the graph execution cycle to stay within the Process Domain window. This enables large models to run successfully on a single +NPU device. -Here is an example of running GPT-OSS-20B model on a newer Snapdragon device with 16GB of DDR. +Alternatively, users can choose to use standard llama.cpp/GGML layer-splitting mode to partition and split the model across +multiple Hexagon devices or virtual sessions (which behave like multiple GPUs from the offload and splitting perspective). + +Here is an example of running GPT-OSS-20B model on a Snapdragon device using 4 virtual sessions on a single NPU (physical index 0). ``` -M=gpt-oss-20b-Q4_0.gguf NDEV=4 D=HTP0,HTP1,HTP2,HTP3 P=surfing.txt scripts/snapdragon/adb/run-completion.sh -f surfing.txt -n 32 -... -LD_LIBRARY_PATH=/data/local/tmp/llama.cpp/lib -ADSP_LIBRARY_PATH=/data/local/tmp/llama.cpp/lib -GGML_HEXAGON_NDEV=4 ./bin/llama-cli --load-mode none -m /data/local/tmp/llama.cpp/../gguf/gpt-oss-20b-Q4_0.gguf - -t 4 --ctx-size 8192 --batch-size 128 -ctk q8_0 -ctv q8_0 -fa on -ngl 99 --device HTP0,HTP1,HTP2,HTP3 -no-cnv -f surfing.txt +~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb --devices HTP0:0,HTP0:1,HTP0:2,HTP0:3 -- llama-cli --load-mode none -m /data/local/tmp/gguf/gpt-oss-20b-Q4_0.gguf -t 4 --ctx-size 8192 --batch-size 128 -ctk q8_0 -ctv q8_0 -fa on -ngl 99 -no-cnv -f surfing.txt ... llama_model_loader: - type f32: 289 tensors llama_model_loader: - type q4_0: 96 tensors @@ -63,33 +62,29 @@ llama_model_loader: - type mxfp4: 72 tensors ... load_tensors: offloaded 25/25 layers to GPU load_tensors: CPU model buffer size = 1182.09 MiB -load_tensors: HTP1 model buffer size = 6.64 MiB -load_tensors: HTP1-REPACK model buffer size = 2505.94 MiB -load_tensors: HTP3 model buffer size = 5.55 MiB -load_tensors: HTP3-REPACK model buffer size = 2088.28 MiB -load_tensors: HTP0 model buffer size = 7.75 MiB -load_tensors: HTP0-REPACK model buffer size = 2923.59 MiB -load_tensors: HTP2 model buffer size = 6.64 MiB -load_tensors: HTP2-REPACK model buffer size = 2505.94 MiB +load_tensors: HTP0:1 model buffer size = 2512.58 MiB +load_tensors: HTP0:3 model buffer size = 2093.83 MiB +load_tensors: HTP0:0 model buffer size = 2931.34 MiB +load_tensors: HTP0:2 model buffer size = 2512.58 MiB ... llama_context: n_ctx_per_seq (8192) < n_ctx_train (131072) -- the full capacity of the model will not be utilized llama_context: CPU output buffer size = 0.77 MiB llama_kv_cache_iswa: creating non-SWA KV cache, size = 8192 cells -llama_kv_cache: HTP1 KV buffer size = 25.50 MiB -llama_kv_cache: HTP3 KV buffer size = 25.50 MiB -llama_kv_cache: HTP0 KV buffer size = 25.50 MiB -llama_kv_cache: HTP2 KV buffer size = 25.50 MiB +llama_kv_cache: HTP0:1 KV buffer size = 25.50 MiB +llama_kv_cache: HTP0:3 KV buffer size = 25.50 MiB +llama_kv_cache: HTP0:0 KV buffer size = 25.50 MiB +llama_kv_cache: HTP0:2 KV buffer size = 25.50 MiB llama_kv_cache: size = 102.00 MiB ( 8192 cells, 12 layers, 1/1 seqs), K (q8_0): 51.00 MiB, V (q8_0): 51.00 MiB llama_kv_cache_iswa: creating SWA KV cache, size = 256 cells -llama_kv_cache: HTP1 KV buffer size = 0.80 MiB -llama_kv_cache: HTP3 KV buffer size = 0.53 MiB -llama_kv_cache: HTP0 KV buffer size = 1.06 MiB -llama_kv_cache: HTP2 KV buffer size = 0.80 MiB +llama_kv_cache: HTP0:1 KV buffer size = 0.80 MiB +llama_kv_cache: HTP0:3 KV buffer size = 0.53 MiB +llama_kv_cache: HTP0:0 KV buffer size = 1.06 MiB +llama_kv_cache: HTP0:2 KV buffer size = 0.80 MiB llama_kv_cache: size = 3.19 MiB ( 256 cells, 12 layers, 1/1 seqs), K (q8_0): 1.59 MiB, V (q8_0): 1.59 MiB -llama_context: HTP0 compute buffer size = 16.06 MiB -llama_context: HTP1 compute buffer size = 16.06 MiB -llama_context: HTP2 compute buffer size = 16.06 MiB -llama_context: HTP3 compute buffer size = 16.06 MiB +llama_context: HTP0:0 compute buffer size = 16.06 MiB +llama_context: HTP0:1 compute buffer size = 16.06 MiB +llama_context: HTP0:2 compute buffer size = 16.06 MiB +llama_context: HTP0:3 compute buffer size = 16.06 MiB llama_context: CPU compute buffer size = 98.19 MiB ... llama_perf_context_print: prompt eval time = 3843.67 ms / 197 tokens ( 19.51 ms per token, 51.25 tokens per second) @@ -97,13 +92,9 @@ llama_perf_context_print: eval time = 1686.13 ms / 31 runs ( 54.3 llama_perf_context_print: total time = 6266.30 ms / 228 tokens llama_perf_context_print: graphs reused = 30 llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | -llama_memory_breakdown_print: | - HTP0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - HTP1 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - HTP2 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - HTP3 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP0:0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP0:1 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP0:2 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP0:3 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | llama_memory_breakdown_print: | - Host | 1476 = 1208 + 105 + 162 | -llama_memory_breakdown_print: | - HTP1-REPACK | 2505 = 2505 + 0 + 0 | -llama_memory_breakdown_print: | - HTP3-REPACK | 2088 = 2088 + 0 + 0 | -llama_memory_breakdown_print: | - HTP0-REPACK | 2923 = 2923 + 0 + 0 | -llama_memory_breakdown_print: | - HTP2-REPACK | 2505 = 2505 + 0 + 0 | ``` diff --git a/docs/backend/snapdragon/linux.md b/docs/backend/snapdragon/linux.md index 90fdadb6c..d4ecd9b1b 100644 --- a/docs/backend/snapdragon/linux.md +++ b/docs/backend/snapdragon/linux.md @@ -1,25 +1,37 @@ # Snapdragon-based Linux devices -## Docker Setup +The cross-compilation is performed using the Snapdragon Linux Docker toolchain image (see +[github.com/snapdragon-toolchain](https://github.com/snapdragon-toolchain)): -The easiest way to build llama.cpp for a Snapdragon-based Linux device is using the toolchain Docker image (see [github.com/snapdragon-toolchain](https://github.com/snapdragon-toolchain)). -This image includes OpenCL SDK, Hexagon SDK, CMake, and the ARM64 Linux cross-compilation toolchain. +* **Linux toolchain**: `ghcr.io/snapdragon-toolchain/arm64-linux:v0.7` -Cross-compilation is supported on **Linux X86** hosts. The resulting binaries are deployed to and run on the target **Qualcomm Snapdragon ARM64 Linux** device. - -``` -~/src/llama.cpp$ docker run -it -u $(id -u):$(id -g) --volume $(pwd):/workspace --platform linux/amd64 ghcr.io/snapdragon-toolchain/arm64-linux:v0.1 -[d]/> cd /workspace -``` - -Note: The rest of the **Linux** build process assumes that you're running inside the toolchain container. +The unified build utility (`scripts/snapdragon/build.py`) automatically pulls +and orchestrates this container to perform target compilation. You only need to +ensure that Docker is running on your host machine. ## How to Build -Let's build llama.cpp with CPU, OpenCL, and Hexagon backends via CMake presets: +### Using build.py script (Recommended) +The easiest way to build llama.cpp is by using the `scripts/snapdragon/build.py` script. It automatically copies the CMake presets, +launches the correct compilation Docker container, builds the libraries and tools, +installs them, and optionally pushes them to your target device. + +Build and deploy for a Linux target (using SSH deployment alias `lnx` or `linux`): ``` +$ ./scripts/snapdragon/build.py --target lnx:user@host --push +``` + +### Manual CMake Build + +Alternatively, you can build llama.cpp manually by entering the cross-compilation Docker container and running the CMake commands: + +```bash +# Start the cross-compilation container manually: +~/src/llama.cpp$ docker run -it --rm -u $(id -u):$(id -g) --volume $(pwd):/workspace --platform linux/amd64 ghcr.io/snapdragon-toolchain/arm64-linux:v0.7 + +# Inside the container, build the project using presets: [d]/workspace> cp docs/backend/snapdragon/CMakeUserPresets.json . [d]/workspace> cmake --preset arm64-linux-snapdragon-release -B build-snapdragon @@ -30,17 +42,19 @@ Let's build llama.cpp with CPU, OpenCL, and Hexagon backends via CMake presets: To generate an installable "package" simply use cmake --install, then zip it: ``` -[d]/workspace> cmake --install build-snapdragon --prefix pkg-snapdragon -[d]/workspace> zip -r pkg-snapdragon.zip pkg-snapdragon +[d]/workspace> cmake --install build-snapdragon --prefix pkg-linux +[d]/workspace> zip -r pkg-linux.zip pkg-linux ``` ## How to Install -For this step, you will deploy the built binaries and libraries to the target Linux device. Transfer `pkg-snapdragon.zip` to the target device, then unzip it and set up the environment variables: +For this step, you will deploy the built binaries and libraries to the target +Linux device. Transfer `pkg-linux.zip` to the target device, then unzip it +and set up the environment variables: ``` -$ unzip pkg-snapdragon.zip -$ cd pkg-snapdragon +$ unzip pkg-linux.zip +$ cd pkg-linux $ export LD_LIBRARY_PATH=./lib $ export ADSP_LIBRARY_PATH=./lib ``` @@ -52,7 +66,28 @@ $ wget https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF/resolve/main/ ``` ## How to Run -Next, since we have setup the environment variables, we can run the llama-cli with the Hexagon backends: +You can run locally on the Snapdragon Linux device: +``` +$ ./scripts/snapdragon/run.py --devices HTP0 -- llama-cli -m Llama-3.2-3B-Instruct-Q4_0.gguf -ngl 99 -p "what is the most popular cookie in the world?" +``` + +Or run remotely from your host development machine using the SSH target option: +``` +$ ./scripts/snapdragon/run.py --target lnx:user@host --devices HTP0 -- llama-cli -m Llama-3.2-3B-Instruct-Q4_0.gguf -ngl 99 -p "what is the most popular cookie in the world?" +``` + +For multi-NPU systems, you can run a tensor split completion command targeting a remote Linux system: +``` +$ ./scripts/snapdragon/run.py --target ubuntu:maxk@192.168.1.87 --device HTP0:0,HTP1:0 -- llama-completion -m models/gemma-2b-it-Q4_0.gguf -f prompts/sample_prompt_1024.txt --jinja -st --split-mode tensor --ctx-size 8192 +``` + +This translates to the following command being executed remotely via SSH: +``` ++ ssh maxk@192.168.1.87 "cd ~/llama.cpp && ulimit -c unlimited && LD_LIBRARY_PATH=./lib ADSP_LIBRARY_PATH=./lib GGML_HEXAGON_DEVICES=HTP0:0,HTP1:0 GGML_HEXAGON_OPPOLL=1 ./bin/llama-completion -m models/gemma-2b-it-Q4_0.gguf -f prompts/sample_prompt_1024.txt --jinja -st --split-mode tensor --ctx-size 8192 -v -n 16 --device HTP0:0,HTP1:0 -ngl 99 --ubatch-size 1024 -fa on -t 6" +``` + +Alternatively, you can run the binary directly on the device: ``` $ ./bin/llama-cli -m Llama-3.2-3B-Instruct-Q4_0.gguf --device HTP0 -ngl 99 -p "what is the most popular cookie in the world?" ``` + diff --git a/docs/backend/snapdragon/windows.md b/docs/backend/snapdragon/windows.md index aa731413c..3f7d60dd9 100644 --- a/docs/backend/snapdragon/windows.md +++ b/docs/backend/snapdragon/windows.md @@ -1,3 +1,18 @@ +# Snapdragon-based Windows devices + +## Tool Dependencies + +Native Windows 11 arm64 builds have the following tool dependencies: +- MS Visual Studio 2026 (Community Edition or Pro) + - MSVC arm64 standard and runtime libraries + - UCRT and Driver Kit +- LLVM core libraries and Clang compiler (winget) +- CMake, Git, Python (winget) +- Hexagon SDK Community Edition 6.6 or later (see below) +- OpenCL SDK 2.3 or later (see below) + +Note: The rest of the **Windows** build process assumes that you're running natively in Powershell. + ## Overview The document covers procedures for installing the latest GPU and NPU drivers, and OpenCL and Hexagon SDKs. @@ -53,7 +68,8 @@ Download the driver from https://softwarecenter.qualcomm.com/catalog/item/Qualcomm_HND -After the automated installation and reboot please make sure that the Hexagon NPU device shows up in the `Device Manager` (under `Neural Processors`). +After the automated installation and reboot please make sure that the Hexagon NPU device shows up in the `Device Manager` +(under `Neural Processors`). If the device is not available you can try installing all components (`qcnspmcdm8380`, `qcnspmcdm8380_ext`) manually. The components are extracted into @@ -130,12 +146,12 @@ However, additional settings are required for generating and signing HTP Ops lib > cmake --preset arm64-windows-snapdragon-release -B build-wos ... -> cmake --install build-wos --prefix pkg-snapdragon +> cmake --install build-wos --prefix pkg-wos ``` Once the build is complete HTP ops libraries will be installed like this ``` -> dir pkg-snapdragon/lib +> dir pkg-wos/lib ... -a---- 1/22/2026 6:01 PM 187656 libggml-htp-v73.so -a---- 1/22/2026 6:01 PM 191752 libggml-htp-v75.so @@ -147,8 +163,8 @@ Once the build is complete HTP ops libraries will be installed like this The .cat file, the signature and proper certificate installation can be verified with ``` -> signtool.exe verify /v /pa .\pkg-snapdragon\lib\libggml-htp.cat -Verifying: .\pkg-snapdragon\lib\libggml-htp.cat +> signtool.exe verify /v /pa .\pkg-wos\lib\libggml-htp.cat +Verifying: .\pkg-wos\lib\libggml-htp.cat Signature Index: 0 (Primary Signature) Hash of file (sha256): 9820C664DA59D5EAE31DBB664127FCDAEF59CDC31502496BC567544EC2F401CF @@ -156,6 +172,6 @@ Hash of file (sha256): 9820C664DA59D5EAE31DBB664127FCDAEF59CDC31502496BC567544EC Signing Certificate Chain: Issued to: GGML.HTP.v1 ... -Successfully verified: .\pkg-snapdragon\lib\libggml-htp.cat +Successfully verified: .\pkg-wos\lib\libggml-htp.cat ... ``` diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index e8a5009b3..c1e9f919d 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include #include #ifdef _WIN32 @@ -52,6 +54,8 @@ #include "htp/matmul-ops.h" #include "htp/flash-attn-ops.h" #include "htp/unary-ops.h" +#include "htp/get-rows-ops.h" +#include "htp/set-rows-ops.h" #include "htp_iface.h" #include "htp-drv.h" @@ -59,6 +63,36 @@ using intvec = std::vector; using uintvec = std::vector; using u32vec = std::vector; +#define GGML_HEXAGON_MAX_SESSIONS 16 + +#define GGML_HEXAGON_FENCE_BUFFER_SIZE 8192 +#define GGML_HEXAGON_FENCE_SLOT_SIZE 128 + +struct ggml_hexagon_device_config { + int physical_idx = 0; + int virtual_idx = 0; + std::string name; +}; + +static ggml_hexagon_device_config opt_device_configs[GGML_HEXAGON_MAX_SESSIONS]; + +static int get_domain_id(int physical_idx) { + switch (physical_idx) { + case 0: return 3; // CDSP0 (all devices) + case 1: return 4; // CDSP1 (IQ9, IQ10) + case 2: return 18; // CDSP2 (IQ10) + case 3: return 19; // CDSP3 (IQ10) + default: return CDSP_DOMAIN_ID + physical_idx; + } +} + +static std::string get_domain_name(int physical_idx) { + if (physical_idx == 0) { + return CDSP_DOMAIN_NAME; + } + return std::string("cdsp") + std::to_string(physical_idx); +} + static int opt_arch = 0; // autodetect static size_t opt_ndev = 1; static size_t opt_nhvx = 0; // use all @@ -68,20 +102,19 @@ static size_t opt_mbuf = 1ul * 1024 * 1024 * 1024; // max buffer size static int opt_etm = 0; static int opt_verbose = 0; static int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu) -static int opt_hostbuf = 1; // hostbuf ON by default +static bool opt_hostbuf = false; static int opt_mm_select = 3; // 3 = HMX -> Tiled -> Flat -> CPU, 2 = Tiled -> Flat -> CPU, 1 = Flat -> CPU static int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported) +static int opt_ar_select = 2; // 2 = fused ALLREDUCE+ADD (DMA, default), 1 = unfused ALLREDUCE (DMA), 0 = fallback to CPY+FENCE // Default PMU events, if profiling with PMU (mode=2) is enabled // See https://docs.qualcomm.com/doc/80-N2040-60/topic/pmu-events.html // https://docs.qualcomm.com/doc/80-N2040-61/topic/hvx-pmu-events.html static u32vec opt_pmu_evt { 0x3, 0x111, 0x100, 0x105, 0x240, 0x256, 0x7D, 0x8C }; -// Enable all stages by default -static int opt_opstage = HTP_OPSTAGE_QUEUE | HTP_OPSTAGE_COMPUTE; static int opt_opbatch = 1024; // max number of ops in a batch -static int opt_opqueue = 16; // max number of pending batches +static int opt_opqueue = 64; // max number of pending batches static int opt_optrace = 0; // trace buffer size per thread (0 means default) static int opt_oppoll = 0; // polling for batch completions static int opt_opfusion = 1; // enable/disable op fusion @@ -121,7 +154,7 @@ static void ggml_hexagon_dump_op_exec(const std::string &sess_name, const htp_op static void ggml_hexagon_dump_op_supp(const std::string &sess_name, const struct ggml_tensor * op, bool supp) { if (!opt_verbose) return; - htp_opformat fmt(htp_opformat(htp_opnode{const_cast(op), {}, HTP_OP_INVALID})); + htp_opformat fmt(htp_opformat(htp_opnode(HTP_OP_INVALID, const_cast(op)))); GGML_LOG_DEBUG("ggml-hex: %s supports-op %s|%s|%s|%s|%s|%s|%s\n", sess_name.c_str(), ggml_op_desc(op), fmt.names, fmt.dims, fmt.types, fmt.strides, fmt.buffs, supp ? "yes" : "no"); } @@ -144,6 +177,7 @@ static const char * htp_event_name(uint16_t id) { case HTP_TRACE_EVT_L2FLUSH: return "L2FLUSH"; case HTP_TRACE_EVT_INIT: return "INIT"; case HTP_TRACE_EVT_BUFF: return "BUFF"; + case HTP_TRACE_EVT_FENCE: return "FENCE"; default: return "UNKNOWN"; } } @@ -205,7 +239,12 @@ static void ggml_hexagon_dump_trace_events(const std::string & sess_name, const } } -// ** +enum ggml_hexagon_tensor_flags { + GGML_HEXAGON_TENSOR_REPACK = (1 << 0), + GGML_HEXAGON_TENSOR_WEIGHT = (1 << 1), + GGML_HEXAGON_TENSOR_FENCE = (1 << 2), + GGML_HEXAGON_TENSOR_FUSEABLE = (1 << 3), +}; static inline bool ggml_hexagon_is_repack_type(enum ggml_type type) { return type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || @@ -227,6 +266,15 @@ static void ggml_hexagon_precompute_matmul_params( struct htp_mm_kernel_params * kparams ); +static void ggml_hexagon_precompute_fused_matmul_add_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * src2, + const struct ggml_tensor * dst, + struct htp_mm_kernel_params * kparams +); + static void ggml_hexagon_precompute_unary_params( const struct ggml_hexagon_session * sess, uint32_t op, @@ -236,25 +284,75 @@ static void ggml_hexagon_precompute_unary_params( struct htp_unary_kernel_params * kparams ); -static void ggml_hexagon_precompute_fused_qkv_params( +static void ggml_hexagon_precompute_get_rows_params( const struct ggml_hexagon_session * sess, const struct ggml_tensor * src0, const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_get_rows_kernel_params * kparams +); + +static void ggml_hexagon_precompute_set_rows_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_set_rows_kernel_params * kparams +); + +static void ggml_hexagon_precompute_fused_mmnx_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + int32_t n_weights, struct htp_mm_kernel_params * kparams ); -static void ggml_hexagon_precompute_fused_ffn_params( +static bool ggml_hexagon_precompute_allreduce_params( const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - struct htp_mm_kernel_params * kparams + const struct ggml_tensor * dst, + uint32_t rank, + uint32_t n_ranks, + bool has_add, + bool is_row_bcast, + struct htp_allreduce_kernel_params * kparams ); +static bool mm_is_hmx_eligible(const ggml_tensor * t); +static bool is_mergeable_mul_mat(const ggml_tensor * t); +static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor * n2); + // ** backend sessions +struct ggml_hexagon_tensor_extra { + std::vector shadow_buf; + size_t shadow_size { 0 }; + uint32_t flags { 0 }; +}; + +static inline bool ggml_hexagon_tensor_is_fuseable(const struct ggml_tensor * t) { + if (!t || !t->extra) return false; + auto extra = (const struct ggml_hexagon_tensor_extra *) t->extra; + return (extra->flags & GGML_HEXAGON_TENSOR_FUSEABLE) != 0; +} + +struct htp_opnode; + struct ggml_hexagon_opbatch; struct ggml_hexagon_opqueue; -struct htp_opnode; +struct ggml_hexagon_shared_buffer; +struct ggml_hexagon_session; + +struct ggml_backend_hexagon_comm_context { + std::vector backends; + size_t n_backends = 0; + uint32_t fence_seq = 0; +}; + +struct ggml_hexagon_event { + ggml_hexagon_session * sess = nullptr; + uint64_t seq = 0; +}; struct ggml_hexagon_session { std::string name; @@ -264,6 +362,8 @@ struct ggml_hexagon_session { uint32_t domain_id; uint64_t queue_id; int dev_id; + int phys_idx; + int virt_idx; bool valid_session; bool valid_handle; bool valid_queue; @@ -273,20 +373,24 @@ struct ggml_hexagon_session { ggml_hexagon_opbatch* op_batch; ggml_hexagon_opqueue* op_queue; + std::unordered_map> cloned_buffers; + std::unordered_set sync_peers; + ggml_backend_buffer_type buffer_type = {}; - ggml_backend_buffer_type repack_buffer_type = {}; + ggml_backend_buffer_type host_buffer_type = {}; - uint32_t n_threads = 0; - uint32_t n_hvx = 0; - uint32_t n_hmx = 0; - uint64_t vtcm_size = 0; - size_t max_vmem = 0; + uint32_t n_threads = 0; + uint32_t n_hvx = 0; + uint32_t n_hmx = 0; + uint64_t vtcm_size = 0; + size_t max_vmem = 0; size_t max_bufsize = 0; + uint32_t fence_seq; - struct { - uint64_t uid = 0; - std::vector htp_nodes; - } cached_graph; + uint64_t cached_uid = 0; + std::vector cached_nodes; + + mutable std::unordered_set needs_repack; ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) noexcept(false); ~ggml_hexagon_session() noexcept(true); @@ -297,10 +401,31 @@ struct ggml_hexagon_session { void release() noexcept(true); void enqueue_op(const htp_opnode & node); - void flush(bool all = true); + void enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor = nullptr, uint32_t fence_seq = 0); + void enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq = 0); + void enqueue_allreduce(const ggml_tensor * dst, const std::vector & src_tensors, const std::vector & sync_tensors, uint32_t rank, uint32_t n_ranks, uint32_t fence_seq_entry = 0, uint32_t fence_seq_exit = 0); + void flush(bool all = true); void flush_pending(bool all = false); - void flush_batch(); + void flush_batch(size_t min_ops = 1); + + uint64_t record_event(); + void wait_event(uint64_t seq); + + bool clone_buffer(const ggml_hexagon_shared_buffer*); + + void add_sync_peer(ggml_hexagon_session * peer) { + sync_peers.insert(peer); + } + + void flush_sync_peers() { + if (sync_peers.empty()) return; + + for (auto * peer : sync_peers) { + peer->flush_batch(); + } + sync_peers.clear(); + } }; // ** backend buffers @@ -315,26 +440,68 @@ struct ggml_backend_hexagon_buffer_type_context { std::string name; }; +struct ggml_hexagon_rpcmem_block { + uint8_t * base = nullptr; + int fd = -1; + size_t size = 0; + + ggml_hexagon_rpcmem_block(size_t size) { + base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, size); + if (!base) { + throw std::runtime_error("ggml-hex: rpcmem_alloc failed"); + } + fd = rpcmem_to_fd(base); + if (fd < 0) { + rpcmem_free(base); + throw std::runtime_error("ggml-hex: rpcmem_to_fd failed"); + } + this->size = size; + } + + ~ggml_hexagon_rpcmem_block() { + if (base) { + rpcmem_free(base); + } + } +}; + struct ggml_hexagon_shared_buffer { - ggml_hexagon_session * sess; - uint8_t * base; - size_t size; - int fd; - bool mapped; - bool pinned; + ggml_hexagon_session * sess; + std::shared_ptr mem; + std::vector tensor_extra; + uint32_t fence_head = 0; + size_t fences_size = 0; + bool mapped; + bool pinned; + + const char * c_name() const { return sess->c_name(); } + uint8_t * base() const { return mem ? mem->base : nullptr; } + size_t size() const { return mem ? mem->size : 0; } + int fd() const { return mem ? mem->fd : -1; } + + uint8_t * alloc_fence() { + if (fences_size == 0) return nullptr; + int max_slots = fences_size / GGML_HEXAGON_FENCE_SLOT_SIZE; + uint32_t slot = (fence_head++) % max_slots; + + size_t guard_offset = size() - fences_size; + uint8_t * fence_ptr = base() + guard_offset + (size_t)slot * GGML_HEXAGON_FENCE_SLOT_SIZE; + return fence_ptr; + } void mmap() { + if (!this->mem) return; fastrpc_map_flags flags = this->pinned ? FASTRPC_MAP_FD : FASTRPC_MAP_FD_DELAYED; - int err = fastrpc_mmap(sess->domain_id, this->fd, (void *) this->base, 0, this->size, flags); + int err = fastrpc_mmap(sess->domain_id, fd(), (void *) base(), 0, size(), flags); if (err != 0) { GGML_LOG_ERROR("ggml-hex: %s buffer mapping failed : domain_id %d size %zu fd %d error 0x%08x\n", sess->c_name(), - sess->domain_id, this->size, this->fd, (unsigned) err); + sess->domain_id, size(), fd(), (unsigned) err); throw std::runtime_error("ggml-hex: fastrpc_mmap failed (see log for details)"); } HEX_VERBOSE("ggml-hex: %s mapped buffer: base %p size %zu fd %d pinned %u\n", - sess->c_name(), (void *) this->base, this->size, this->fd, pinned); + sess->c_name(), (void *) base(), size(), fd(), pinned); this->mapped = true; } @@ -342,66 +509,69 @@ struct ggml_hexagon_shared_buffer { void unmap() { if (!this->mapped) return; - if (!this->pinned) { + if (!this->pinned && mem) { // HTP might still hold a reference, tell it drop it - htp_iface_munmap(sess->handle, this->fd); + htp_iface_munmap(sess->handle, fd()); } - fastrpc_munmap(sess->domain_id, this->fd, (void *) this->base, this->size); + if (mem) { + fastrpc_munmap(sess->domain_id, fd(), (void *) base(), size()); + } HEX_VERBOSE("ggml-hex: %s unmapped buffer: base %p size %zu fd %d\n", sess->c_name(), - (void *) this->base, size, this->fd); + (void *) base(), size(), fd()); this->mapped = false; - this->fd = -1; } void alloc(size_t size) { - if (this->base) return; + if (this->mem) return; - this->base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, size); - if (!this->base) { - GGML_LOG_ERROR("ggml-hex: %s failed to allocate buffer : size %zu\n", sess->c_name(), size); - throw std::runtime_error("ggml-hex: rpcmem_alloc failed (see log for details)"); - } - - this->fd = rpcmem_to_fd(this->base); - if (this->fd < 0) { - GGML_LOG_ERROR("ggml-hex: %s failed to get FD for buffer %p\n", sess->c_name(), (void *) this->base); - throw std::runtime_error("ggml-hex: rpcmem_to_fd failed (see log for details)"); - } - this->size = size; + this->mem = std::make_shared(size); HEX_VERBOSE("ggml-hex: %s allocated buffer: base %p size %zu fd %d pinned %d\n", sess->c_name(), - (void *) this->base, this->size, this->fd, (int) pinned); + (void *) base(), this->size(), fd(), (int) pinned); mmap(); } void free() { - if (!this->base) return; - unmap(); - rpcmem_free(this->base); - - HEX_VERBOSE("ggml-hex: %s freed buffer: base %p size %zu fd %d\n", sess->c_name(), - (void *) this->base, size, this->fd); - - this->base = NULL; + // The memory is freed when the shared_ptr refcount drops to 0. + HEX_VERBOSE("ggml-hex: %s release ref on buffer: base %p size %zu fd %d\n", sess->c_name(), + (void *) base(), size(), fd()); + this->mem = nullptr; } - ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, size_t size, bool pinned = false) { - this->sess = sess; - this->size = 0; - this->base = nullptr; - this->fd = -1; - this->mapped = false; - this->pinned = pinned; + ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, size_t size, bool pinned = false, size_t fence_size = 0) { + this->sess = sess; + this->mapped = false; + this->pinned = pinned; + this->fences_size = fence_size; - alloc(size); + // Size adjustment inside the buffer class + size_t guard_offset = (size + 4095) & ~4095; + size_t total_size = guard_offset; + if (fence_size > 0) { + total_size += 4096 + fence_size; + } + + alloc(total_size); + } + + // Clone constructor for cross-session mapping + ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, const ggml_hexagon_shared_buffer & other) { + this->sess = sess; + this->mem = other.mem; + this->mapped = false; + this->pinned = other.pinned; + this->fences_size = other.fences_size; } ~ggml_hexagon_shared_buffer() { free(); + for (auto * extra : tensor_extra) { + delete extra; + } } }; @@ -416,18 +586,25 @@ static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer static void * ggml_backend_hexagon_buffer_get_base(ggml_backend_buffer_t buffer) { auto sbuf = static_cast(buffer->context); - return sbuf->base; + return sbuf->base(); } static enum ggml_status ggml_backend_hexagon_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { auto sbuf = static_cast(buffer->context); auto sess = sbuf->sess; - HEX_VERBOSE("ggml-hex: %s init-tensor %s : base %p data %p nbytes %zu usage %d\n", sess->c_name(), - tensor->name, (void *) sbuf->base, tensor->data, ggml_nbytes(tensor), (int) buffer->usage); + HEX_VERBOSE("ggml-hex: %s init-tensor %s : base %p data %p nbytes %zu\n", sess->c_name(), + tensor->name, (void *) sbuf->base(), tensor->data, ggml_nbytes(tensor)); - if (tensor->view_src != NULL && tensor->view_offs == 0) { - return GGML_STATUS_SUCCESS; // nothing to do for the view + auto extra = new ggml_hexagon_tensor_extra(); + sbuf->tensor_extra.push_back(extra); + + tensor->extra = extra; + if (ggml_hexagon_is_repack_type(tensor->type)) { + if (sess->needs_repack.count(tensor)) { + extra->flags |= GGML_HEXAGON_TENSOR_REPACK; + sess->needs_repack.erase(tensor); + } } return GGML_STATUS_SUCCESS; @@ -499,7 +676,7 @@ static void pack_mxfp4_quants(block_mxfp4 * x, const uint8_t * qs, unsigned int } // repack q4_0 data into q4_0_tiled tensor -static void repack_q4_0_tiled(ggml_tensor * t, const void * data, size_t size) { +static void repack_q4_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { const block_q4_0 * src_matrix = (const block_q4_0 *) data; int64_t ne0 = t->ne[0]; int64_t ne1 = t->ne[1]; @@ -513,46 +690,49 @@ static void repack_q4_0_tiled(ggml_tensor * t, const void * data, size_t size) { const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_0; const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - const block_q4_0 * src_expert = src_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); - uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + size_t slice_size = ne1 * ggml_row_size(t->type, ne0); + int64_t start_slice = offset / slice_size; + int64_t end_slice = (offset + size + slice_size - 1) / slice_size; + if (end_slice > ne2 * ne3) { + end_slice = ne2 * ne3; + } - for (int ct = 0; ct < n_col_tiles; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const block_q4_0 * src_slice = src_matrix + (slice_idx - start_slice) * (ne1 * (ne0 / 32)); + uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size; - uint8_t tile_quants[32][32]; + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; + + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + unpack_q4_0_quants(tile_quants[row], &src_slice[r * (ne0 / 32) + kt], 0); + } else { + memset(tile_quants[row], 8, 32); + } + } + + for (int cp = 0; cp < 16; cp++) { for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - unpack_q4_0_quants(tile_quants[row], &src_expert[r * (ne0 / 32) + kt], 0); - } else { - memset(tile_quants[row], 8, 32); - } + tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; } + } - for (int cp = 0; cp < 16; cp++) { - for (int row = 0; row < 32; row++) { - tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; - } - } - - ggml_half * scale_dst = (ggml_half *)(tile_dst + 512); - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_expert[r * (ne0 / 32) + kt].d : 0; - } + ggml_half * scale_dst = (ggml_half *)(tile_dst + 512); + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_slice[r * (ne0 / 32) + kt].d : 0; } } } } - - GGML_UNUSED(size); } // repack q4_0_tiled tensor into q4_0 data -static void repack_tiled_q4_0(void * data, const ggml_tensor * t, size_t size) { +static void repack_tiled_q4_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { block_q4_0 * dst_matrix = (block_q4_0 *) data; int64_t ne0 = t->ne[0]; int64_t ne1 = t->ne[1]; @@ -566,48 +746,65 @@ static void repack_tiled_q4_0(void * data, const ggml_tensor * t, size_t size) { const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_0; const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - block_q4_0 * dst_expert = dst_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); - const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + size_t slice_size = ne1 * ggml_row_size(t->type, ne0); + size_t row_size_bytes = ggml_row_size(t->type, ne0); + int64_t start_slice = offset / slice_size; + int64_t end_slice = (offset + size + slice_size - 1) / slice_size; + if (end_slice > ne2 * ne3) { + end_slice = ne2 * ne3; + } - for (int ct = 0; ct < n_col_tiles; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size); + size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size); + size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size; + size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size; - uint8_t tile_quants[32][32]; - for (int cp = 0; cp < 16; cp++) { - for (int row = 0; row < 32; row++) { - uint8_t val = tile_src[cp * 32 + row]; - tile_quants[row][2 * cp + 0] = val & 0x0F; - tile_quants[row][2 * cp + 1] = val >> 4; - } - } + int64_t start_row = slice_offset_start / row_size_bytes; + int64_t end_row = (slice_offset_end + row_size_bytes - 1) / row_size_bytes; + end_row = (std::min)(end_row, ne1); + int start_ct = start_row / 32; + int end_ct = (end_row + 31) / 32; + end_ct = (std::min)(end_ct, n_col_tiles); + + block_q4_0 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_q4_0); + const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size; + + for (int ct = start_ct; ct < end_ct; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; + + uint8_t tile_quants[32][32]; + for (int cp = 0; cp < 16; cp++) { for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - pack_q4_0_quants(&dst_expert[r * (ne0 / 32) + kt], tile_quants[row], 0); - } + uint8_t val = tile_src[cp * 32 + row]; + tile_quants[row][2 * cp + 0] = val & 0x0F; + tile_quants[row][2 * cp + 1] = val >> 4; } + } - const ggml_half * scale_src = (const ggml_half *)(tile_src + 512); - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - dst_expert[r * (ne0 / 32) + kt].d = scale_src[row]; - } + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + pack_q4_0_quants(&dst_slice[(r - start_row) * (ne0 / 32) + kt], tile_quants[row], 0); + } + } + + const ggml_half * scale_src = (const ggml_half *)(tile_src + 512); + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + dst_slice[(r - start_row) * (ne0 / 32) + kt].d = scale_src[row]; } } } } } - - GGML_UNUSED(size); } // repack q4_1 data into q4_1_tiled tensor -static void repack_q4_1_tiled(ggml_tensor * t, const void * data, size_t size) { +static void repack_q4_1_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { const block_q4_1 * src_matrix = (const block_q4_1 *) data; int64_t ne0 = t->ne[0]; int64_t ne1 = t->ne[1]; @@ -621,52 +818,55 @@ static void repack_q4_1_tiled(ggml_tensor * t, const void * data, size_t size) { const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1; const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - const block_q4_1 * src_expert = src_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); - uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + size_t slice_size = ne1 * ggml_row_size(t->type, ne0); + int64_t start_slice = offset / slice_size; + int64_t end_slice = (offset + size + slice_size - 1) / slice_size; + if (end_slice > ne2 * ne3) { + end_slice = ne2 * ne3; + } - for (int ct = 0; ct < n_col_tiles; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const block_q4_1 * src_slice = src_matrix + (slice_idx - start_slice) * (ne1 * (ne0 / 32)); + uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size; - uint8_t tile_quants[32][32]; - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - unpack_q4_1_quants(tile_quants[row], &src_expert[r * (ne0 / 32) + kt], 0); - } else { - memset(tile_quants[row], 0, 32); - } + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; + + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + unpack_q4_1_quants(tile_quants[row], &src_slice[r * (ne0 / 32) + kt], 0); + } else { + memset(tile_quants[row], 0, 32); } + } - for (int cp = 0; cp < 16; cp++) { - for (int row = 0; row < 32; row++) { - tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; - } - } - - ggml_half * scale_dst = (ggml_half *)(tile_dst + 512); + for (int cp = 0; cp < 16; cp++) { for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - scale_dst[2 * row + 0] = src_expert[r * (ne0 / 32) + kt].d; - scale_dst[2 * row + 1] = src_expert[r * (ne0 / 32) + kt].m; - } else { - scale_dst[2 * row + 0] = 0; - scale_dst[2 * row + 1] = 0; - } + tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; + } + } + + ggml_half * scale_dst = (ggml_half *)(tile_dst + 512); + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + scale_dst[2 * row + 0] = src_slice[r * (ne0 / 32) + kt].d; + scale_dst[2 * row + 1] = src_slice[r * (ne0 / 32) + kt].m; + } else { + scale_dst[2 * row + 0] = 0; + scale_dst[2 * row + 1] = 0; } } } } } - - GGML_UNUSED(size); } // repack q4_1_tiled tensor into q4_1 data -static void repack_tiled_q4_1(void * data, const ggml_tensor * t, size_t size) { +static void repack_tiled_q4_1(void * data, const ggml_tensor * t, size_t offset, size_t size) { block_q4_1 * dst_matrix = (block_q4_1 *) data; int64_t ne0 = t->ne[0]; int64_t ne1 = t->ne[1]; @@ -680,49 +880,66 @@ static void repack_tiled_q4_1(void * data, const ggml_tensor * t, size_t size) { const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1; const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - block_q4_1 * dst_expert = dst_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); - const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + size_t slice_size = ne1 * ggml_row_size(t->type, ne0); + size_t row_size_bytes = ggml_row_size(t->type, ne0); + int64_t start_slice = offset / slice_size; + int64_t end_slice = (offset + size + slice_size - 1) / slice_size; + if (end_slice > ne2 * ne3) { + end_slice = ne2 * ne3; + } - for (int ct = 0; ct < n_col_tiles; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size); + size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size); + size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size; + size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size; - uint8_t tile_quants[32][32]; - for (int cp = 0; cp < 16; cp++) { - for (int row = 0; row < 32; row++) { - uint8_t val = tile_src[cp * 32 + row]; - tile_quants[row][2 * cp + 0] = val & 0x0F; - tile_quants[row][2 * cp + 1] = val >> 4; - } - } + int64_t start_row = slice_offset_start / row_size_bytes; + int64_t end_row = (slice_offset_end + row_size_bytes - 1) / row_size_bytes; + end_row = (std::min)(end_row, ne1); + int start_ct = start_row / 32; + int end_ct = (end_row + 31) / 32; + end_ct = (std::min)(end_ct, n_col_tiles); + + block_q4_1 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_q4_1); + const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size; + + for (int ct = start_ct; ct < end_ct; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; + + uint8_t tile_quants[32][32]; + for (int cp = 0; cp < 16; cp++) { for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - pack_q4_1_quants(&dst_expert[r * (ne0 / 32) + kt], tile_quants[row], 0); - } + uint8_t val = tile_src[cp * 32 + row]; + tile_quants[row][2 * cp + 0] = val & 0x0F; + tile_quants[row][2 * cp + 1] = val >> 4; } + } - const ggml_half * scale_src = (const ggml_half *)(tile_src + 512); - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - dst_expert[r * (ne0 / 32) + kt].d = scale_src[2 * row]; - dst_expert[r * (ne0 / 32) + kt].m = scale_src[2 * row + 1]; - } + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + pack_q4_1_quants(&dst_slice[(r - start_row) * (ne0 / 32) + kt], tile_quants[row], 0); + } + } + + const ggml_half * scale_src = (const ggml_half *)(tile_src + 512); + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + dst_slice[(r - start_row) * (ne0 / 32) + kt].d = scale_src[2 * row]; + dst_slice[(r - start_row) * (ne0 / 32) + kt].m = scale_src[2 * row + 1]; } } } } } - - GGML_UNUSED(size); } // repack q8_0 data into q8_0_tiled tensor -static void repack_q8_0_tiled(ggml_tensor * t, const void * data, size_t size) { +static void repack_q8_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { const block_q8_0 * src_matrix = (const block_q8_0 *) data; int64_t ne0 = t->ne[0]; int64_t ne1 = t->ne[1]; @@ -736,41 +953,44 @@ static void repack_q8_0_tiled(ggml_tensor * t, const void * data, size_t size) { const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q8_0; const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - const block_q8_0 * src_expert = src_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); - uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + size_t slice_size = ne1 * ggml_row_size(t->type, ne0); + int64_t start_slice = offset / slice_size; + int64_t end_slice = (offset + size + slice_size - 1) / slice_size; + if (end_slice > ne2 * ne3) { + end_slice = ne2 * ne3; + } - for (int ct = 0; ct < n_col_tiles; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const block_q8_0 * src_slice = src_matrix + (slice_idx - start_slice) * (ne1 * (ne0 / 32)); + uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size; - for (int cp = 0; cp < 16; cp++) { - int col0 = cp * 2; - int col1 = col0 + 1; - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - const block_q8_0 * b = (r < ne1 && kt < ne0 / 32) ? &src_expert[r * (ne0 / 32) + kt] : NULL; - tile_dst[cp * 64 + 2 * row + 0] = b ? b->qs[col0] : 0; - tile_dst[cp * 64 + 2 * row + 1] = b ? b->qs[col1] : 0; - } - } + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; - ggml_half * scale_dst = (ggml_half *)(tile_dst + 1024); + for (int cp = 0; cp < 16; cp++) { + int col0 = cp * 2; + int col1 = col0 + 1; for (int row = 0; row < 32; row++) { int64_t r = ct * 32 + row; - scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_expert[r * (ne0 / 32) + kt].d : 0; + const block_q8_0 * b = (r < ne1 && kt < ne0 / 32) ? &src_slice[r * (ne0 / 32) + kt] : NULL; + tile_dst[cp * 64 + 2 * row + 0] = b ? b->qs[col0] : 0; + tile_dst[cp * 64 + 2 * row + 1] = b ? b->qs[col1] : 0; } } + + ggml_half * scale_dst = (ggml_half *)(tile_dst + 1024); + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_slice[r * (ne0 / 32) + kt].d : 0; + } } } } - - GGML_UNUSED(size); } // repack q8_0_tiled tensor into q8_0 data -static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t size) { +static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { block_q8_0 * dst_matrix = (block_q8_0 *) data; int64_t ne0 = t->ne[0]; int64_t ne1 = t->ne[1]; @@ -784,45 +1004,62 @@ static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t size) { const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q8_0; const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - block_q8_0 * dst_expert = dst_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); - const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + size_t slice_size = ne1 * ggml_row_size(t->type, ne0); + size_t row_size_bytes = ggml_row_size(t->type, ne0); + int64_t start_slice = offset / slice_size; + int64_t end_slice = (offset + size + slice_size - 1) / slice_size; + if (end_slice > ne2 * ne3) { + end_slice = ne2 * ne3; + } - for (int ct = 0; ct < n_col_tiles; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size); + size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size); + size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size; + size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size; - for (int cp = 0; cp < 16; cp++) { - int col0 = cp * 2; - int col1 = col0 + 1; - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - block_q8_0 & b = dst_expert[r * (ne0 / 32) + kt]; - b.qs[col0] = tile_src[cp * 64 + 2 * row + 0]; - b.qs[col1] = tile_src[cp * 64 + 2 * row + 1]; - } - } - } + int64_t start_row = slice_offset_start / row_size_bytes; + int64_t end_row = (slice_offset_end + row_size_bytes - 1) / row_size_bytes; + end_row = (std::min)(end_row, ne1); - const ggml_half * scale_src = (const ggml_half *)(tile_src + 1024); + int start_ct = start_row / 32; + int end_ct = (end_row + 31) / 32; + end_ct = (std::min)(end_ct, n_col_tiles); + + block_q8_0 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_q8_0); + const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size; + + for (int ct = start_ct; ct < end_ct; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; + + for (int cp = 0; cp < 16; cp++) { + int col0 = cp * 2; + int col1 = col0 + 1; for (int row = 0; row < 32; row++) { int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - dst_expert[r * (ne0 / 32) + kt].d = scale_src[row]; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + block_q8_0 & b = dst_slice[(r - start_row) * (ne0 / 32) + kt]; + b.qs[col0] = tile_src[cp * 64 + 2 * row + 0]; + b.qs[col1] = tile_src[cp * 64 + 2 * row + 1]; } } } + + const ggml_half * scale_src = (const ggml_half *)(tile_src + 1024); + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + dst_slice[(r - start_row) * (ne0 / 32) + kt].d = scale_src[row]; + } + } } } } - - GGML_UNUSED(size); } // repack mxfp4 data into mxfp4_tiled tensor -static void repack_mxfp4_tiled(ggml_tensor * t, const void * data, size_t size) { +static void repack_mxfp4_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { const block_mxfp4 * src_matrix = (const block_mxfp4 *) data; int64_t ne0 = t->ne[0]; int64_t ne1 = t->ne[1]; @@ -836,46 +1073,49 @@ static void repack_mxfp4_tiled(ggml_tensor * t, const void * data, size_t size) const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_MXFP4; const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - const block_mxfp4 * src_expert = src_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); - uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + size_t slice_size = ne1 * ggml_row_size(t->type, ne0); + int64_t start_slice = offset / slice_size; + int64_t end_slice = (offset + size + slice_size - 1) / slice_size; + if (end_slice > ne2 * ne3) { + end_slice = ne2 * ne3; + } - for (int ct = 0; ct < n_col_tiles; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const block_mxfp4 * src_slice = src_matrix + (slice_idx - start_slice) * (ne1 * (ne0 / 32)); + uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size; - uint8_t tile_quants[32][32]; + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; + + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + unpack_mxfp4_quants(tile_quants[row], &src_slice[r * (ne0 / 32) + kt], 0); + } else { + memset(tile_quants[row], 0, 32); + } + } + + for (int cp = 0; cp < 16; cp++) { for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - unpack_mxfp4_quants(tile_quants[row], &src_expert[r * (ne0 / 32) + kt], 0); - } else { - memset(tile_quants[row], 0, 32); - } + tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; } + } - for (int cp = 0; cp < 16; cp++) { - for (int row = 0; row < 32; row++) { - tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; - } - } - - uint8_t * scale_dst = tile_dst + 512; - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_expert[r * (ne0 / 32) + kt].e : 0; - } + uint8_t * scale_dst = tile_dst + 512; + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_slice[r * (ne0 / 32) + kt].e : 0; } } } } - - GGML_UNUSED(size); } // repack mxfp4_tiled tensor into mxfp4 data -static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t size) { +static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t offset, size_t size) { block_mxfp4 * dst_matrix = (block_mxfp4 *) data; int64_t ne0 = t->ne[0]; int64_t ne1 = t->ne[1]; @@ -889,44 +1129,88 @@ static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t size) const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_MXFP4; const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - block_mxfp4 * dst_expert = dst_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); - const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + size_t slice_size = ne1 * ggml_row_size(t->type, ne0); + size_t row_size_bytes = ggml_row_size(t->type, ne0); + int64_t start_slice = offset / slice_size; + int64_t end_slice = (offset + size + slice_size - 1) / slice_size; + if (end_slice > ne2 * ne3) { + end_slice = ne2 * ne3; + } - for (int ct = 0; ct < n_col_tiles; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size); + size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size); + size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size; + size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size; - uint8_t tile_quants[32][32]; - for (int cp = 0; cp < 16; cp++) { - for (int row = 0; row < 32; row++) { - uint8_t val = tile_src[cp * 32 + row]; - tile_quants[row][2 * cp + 0] = val & 0x0F; - tile_quants[row][2 * cp + 1] = val >> 4; - } - } + int64_t start_row = slice_offset_start / row_size_bytes; + int64_t end_row = (slice_offset_end + row_size_bytes - 1) / row_size_bytes; + end_row = (std::min)(end_row, ne1); + int start_ct = start_row / 32; + int end_ct = (end_row + 31) / 32; + end_ct = (std::min)(end_ct, n_col_tiles); + + block_mxfp4 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_mxfp4); + const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size; + + for (int ct = start_ct; ct < end_ct; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; + + uint8_t tile_quants[32][32]; + for (int cp = 0; cp < 16; cp++) { for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - pack_mxfp4_quants(&dst_expert[r * (ne0 / 32) + kt], tile_quants[row], 0); - } + uint8_t val = tile_src[cp * 32 + row]; + tile_quants[row][2 * cp + 0] = val & 0x0F; + tile_quants[row][2 * cp + 1] = val >> 4; } + } - const uint8_t * scale_src = tile_src + 512; - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - dst_expert[r * (ne0 / 32) + kt].e = scale_src[row]; - } + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + pack_mxfp4_quants(&dst_slice[(r - start_row) * (ne0 / 32) + kt], tile_quants[row], 0); + } + } + + const uint8_t * scale_src = tile_src + 512; + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + dst_slice[(r - start_row) * (ne0 / 32) + kt].e = scale_src[row]; } } } } } +} - GGML_UNUSED(size); +static void repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t size) { + switch (tensor->type) { + case GGML_TYPE_Q4_0: + repack_q4_0_tiled(tensor, data, 0, size); + break; + + case GGML_TYPE_Q4_1: + repack_q4_1_tiled(tensor, data, 0, size); + break; + + case GGML_TYPE_Q8_0: + repack_q8_0_tiled(tensor, data, 0, size); + break; + + case GGML_TYPE_IQ4_NL: + repack_q4_0_tiled(tensor, data, 0, size); + break; + + case GGML_TYPE_MXFP4: + repack_mxfp4_tiled(tensor, data, 0, size); + break; + + default: + break; + } } static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer, @@ -934,46 +1218,41 @@ static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer, const void * data, size_t offset, size_t size) { - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; + auto extra = (ggml_hexagon_tensor_extra *) tensor->extra; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; - HEX_VERBOSE("ggml-hex: %s set-tensor %s : data %p offset %zu size %zu\n", sess->c_name(), tensor->name, data, offset, size); + if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { + extra->flags |= GGML_HEXAGON_TENSOR_WEIGHT; + if (ggml_hexagon_is_repack_type(tensor->type)) { + extra->flags |= GGML_HEXAGON_TENSOR_REPACK; + } + } - switch (tensor->type) { - case GGML_TYPE_Q4_0: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_q4_0_tiled(tensor, data, size); - break; + HEX_VERBOSE("ggml-hex: %s set-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n", + sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags); - case GGML_TYPE_Q4_1: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_q4_1_tiled(tensor, data, size); - break; + if ((extra->flags & GGML_HEXAGON_TENSOR_REPACK) == 0) { + memcpy((char *) tensor->data + offset, data, size); + return; + } - case GGML_TYPE_Q8_0: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_q8_0_tiled(tensor, data, size); - break; + if (offset == 0 && size == ggml_nbytes(tensor) && extra->shadow_buf.empty()) { + repack_tensor_tiled(tensor, data, size); + return; + } - case GGML_TYPE_IQ4_NL: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - // IQ4_NL has identical block layout to Q4_0 (ggml_half d + uint8_t qs[16]) - repack_q4_0_tiled(tensor, data, size); - break; + if (extra->shadow_buf.size() < ggml_nbytes(tensor)) { + extra->shadow_buf.resize(ggml_nbytes(tensor)); + } + memcpy(extra->shadow_buf.data() + offset, data, size); + extra->shadow_size += size; - case GGML_TYPE_MXFP4: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_mxfp4_tiled(tensor, data, size); - break; - - default: - memcpy((char *) tensor->data + offset, data, size); - break; + if (extra->shadow_size >= ggml_nbytes(tensor)) { + repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size()); + extra->shadow_buf.clear(); + extra->shadow_buf.shrink_to_fit(); + extra->shadow_size = 0; } } @@ -982,40 +1261,47 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer, void * data, size_t offset, size_t size) { - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; + auto extra = (ggml_hexagon_tensor_extra *) tensor->extra; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; - HEX_VERBOSE("ggml-hex: %s get-tensor %s : data %p offset %zu size %zu\n", sess->c_name(), tensor->name, data, offset, size); + HEX_VERBOSE("ggml-hex: %s get-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n", + sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags); + + if ((extra->flags & GGML_HEXAGON_TENSOR_REPACK) == 0) { + memcpy(data, (const char *) tensor->data + offset, size); + return; + } switch (tensor->type) { case GGML_TYPE_Q4_0: GGML_ASSERT(offset == 0); GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_tiled_q4_0(data, tensor, size); + repack_tiled_q4_0(data, tensor, offset, size); break; case GGML_TYPE_Q4_1: GGML_ASSERT(offset == 0); GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_tiled_q4_1(data, tensor, size); + repack_tiled_q4_1(data, tensor, offset, size); break; case GGML_TYPE_Q8_0: GGML_ASSERT(offset == 0); GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_tiled_q8_0(data, tensor, size); + repack_tiled_q8_0(data, tensor, offset, size); break; case GGML_TYPE_IQ4_NL: GGML_ASSERT(offset == 0); GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_tiled_q4_0(data, tensor, size); + repack_tiled_q4_0(data, tensor, offset, size); break; case GGML_TYPE_MXFP4: GGML_ASSERT(offset == 0); GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_tiled_mxfp4(data, tensor, size); + repack_tiled_mxfp4(data, tensor, offset, size); break; default: @@ -1035,11 +1321,121 @@ static bool ggml_backend_hexagon_buffer_cpy_tensor(ggml_backend_buffer_t bu GGML_UNUSED(dst); } +static void ggml_backend_hexagon_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, + ggml_tensor * tensor, + const void * data, + size_t offset, + size_t size, + size_t n_copies, + size_t stride_tensor, + size_t stride_data) { + auto extra = (ggml_hexagon_tensor_extra *) tensor->extra; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + + if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { + extra->flags |= GGML_HEXAGON_TENSOR_WEIGHT; + if (ggml_hexagon_is_repack_type(tensor->type)) { + extra->flags |= GGML_HEXAGON_TENSOR_REPACK; + } + } + + HEX_VERBOSE("ggml-hex: %s set-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d flags 0x%x\n", + sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage, extra->flags); + + if ((extra->flags & GGML_HEXAGON_TENSOR_REPACK) == 0) { + for (size_t i = 0; i < n_copies; i++) { + memcpy((uint8_t *) tensor->data + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size); + } + return; + } + + if (extra->shadow_buf.size() < ggml_nbytes(tensor)) { + extra->shadow_buf.resize(ggml_nbytes(tensor)); + } + for (size_t i = 0; i < n_copies; i++) { + memcpy(extra->shadow_buf.data() + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size); + } + extra->shadow_size += n_copies * size; + + if (extra->shadow_size >= ggml_nbytes(tensor)) { + repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size()); + extra->shadow_buf.clear(); + extra->shadow_buf.shrink_to_fit(); + extra->shadow_size = 0; + } +} + +static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, + const ggml_tensor * tensor, + void * data, + size_t offset, + size_t size, + size_t n_copies, + size_t stride_tensor, + size_t stride_data) { + auto extra = (ggml_hexagon_tensor_extra *) tensor->extra; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + + HEX_VERBOSE("ggml-hex: %s get-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n", + sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage); + + if ((extra->flags & GGML_HEXAGON_TENSOR_REPACK) == 0) { + for (size_t i = 0; i < n_copies; i++) { + memcpy((uint8_t *)data + i * stride_data, (const uint8_t *)tensor->data + offset + i * stride_tensor, size); + } + return; + } + + size_t temp_size = n_copies > 0 ? (n_copies - 1) * stride_tensor + size : 0; + size_t slice_size = tensor->ne[1] * ggml_row_size(tensor->type, tensor->ne[0]); + size_t slice_offset = offset % slice_size; + size_t row_size_bytes = ggml_row_size(tensor->type, tensor->ne[0]); + + GGML_ASSERT((slice_offset % row_size_bytes) == 0 && "offset must be aligned to row boundary"); + GGML_ASSERT((temp_size % row_size_bytes) == 0 && "temp_size must be a multiple of row size"); + GGML_ASSERT((slice_offset / row_size_bytes) % 32 == 0 && "offset must be aligned to tile size (32 rows)"); + GGML_ASSERT((offset + temp_size) <= ggml_nbytes(tensor)); + + std::vector temp_buf(temp_size); + + switch (tensor->type) { + case GGML_TYPE_Q4_0: + repack_tiled_q4_0(temp_buf.data(), tensor, offset, temp_size); + break; + + case GGML_TYPE_Q4_1: + repack_tiled_q4_1(temp_buf.data(), tensor, offset, temp_size); + break; + + case GGML_TYPE_Q8_0: + repack_tiled_q8_0(temp_buf.data(), tensor, offset, temp_size); + break; + + case GGML_TYPE_IQ4_NL: + repack_tiled_q4_0(temp_buf.data(), tensor, offset, temp_size); + break; + + case GGML_TYPE_MXFP4: + repack_tiled_mxfp4(temp_buf.data(), tensor, offset, temp_size); + break; + + default: + memcpy(temp_buf.data(), (const uint8_t *) tensor->data + offset, temp_size); + break; + } + + for (size_t i = 0; i < n_copies; i++) { + memcpy((uint8_t *) data + i * stride_data, temp_buf.data() + i * stride_tensor, size); + } +} + static void ggml_backend_hexagon_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; auto sess = sbuf->sess; - HEX_VERBOSE("ggml-hex: %s clear-buff base %p size %zu\n", sess->c_name(), (void *) sbuf->base, sbuf->size); - memset(sbuf->base, value, sbuf->size); + HEX_VERBOSE("ggml-hex: %s clear-buff base %p size %zu\n", sess->c_name(), (void *) sbuf->base(), sbuf->size()); + memset(sbuf->base(), value, sbuf->size()); } static ggml_backend_buffer_i ggml_backend_hexagon_buffer_interface = { @@ -1049,6 +1445,40 @@ static ggml_backend_buffer_i ggml_backend_hexagon_buffer_interface = { /* .memset_tensor = */ NULL, /* .set_tensor = */ ggml_backend_hexagon_buffer_set_tensor, /* .get_tensor = */ ggml_backend_hexagon_buffer_get_tensor, + /* .set_tensor_2d = */ ggml_backend_hexagon_buffer_set_tensor_2d, + /* .get_tensor_2d = */ ggml_backend_hexagon_buffer_get_tensor_2d, + /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor, + /* .clear = */ ggml_backend_hexagon_buffer_clear, + /* .reset = */ NULL, +}; + +// ** backend buffer type + +static void ggml_backend_hexagon_host_buffer_set_tensor(ggml_backend_buffer_t buffer, + ggml_tensor * tensor, + const void * data, + size_t offset, + size_t size) { + memcpy((char *) tensor->data + offset, data, size); + GGML_UNUSED(buffer); +} + +static void ggml_backend_hexagon_host_buffer_get_tensor(ggml_backend_buffer_t buffer, + const ggml_tensor * tensor, + void * data, + size_t offset, + size_t size) { + memcpy(data, (const char *) tensor->data + offset, size); + GGML_UNUSED(buffer); +} + +static ggml_backend_buffer_i ggml_backend_hexagon_host_buffer_interface = { + /* .free_buffer = */ ggml_backend_hexagon_buffer_free_buffer, + /* .get_base = */ ggml_backend_hexagon_buffer_get_base, + /* .init_tensor = */ ggml_backend_hexagon_buffer_init_tensor, + /* .memset_tensor = */ NULL, + /* .set_tensor = */ ggml_backend_hexagon_host_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_hexagon_host_buffer_get_tensor, /* .set_tensor_2d = */ NULL, /* .get_tensor_2d = */ NULL, /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor, @@ -1066,24 +1496,22 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer( ggml_backend_buffer_type_t buffer_type, size_t size) { auto sess = static_cast(buffer_type->context)->sess; try { - size += 4 * 1024; // guard page - ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size); + ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false, GGML_HEXAGON_FENCE_BUFFER_SIZE); return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, sbuf, size); } catch (const std::exception & exc) { - GGML_LOG_ERROR("ggml-hex: %s failed to allocate buffer context (host): %s\n", sess->c_name(), exc.what()); + GGML_LOG_ERROR("ggml-hex: %s failed to allocate device buffer context: %s\n", sess->c_name(), exc.what()); return nullptr; } } -static ggml_backend_buffer_t ggml_backend_hexagon_repack_buffer_type_alloc_buffer( +static ggml_backend_buffer_t ggml_backend_hexagon_host_buffer_type_alloc_buffer( ggml_backend_buffer_type_t buffer_type, size_t size) { auto sess = static_cast(buffer_type->context)->sess; try { - size += 4 * 1024; // guard page - ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size); - return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, sbuf, size); + ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false, GGML_HEXAGON_FENCE_BUFFER_SIZE); + return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_host_buffer_interface, sbuf, size); } catch (const std::exception & exc) { - GGML_LOG_ERROR("ggml-hex: %s failed to allocate buffer context (repack): %s\n", sess->c_name(), exc.what()); + GGML_LOG_ERROR("ggml-hex: %s failed to allocate host buffer context: %s\n", sess->c_name(), exc.what()); return nullptr; } } @@ -1094,7 +1522,7 @@ static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer } static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * t) { - if (t->type == GGML_TYPE_Q4_0 || t->type == GGML_TYPE_Q4_1 || t->type == GGML_TYPE_Q8_0 || t->type == GGML_TYPE_IQ4_NL || t->type == GGML_TYPE_MXFP4) { + if (ggml_hexagon_is_repack_type(t->type)) { int64_t ne0 = hex_round_up(t->ne[0], 32); int64_t ne1 = hex_round_up(t->ne[1], 32); int64_t ne2 = t->ne[2]; @@ -1112,14 +1540,12 @@ static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_ } static bool ggml_backend_hexagon_buffer_type_is_host(ggml_backend_buffer_type_t buft) { - return opt_hostbuf; - + return false; GGML_UNUSED(buft); } -static bool ggml_backend_hexagon_repack_buffer_type_is_host(ggml_backend_buffer_type_t buft) { - return false; - +static bool ggml_backend_hexagon_host_buffer_type_is_host(ggml_backend_buffer_type_t buft) { + return true; GGML_UNUSED(buft); } @@ -1132,26 +1558,19 @@ static ggml_backend_buffer_type_i ggml_backend_hexagon_buffer_type_interface = { /* .is_host = */ ggml_backend_hexagon_buffer_type_is_host, }; -static ggml_backend_buffer_type_i ggml_backend_hexagon_repack_buffer_type_interface = { +static ggml_backend_buffer_type_i ggml_backend_hexagon_host_buffer_type_interface = { /* .get_name = */ ggml_backend_hexagon_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_hexagon_repack_buffer_type_alloc_buffer, + /* .alloc_buffer = */ ggml_backend_hexagon_host_buffer_type_alloc_buffer, /* .get_alignment = */ ggml_backend_hexagon_buffer_type_get_alignment, /* .get_max_size = */ ggml_backend_hexagon_buffer_type_get_max_size, /* .get_alloc_size = */ ggml_backend_hexagon_buffer_type_get_alloc_size, - /* .is_host = */ ggml_backend_hexagon_repack_buffer_type_is_host, + /* .is_host = */ ggml_backend_hexagon_host_buffer_type_is_host, }; static bool ggml_backend_buffer_is_hexagon(const struct ggml_backend_buffer * b) { return b->buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment; } -static inline bool ggml_backend_buffer_is_hexagon_repack(const struct ggml_backend_buffer * b) { - if (!opt_hostbuf) { - return ggml_backend_buffer_is_hexagon(b); - } - return b->buft->iface.alloc_buffer == ggml_backend_hexagon_repack_buffer_type_alloc_buffer; -} - struct ggml_hexagon_opbatch { ggml_hexagon_session* sess; @@ -1165,8 +1584,6 @@ struct ggml_hexagon_opbatch { std::unordered_map t_map; // tensor ptr to index std::unordered_multimap d_map; // tensor data to index - - unsigned int n_bufs; // num buffers in the batch unsigned int n_tens; // num tensors ... unsigned int n_ops; // num ops ... @@ -1186,6 +1603,7 @@ struct ggml_hexagon_opbatch { b_map.clear(); t_map.clear(); d_map.clear(); + ops.resize(n_ops_max); } ggml_hexagon_opbatch(ggml_hexagon_session *sess, size_t batch_size, size_t max_vmem) { @@ -1218,39 +1636,39 @@ struct ggml_hexagon_opbatch { // add buffer and return its index int add_buffer(ggml_hexagon_shared_buffer * sbuf) { // Lookup by fd - auto it = b_map.find(sbuf->fd); + auto it = b_map.find(sbuf->fd()); if (it != b_map.end()) { return it->second; } // Add new buffer to the batch int bi = n_bufs++; GGML_ASSERT(n_bufs < HTP_OP_MAX_BUFS); - b_map.insert({sbuf->fd, bi}); + b_map.insert({sbuf->fd(), bi}); htp_buf_desc &b = h_bufs[bi]; - b.base = (uint64_t) sbuf->base; - b.fd = sbuf->fd; - b.size = sbuf->size; + b.base = (uint64_t) sbuf->base(); + b.fd = sbuf->fd(); + b.size = sbuf->size(); b_vmem += b.size; - HEX_VERBOSE("ggml-hex: %s add-buffer #%u : fd %d base %p size %zu : vmem %zu\n", sess->c_name(), bi, b.fd, (void*) sbuf->base, (size_t) b.size, b_vmem); + HEX_VERBOSE("ggml-hex: %s add-buffer #%u : fd %d base %p size %zu : vmem %zu\n", sess->c_name(), bi, b.fd, (void*) sbuf->base(), (size_t) b.size, b_vmem); return bi; } - - bool same_shape(const htp_tensor * h, const ggml_tensor * t) const { + auto extra = (ggml_hexagon_tensor_extra *) t->extra; + int64_t ne0 = t->ne[0]; int64_t ne1 = t->ne[1]; - const bool is_repack = ggml_backend_buffer_is_hexagon_repack(t->buffer) && ggml_hexagon_is_repack_type(t->type); + const bool is_repack = (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0; if (is_repack) { ne0 = hex_round_up(ne0, 32); ne1 = hex_round_up(ne1, 32); } int64_t nb1 = is_repack ? ggml_row_size(t->type, ne0) : t->nb[1]; - int64_t nb2 = is_repack ? nb1 * ne1 : t->nb[2]; + int64_t nb2 = is_repack ? nb1 * ne1 : t->nb[2]; int64_t nb3 = is_repack ? nb2 * t->ne[2] : t->nb[3]; return (h->type == t->type) && @@ -1260,7 +1678,8 @@ struct ggml_hexagon_opbatch { // add tensor and return its index int add_tensor(const ggml_tensor * t) { - auto sbuf = static_cast(t->buffer->context); + auto extra = (ggml_hexagon_tensor_extra *) t->extra; + auto sbuf = static_cast(t->buffer->context); // First lookup by tensor data auto range = d_map.equal_range(t->data); @@ -1280,7 +1699,7 @@ struct ggml_hexagon_opbatch { t_map.insert({t, ti}); d_map.insert({t->data, ti}); - uint64_t t_offset = (uint8_t *) t->data - sbuf->base; + uint64_t t_offset = (uint8_t *) t->data - sbuf->base(); size_t t_size = ggml_nbytes(t); htp_tensor &h = h_tens[ti]; @@ -1289,7 +1708,7 @@ struct ggml_hexagon_opbatch { h.data = t_offset; h.type = t->type; - const bool is_repack = ggml_backend_buffer_is_hexagon_repack(t->buffer) && ggml_hexagon_is_repack_type(t->type); + const bool is_repack = (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0; if (is_repack) { h.ne[0] = hex_round_up(t->ne[0], 32); h.ne[1] = hex_round_up(t->ne[1], 32); @@ -1308,11 +1727,15 @@ struct ggml_hexagon_opbatch { h.nb[0] = t->nb[0]; h.nb[1] = t->nb[1]; h.nb[2] = t->nb[2]; h.nb[3] = t->nb[3]; } - - h.flags = 0; - if (ggml_backend_buffer_get_usage(t->buffer) != GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { - h.flags |= HTP_TENSOR_COMPUTE; + if ((extra->flags & GGML_HEXAGON_TENSOR_WEIGHT) != 0) { + h.flags |= HTP_TENSOR_WEIGHT; + } + if ((extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0) { + h.flags |= HTP_TENSOR_REPACK; + } + if ((extra->flags & GGML_HEXAGON_TENSOR_FENCE) != 0) { + h.flags |= HTP_TENSOR_FENCE; } HEX_VERBOSE("ggml-hex: %s add-tensor #%u %s : bi %d data %p offset %zu size %zu flags 0x%x : %zu:%zu:%zu:%zu\n", sess->c_name(), @@ -1336,8 +1759,8 @@ struct ggml_hexagon_opbatch { extra_tens++; auto sbuf = static_cast(t->buffer->context); - if (!b_map.count(sbuf->fd)) { - extra_vmem += sbuf->size; + if (!b_map.count(sbuf->fd())) { + extra_vmem += sbuf->size(); extra_bufs += 1; } } @@ -1372,10 +1795,6 @@ struct ggml_hexagon_opbatch { o.opcode = node.opcode; o.flags = 0; - if (!(opt_opstage & HTP_OPSTAGE_COMPUTE)) { - o.flags |= HTP_OPFLAGS_SKIP_COMPUTE; - } - ggml_hexagon_dump_op_exec(sess->c_name(), ops[n], o.flags); auto inputs = node.get_inputs(); @@ -1389,8 +1808,474 @@ struct ggml_hexagon_opbatch { } } - void finalize_ranges() { + bool try_fuse_allreduce_add(const htp_opnode & node) { + if (n_ops == 0 || opt_ar_select != 2) return false; + if (node.opcode != HTP_OP_ADD) return false; + + htp_opnode & last_node = ops[n_ops - 1]; + if (last_node.opcode != HTP_OP_ALLREDUCE) return false; + + auto * ar_kparams = (struct htp_allreduce_kernel_params *) last_node.kernel_params; + const uint32_t rank = (uint32_t) ar_kparams->rank; + const ggml_tensor * ar_local = (rank < last_node.inputs.size()) ? last_node.inputs[rank] : nullptr; + const ggml_tensor * add_src0 = node.src0(); + const ggml_tensor * add_src1 = node.src1(); + + if (!add_src0 || !add_src1 || !ar_local) return false; + if (!ggml_hexagon_tensor_is_fuseable(ar_local)) return false; + + const ggml_tensor * res_tensor = nullptr; + if (add_src0 == ar_local || add_src0->data == ar_local->data) { + res_tensor = add_src1; + } else if (add_src1 == ar_local || add_src1->data == ar_local->data) { + res_tensor = add_src0; + } else { + return false; + } + + if (!res_tensor || !res_tensor->data) return false; + + if (ar_local->type != res_tensor->type) return false; + + const bool is_same_shape = (ar_local->ne[0] == res_tensor->ne[0] && ar_local->ne[1] == res_tensor->ne[1] && + ar_local->ne[2] == res_tensor->ne[2] && ar_local->ne[3] == res_tensor->ne[3]); + const bool is_row_bcast = (ar_local->ne[0] == res_tensor->ne[0] && + res_tensor->ne[1] == 1 && res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1); + + if (!is_same_shape && !is_row_bcast) return false; + + if (is_same_shape) { + if (ar_local->nb[1] != res_tensor->nb[1] || ar_local->nb[2] != res_tensor->nb[2] || + ar_local->nb[3] != res_tensor->nb[3]) { + return false; + } + if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(res_tensor)) { + return false; + } + } + if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(node.dst())) { + return false; + } + + struct htp_allreduce_kernel_params new_kparams; + if (!ggml_hexagon_precompute_allreduce_params( + sess, node.dst(), (uint32_t) ar_kparams->rank, (uint32_t) ar_kparams->n_ranks, true, is_row_bcast, &new_kparams + )) { + HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: solver failed\n", sess->c_name()); + return false; + } + + size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; + auto fit_t = [&](const ggml_tensor * t) { + if (!t) return; + if (!t_map.count(t)) { + extra_tens++; + auto sbuf = static_cast(t->buffer->context); + if (!b_map.count(sbuf->fd())) { + extra_vmem += sbuf->size(); + extra_bufs += 1; + } + } + }; + fit_t(res_tensor); + fit_t(node.dst()); + if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { + return false; + } + + last_node.opcode = HTP_OP_ALLREDUCE_ADD; + last_node.name = "ALLREDUCE+ADD"; + last_node.inputs.push_back(res_tensor); + last_node.outputs.clear(); + last_node.outputs.push_back(node.dst()); + last_node.fused.push_back(node.node); + memcpy(last_node.kernel_params, &new_kparams, sizeof(new_kparams)); + + htp_op_desc & o = h_ops[n_ops - 1]; + o.opcode = HTP_OP_ALLREDUCE_ADD; + memcpy(o.kernel_params, &new_kparams, sizeof(new_kparams)); + + const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks; + o.src[2 * n_ranks] = add_tensor(res_tensor); + o.dst[0] = add_tensor(node.dst()); + for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) { + o.dst[d] = 0xffff; + } + + HEX_VERBOSE("ggml-hex: %s fused ALLREDUCE+ADD (#%u)\n", sess->c_name(), n_ops - 1); + return true; } + + bool try_fuse_rms_norm_mul(const htp_opnode & node) { + if (n_ops == 0) return false; + if (node.opcode != HTP_OP_MUL) return false; + + htp_opnode & last_node = ops[n_ops - 1]; + if (last_node.opcode != HTP_OP_RMS_NORM) return false; + + const ggml_tensor * mul_src0 = node.src0(); + const ggml_tensor * mul_src1 = node.src1(); + const ggml_tensor * rms_out = last_node.dst(); + + if (!mul_src0 || !mul_src1 || !rms_out) return false; + if (!ggml_hexagon_tensor_is_fuseable(rms_out)) return false; + + const ggml_tensor * weight = nullptr; + if (mul_src0 == rms_out || mul_src0->data == rms_out->data) { + weight = mul_src1; + } else if (mul_src1 == rms_out || mul_src1->data == rms_out->data) { + weight = mul_src0; + } else { + return false; + } + + if (!weight || !weight->data) return false; + + const ggml_tensor * src0 = last_node.src0(); + if (!src0 || !src0->data) return false; + + if (src0->ne[0] != weight->ne[0] || src0->ne[0] != node.dst()->ne[0]) { + return false; + } + + const bool is_row_bcast = (weight->ne[1] == 1 && weight->ne[2] == 1 && weight->ne[3] == 1); + const bool is_same_shape = (src0->ne[0] == weight->ne[0] && src0->ne[1] == weight->ne[1] && + src0->ne[2] == weight->ne[2] && src0->ne[3] == weight->ne[3]); + if (!is_row_bcast && !is_same_shape) return false; + + if (!ggml_are_same_shape(src0, node.dst())) { + return false; + } + if (ggml_is_contiguous(src0) != ggml_is_contiguous(node.dst())) { + return false; + } + + struct htp_unary_kernel_params new_kparams; + ggml_hexagon_precompute_unary_params( + sess, HTP_OP_RMS_NORM_MUL, src0, weight, node.dst(), &new_kparams + ); + + if ((size_t) new_kparams.vtcm_size > sess->vtcm_size) { + HEX_VERBOSE("ggml-hex: %s skip RMS_NORM_MUL fusion: VTCM needed (%d) > budget (%zu)\n", + sess->c_name(), new_kparams.vtcm_size, sess->vtcm_size); + return false; + } + + size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; + auto fit_t = [&](const ggml_tensor * t) { + if (!t) return; + if (!t_map.count(t)) { + extra_tens++; + auto sbuf = static_cast(t->buffer->context); + if (!b_map.count(sbuf->fd())) { + extra_vmem += sbuf->size(); + extra_bufs += 1; + } + } + }; + fit_t(weight); + fit_t(node.dst()); + if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { + return false; + } + + last_node.opcode = HTP_OP_RMS_NORM_MUL; + last_node.name = "RMS_NORM+MUL"; + last_node.inputs.clear(); + last_node.inputs.push_back(src0); + last_node.inputs.push_back(weight); + last_node.outputs.clear(); + last_node.outputs.push_back(node.dst()); + last_node.fused.push_back(node.node); + memcpy(last_node.kernel_params, &new_kparams, sizeof(new_kparams)); + + htp_op_desc & o = h_ops[n_ops - 1]; + o.opcode = HTP_OP_RMS_NORM_MUL; + memcpy(o.kernel_params, &new_kparams, sizeof(new_kparams)); + + o.src[0] = add_tensor(src0); + o.src[1] = add_tensor(weight); + for (uint32_t s = 2; s < HTP_OP_MAX_INPUTS; s++) { + o.src[s] = 0xffff; + } + o.dst[0] = add_tensor(node.dst()); + for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) { + o.dst[d] = 0xffff; + } + + HEX_VERBOSE("ggml-hex: %s fused RMS_NORM+MUL (#%u)\n", sess->c_name(), n_ops - 1); + return true; + } + + bool try_fuse_mul_mat_add(const htp_opnode & node) { + if (n_ops == 0) return false; + if (node.opcode != HTP_OP_ADD) return false; + + htp_opnode & last_node = ops[n_ops - 1]; + if (last_node.opcode != HTP_OP_MUL_MAT) return false; + + const ggml_tensor * add_src0 = node.src0(); + const ggml_tensor * add_src1 = node.src1(); + const ggml_tensor * mm_out = last_node.dst(); + + if (!add_src0 || !add_src1 || !mm_out) return false; + if (!ggml_hexagon_tensor_is_fuseable(mm_out)) return false; + + const ggml_tensor * src2 = nullptr; + if (add_src0 == mm_out || add_src0->data == mm_out->data) { + src2 = add_src1; + } else if (add_src1 == mm_out || add_src1->data == mm_out->data) { + src2 = add_src0; + } else { + return false; + } + + if (!src2 || !src2->data) return false; + + const ggml_tensor * src0 = last_node.src0(); + const ggml_tensor * src1 = last_node.src1(); + if (!src0 || !src1) return false; + + struct htp_mm_kernel_params kparams; + ggml_hexagon_precompute_fused_matmul_add_params(sess, src0, src1, src2, node.dst(), &kparams); + const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; + const bool can_fuse = (kparams.n_hmx > 0) || (src1_nrows == 1); + if (!can_fuse) return false; + + if ((size_t) kparams.vtcm_size > sess->vtcm_size) { + HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: VTCM needed (%d) > budget (%zu)\n", + sess->c_name(), kparams.vtcm_size, sess->vtcm_size); + return false; + } + + size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; + auto fit_t = [&](const ggml_tensor * t) { + if (!t) return; + if (!t_map.count(t)) { + extra_tens++; + auto sbuf = static_cast(t->buffer->context); + if (!b_map.count(sbuf->fd())) { + extra_vmem += sbuf->size(); + extra_bufs += 1; + } + } + }; + fit_t(src2); + fit_t(node.dst()); + if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { + return false; + } + + last_node.opcode = HTP_OP_MUL_MAT_ADD; + last_node.name = "MUL_MAT+ADD"; + last_node.inputs.clear(); + last_node.inputs.push_back(src0); + last_node.inputs.push_back(src1); + last_node.inputs.push_back(src2); + last_node.outputs.clear(); + last_node.outputs.push_back(node.dst()); + last_node.fused.push_back(node.node); + memcpy(last_node.kernel_params, &kparams, sizeof(kparams)); + + htp_op_desc & o = h_ops[n_ops - 1]; + o.opcode = HTP_OP_MUL_MAT_ADD; + memcpy(o.kernel_params, &kparams, sizeof(kparams)); + + o.src[0] = add_tensor(src0); + o.src[1] = add_tensor(src1); + o.src[2] = add_tensor(src2); + for (uint32_t s = 3; s < HTP_OP_MAX_INPUTS; s++) { + o.src[s] = 0xffff; + } + o.dst[0] = add_tensor(node.dst()); + for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) { + o.dst[d] = 0xffff; + } + + HEX_VERBOSE("ggml-hex: %s fused MUL_MAT+ADD (#%u)\n", sess->c_name(), n_ops - 1); + return true; + } + + bool try_fuse_mul_mat_nx(const htp_opnode & node) { + if (n_ops == 0 || node.opcode != HTP_OP_MUL_MAT) return false; + if (!is_mergeable_mul_mat(node.node)) return false; + + const ggml_tensor * w_in = node.src0(); + const ggml_tensor * x_in = node.src1(); + const ggml_tensor * d_in = node.dst(); + if (!w_in || !x_in || !d_in) return false; + + htp_opnode & last_node = ops[n_ops - 1]; + + // Case 1: last_node is already MUL_MAT_NX + if (last_node.opcode == HTP_OP_MUL_MAT_NX) { + const uint32_t curr_n = (uint32_t) last_node.outputs.size(); + if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 1 >= HTP_OP_MAX_INPUTS) { + return false; + } + + const ggml_tensor * w0 = last_node.inputs[0]; + const ggml_tensor * x = last_node.inputs[curr_n]; + + if (x_in != x || w_in->type != w0->type || w_in->ne[0] != w0->ne[0]) { + return false; + } + + struct htp_mm_kernel_params kparams; + ggml_hexagon_precompute_fused_mmnx_params(sess, w0, x, curr_n + 1, &kparams); + if ((size_t) kparams.vtcm_size > sess->vtcm_size) { + HEX_VERBOSE("ggml-hex: %s skip NX fusion: VTCM needed (%d) > budget (%zu)\n", + sess->c_name(), kparams.vtcm_size, sess->vtcm_size); + return false; + } + + size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; + auto fit_t = [&](const ggml_tensor * t) { + if (!t) return; + if (!t_map.count(t)) { + extra_tens++; + auto sbuf = static_cast(t->buffer->context); + if (!b_map.count(sbuf->fd())) { + extra_vmem += sbuf->size(); + extra_bufs += 1; + } + } + }; + fit_t(w_in); + fit_t(d_in); + if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { + return false; + } + + last_node.inputs[curr_n] = w_in; + last_node.inputs.push_back(x); + last_node.outputs.push_back(d_in); + last_node.fused.push_back(node.node); + memcpy(last_node.kernel_params, &kparams, sizeof(kparams)); + + htp_op_desc & o = h_ops[n_ops - 1]; + memcpy(o.kernel_params, &kparams, sizeof(kparams)); + + for (uint32_t s = 0; s <= curr_n + 1; s++) { + o.src[s] = add_tensor(last_node.inputs[s]); + } + for (uint32_t s = curr_n + 2; s < HTP_OP_MAX_INPUTS; s++) { + o.src[s] = 0xffff; + } + for (uint32_t d = 0; d <= curr_n; d++) { + o.dst[d] = add_tensor(last_node.outputs[d]); + } + for (uint32_t d = curr_n + 1; d < HTP_OP_MAX_OUTPUTS; d++) { + o.dst[d] = 0xffff; + } + + HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_NX (N=%u, #%u)\n", sess->c_name(), curr_n + 1, n_ops - 1); + return true; + } + + // Case 2: last_node is single MUL_MAT + if (last_node.opcode == HTP_OP_MUL_MAT) { + if (!is_mergeable_mul_mat_pair(last_node.node, node.node)) { + return false; + } + + const ggml_tensor * w0 = last_node.src0(); + const ggml_tensor * x = last_node.src1(); + const ggml_tensor * w1 = node.src0(); + if (!w0 || !x || !w1) return false; + + struct htp_mm_kernel_params kparams; + ggml_hexagon_precompute_fused_mmnx_params(sess, w0, x, 2, &kparams); + if ((size_t) kparams.vtcm_size > sess->vtcm_size) { + HEX_VERBOSE("ggml-hex: %s skip NX fusion: VTCM needed (%d) > budget (%zu)\n", + sess->c_name(), kparams.vtcm_size, sess->vtcm_size); + return false; + } + + size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; + auto fit_t = [&](const ggml_tensor * t) { + if (!t) return; + if (!t_map.count(t)) { + extra_tens++; + auto sbuf = static_cast(t->buffer->context); + if (!b_map.count(sbuf->fd())) { + extra_vmem += sbuf->size(); + extra_bufs += 1; + } + } + }; + fit_t(w1); + fit_t(node.dst()); + if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { + return false; + } + + const ggml_tensor * dst_0 = last_node.dst(); + const ggml_tensor * dst_1 = node.dst(); + + last_node.opcode = HTP_OP_MUL_MAT_NX; + last_node.name = "MUL_MAT_NX"; + last_node.inputs.clear(); + last_node.inputs.push_back(w0); + last_node.inputs.push_back(w1); + last_node.inputs.push_back(x); + last_node.outputs.clear(); + last_node.outputs.push_back(dst_0); + last_node.outputs.push_back(dst_1); + last_node.fused.push_back(node.node); + memcpy(last_node.kernel_params, &kparams, sizeof(kparams)); + + htp_op_desc & o = h_ops[n_ops - 1]; + o.opcode = HTP_OP_MUL_MAT_NX; + memcpy(o.kernel_params, &kparams, sizeof(kparams)); + + o.src[0] = add_tensor(w0); + o.src[1] = add_tensor(w1); + o.src[2] = add_tensor(x); + for (uint32_t s = 3; s < HTP_OP_MAX_INPUTS; s++) { + o.src[s] = 0xffff; + } + o.dst[0] = add_tensor(dst_0); + o.dst[1] = add_tensor(dst_1); + for (uint32_t d = 2; d < HTP_OP_MAX_OUTPUTS; d++) { + o.dst[d] = 0xffff; + } + + HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_NX (N=2, #%u)\n", sess->c_name(), n_ops - 1); + return true; + } + + return false; + } + +enum ggml_hexagon_fusion_flags { + GGML_HEXAGON_FUSE_ALLREDUCE_ADD = (1 << 1), // 2 + GGML_HEXAGON_FUSE_RMS_NORM_MUL = (1 << 2), // 4 + GGML_HEXAGON_FUSE_MUL_MAT_ADD = (1 << 3), // 8 + GGML_HEXAGON_FUSE_MUL_MAT_NX = (1 << 4), // 16 +}; + +static inline bool ggml_hexagon_is_fusion_enabled(int flag) { + if (opt_opfusion <= 0) return false; + if (opt_opfusion == 1) return true; // 1 enables all + return (opt_opfusion & flag) != 0; +} + + bool try_fuse(const htp_opnode & node) { + if (!opt_opfusion) return false; + if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_ALLREDUCE_ADD) && try_fuse_allreduce_add(node)) return true; + if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_RMS_NORM_MUL) && try_fuse_rms_norm_mul(node)) return true; + if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_ADD) && try_fuse_mul_mat_add(node)) return true; + if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_NX) && try_fuse_mul_mat_nx(node)) return true; + return false; + } +}; + +struct ggml_hexagon_registry { + ggml_hexagon_registry(ggml_backend_reg_t reg); + ~ggml_hexagon_registry(); + + ggml_backend_device devices[GGML_HEXAGON_MAX_SESSIONS]; }; struct ggml_hexagon_opqueue { @@ -1398,6 +2283,9 @@ struct ggml_hexagon_opqueue { ggml_hexagon_shared_buffer *shm_buf; size_t shm_blk_size; + uint64_t req_seq = 0; + uint64_t rsp_seq = 0; + using opvec = std::vector; std::queue done; // completed batch ids @@ -1429,8 +2317,8 @@ struct ggml_hexagon_opqueue { for (unsigned int i = 0; i < depth; i++) { done.push(i); } if (opt_verbose) { - GGML_LOG_INFO("ggml-hex: %s allocated op-queue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n", - sess->c_name(), batch_size, depth, shm_buf->size, shm_blk_size); + GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n", + sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size); } } @@ -1453,8 +2341,9 @@ struct ggml_hexagon_opqueue { req.n_bufs = op_batch->n_bufs; req.n_tensors = op_batch->n_tens; req.n_ops = op_batch->n_ops; + req.seq = ++req_seq; - op_cache[req.id] = op_batch->ops; + op_cache[req.id] = std::move(op_batch->ops); start_usec[req.id] = ggml_time_us(); const size_t b_size = sizeof(htp_buf_desc) * req.n_bufs; @@ -1470,10 +2359,10 @@ struct ggml_hexagon_opqueue { req.n_traces = 0; } - dbuf.ptr = shm_buf->base + (req.id * shm_blk_size); - dbuf.fd = shm_buf->fd; + dbuf.ptr = shm_buf->base() + (req.id * shm_blk_size); + dbuf.fd = shm_buf->fd(); dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT; - dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base; + dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base(); dbuf.size = b_size + t_size + o_size + p_size + tr_size; GGML_ASSERT(dbuf.size <= shm_blk_size); @@ -1487,7 +2376,7 @@ struct ggml_hexagon_opqueue { memcpy(t_ptr, (void *) op_batch->h_tens.data(), t_size); memcpy(o_ptr, (void *) op_batch->h_ops.data(), o_size); - HEX_VERBOSE("ggml-hex: %s op-queue push batch #%u : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n", + HEX_VERBOSE("ggml-hex: %s opqueue-push batch #%u : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n", shm_buf->sess->c_name(), req.id, req.n_bufs, req.n_tensors, req.n_ops, op_batch->b_vmem, b_size, t_size, o_size, (size_t) dbuf.size); @@ -1530,33 +2419,40 @@ struct ggml_hexagon_opqueue { const size_t m_size = b_size + t_size + o_size + p_size + tr_size; GGML_ASSERT(m_size <= shm_blk_size); - HEX_VERBOSE("ggml-hex: %s op-queue pop batch #%u : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n", + HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%u : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n", shm_buf->sess->c_name(), rsp.id, rsp.n_bufs, rsp.n_tensors, rsp.n_ops, (size_t) dbuf.size, b_size, t_size, o_size); uint8_t * m_ptr = (uint8_t*) dbuf.ptr; uint8_t * p_ptr = m_ptr + (b_size + t_size + o_size); - if (opt_profile && rsp.n_ops > 0) { + if (rsp.n_ops > 0) { auto & ops = op_cache[rsp.id]; - GGML_ASSERT(rsp.n_ops <= ops.size()); const htp_prof_desc * pd = (const htp_prof_desc *) p_ptr; - const htp_trace_desc * trace_events = nullptr; - if (opt_profile == 3) { trace_events = (const htp_trace_desc *) (p_ptr + p_size); } - ggml_hexagon_dump_batch_prof(shm_buf->sess->name, rsp); - - for (uint32_t i = 0; i < rsp.n_ops; i++) { - ggml_hexagon_dump_op_prof(shm_buf->sess->name, ops[i], pd[i]); + if (opt_profile) { + ggml_hexagon_dump_batch_prof(shm_buf->sess->name, rsp); } - ggml_hexagon_dump_trace_events(shm_buf->sess->name, rsp, trace_events, n_traces); + for (uint32_t i = 0; i < rsp.n_ops; i++) { + if (opt_profile) { + ggml_hexagon_dump_op_prof(shm_buf->sess->name, ops[i], pd[i]); + } + } + + if (opt_profile) { + ggml_hexagon_dump_trace_events(shm_buf->sess->name, rsp, trace_events, n_traces); + } + } + + if (rsp.seq > rsp_seq) { + rsp_seq = rsp.seq; } } }; @@ -1601,10 +2497,8 @@ void ggml_hexagon_session::flush_pending(bool all) { } } -void ggml_hexagon_session::flush_batch() { - if (op_batch->empty()) { return; } - - op_batch->finalize_ranges(); +void ggml_hexagon_session::flush_batch(size_t min_ops) { + if (op_batch->n_ops < min_ops) { return; } htp_opbatch_req req {}; dspqueue_buffer dbuf{}; @@ -1625,17 +2519,267 @@ void ggml_hexagon_session::flush_batch() { } } +void ggml_hexagon_session::flush(bool all) { + flush_sync_peers(); + flush_batch(); + flush_pending(all); +} + void ggml_hexagon_session::enqueue_op(const htp_opnode & node) { + for (auto t : node.get_inputs()) { + if (t && t->buffer && ggml_backend_buffer_is_hexagon(t->buffer)) { + if (ggml_backend_hexagon_buffer_get_sess(t->buffer) != this) { + this->clone_buffer(static_cast(t->buffer->context)); + } + } + } + for (auto t : node.get_outputs()) { + if (t && t->buffer && ggml_backend_buffer_is_hexagon(t->buffer)) { + if (ggml_backend_hexagon_buffer_get_sess(t->buffer) != this) { + this->clone_buffer(static_cast(t->buffer->context)); + } + } + } + + if (opt_opfusion && op_batch->try_fuse(node)) { + return; + } + if (!op_batch->fit_op(node)) { flush_batch(); } op_batch->add_op(node); } -// Flush HTP response queue i.e wait for all outstanding requests to complete -void ggml_hexagon_session::flush(bool all) { +void ggml_hexagon_session::enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor, uint32_t fence_seq) { + htp_opnode cpy_node(HTP_OP_CPY); + + ggml_tensor* node = cpy_node.add_dummy(*dst); + node->op = GGML_OP_CPY; + node->src[0] = const_cast(src); + node->src[1] = sync_tensor ? cpy_node.add_dummy(*sync_tensor) : nullptr; + if (sync_tensor) { + node->op_params[0] = (int32_t) fence_seq; + } + + cpy_node.init(node); + if (sync_tensor) { + cpy_node.name = "CPY+FENCE"; + } + this->enqueue_op(cpy_node); +} + +void ggml_hexagon_session::enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq) { + htp_opnode sync_node(HTP_OP_FENCE); + + ggml_tensor* node = sync_node.add_dummy(*sync_tensor); + node->op = GGML_OP_NONE; + node->src[0] = node; + node->op_params[0] = (int32_t) fence_seq; + + sync_node.init(node); + sync_node.name = "FENCE"; + this->enqueue_op(sync_node); +} + +static bool ggml_hexagon_precompute_allreduce_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * dst, + uint32_t rank, + uint32_t n_ranks, + bool has_add, + bool is_row_bcast, + struct htp_allreduce_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + kparams->rank = (int32_t) rank; + kparams->n_ranks = (int32_t) n_ranks; + kparams->is_row_bcast = (has_add && is_row_bcast) ? 1 : 0; + + const uint32_t n_bufs = n_ranks + 1 + (has_add ? 1 : 0); + const uint32_t nelem = (uint32_t) ggml_nelements(dst); + const uint32_t elem_size = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float); + const bool is_contiguous = ggml_is_contiguous(dst); + + const uint32_t ne0 = (uint32_t) dst->ne[0]; + const uint32_t ne1 = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); + kparams->ne0 = (int32_t) ne0; + kparams->ne1 = (int32_t) ne1; + + const bool use_1d = is_contiguous && !(has_add && is_row_bcast && ne1 > 1); + + if (has_add) { + kparams->n_dsts = 1; + if (use_1d) { + kparams->rank_elem_start = 0; + kparams->rank_nelem = (int32_t) nelem; + } else { + kparams->rank_elem_start = 0; + kparams->rank_nelem = (int32_t) ne1; + } + } else { + kparams->n_dsts = (int32_t) n_ranks; + if (use_1d) { + const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128); + const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem); + const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem); + const uint32_t rank_nelem = rank_elem_end - rank_elem_start; + kparams->rank_elem_start = (int32_t) rank_elem_start; + kparams->rank_nelem = (int32_t) rank_nelem; + } else { + const uint32_t rank_chunk_rows = (ne1 + n_ranks - 1) / n_ranks; + const uint32_t rank_r0 = (std::min)(rank * rank_chunk_rows, ne1); + const uint32_t rank_r1 = (std::min)(rank_r0 + rank_chunk_rows, ne1); + const uint32_t rank_nrows = rank_r1 - rank_r0; + kparams->rank_elem_start = (int32_t) rank_r0; + kparams->rank_nelem = (int32_t) rank_nrows; + } + } + + if (use_1d) { + const uint32_t rank_nelem = (uint32_t) kparams->rank_nelem; + const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nelem / 128)); + kparams->n_threads = n_threads; + + uint32_t block_elems = 65536; + if (block_elems > rank_nelem / n_threads && rank_nelem / n_threads > 128) { + block_elems = hex_round_up(rank_nelem / (n_threads * 2), 128); + } + block_elems = (std::max)(128u, block_elems); + + kparams->block_elems = block_elems; + kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; + kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + + while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_elems > 128) { + const size_t max_bytes_per_buf = sess->vtcm_size / (n_threads * n_bufs * 2); + block_elems = (uint32_t) hex_align_down((size_t) (max_bytes_per_buf / elem_size), 128); + if (block_elems < 128) break; + kparams->block_elems = block_elems; + kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; + kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + } + + if (sess->vtcm_size < (size_t) kparams->vtcm_size || block_elems < 128) { + HEX_VERBOSE("ggml-hex: %s allreduce 1D solver failed to fit VTCM (%d > %zu)\n", + sess->c_name(), kparams->vtcm_size, sess->vtcm_size); + return false; + } + + kparams->elems_per_thread = hex_round_up((rank_nelem + n_threads - 1) / n_threads, block_elems); + kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_1D; + return true; + } else { + const uint32_t rank_nrows = (uint32_t) kparams->rank_nelem; + const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nrows)); + kparams->n_threads = n_threads; + + const uint32_t row_bytes = ne0 * elem_size; + const uint32_t row_size_aligned = (uint32_t) hex_align_up(row_bytes, 128); + kparams->row_size_aligned = row_size_aligned; + + const uint32_t nrows_per_thread = (rank_nrows + n_threads - 1) / n_threads; + uint32_t block_rows = (std::min)(128u, nrows_per_thread); + block_rows = (std::max)(1u, block_rows); + kparams->block_elems = block_rows; + + kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); + kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + + while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_rows > 1) { + const size_t max_rows_per_buf = sess->vtcm_size / (n_threads * n_bufs * 2 * row_size_aligned); + block_rows = (std::max)(1u, (uint32_t) max_rows_per_buf); + kparams->block_elems = block_rows; + kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); + kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + if (max_rows_per_buf == 0) break; + } + + if (sess->vtcm_size < (size_t) kparams->vtcm_size || block_rows < 1) { + HEX_VERBOSE("ggml-hex: %s allreduce 2D solver failed to fit VTCM (%d > %zu)\n", + sess->c_name(), kparams->vtcm_size, sess->vtcm_size); + return false; + } + + kparams->elems_per_thread = nrows_per_thread; + kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_2D; + return true; + } +} + +void ggml_hexagon_session::enqueue_allreduce( + const ggml_tensor * dst, + const std::vector & src_tensors, + const std::vector & sync_tensors, + uint32_t rank, + uint32_t n_ranks, + uint32_t fence_seq_entry, + uint32_t fence_seq_exit +) { + htp_opnode ar_node(HTP_OP_ALLREDUCE); + + ggml_tensor* node = ar_node.add_dummy(*dst); + node->op = GGML_OP_NONE; + node->op_params[0] = (int32_t) fence_seq_entry; + node->op_params[1] = (int32_t) fence_seq_exit; + + ar_node.init(node); + + ar_node.inputs.clear(); + for (size_t i = 0; i < src_tensors.size(); i++) { + ar_node.inputs.push_back(src_tensors[i]); + } + for (size_t i = 0; i < sync_tensors.size(); i++) { + ar_node.inputs.push_back(ar_node.add_dummy(*sync_tensors[i])); + } + + ar_node.outputs.clear(); + for (size_t i = 0; i < src_tensors.size(); i++) { + ar_node.outputs.push_back(src_tensors[i]); + } + + ggml_hexagon_precompute_allreduce_params( + this, dst, rank, n_ranks, false, false, + (struct htp_allreduce_kernel_params *) ar_node.kernel_params + ); + + ar_node.name = "ALLREDUCE"; + this->enqueue_op(ar_node); +} + +void ggml_hexagon_session::wait_event(uint64_t seq) { + flush_sync_peers(); + HEX_VERBOSE("ggml-hex: %s opqueue-wait start: seq %llu, current rsp-seq %llu, pending %d\n", + this->name.c_str(), (unsigned long long)seq, (unsigned long long)op_queue->rsp_seq, (int)this->op_pending); + while (op_queue->rsp_seq < seq && this->op_pending > 0) { + this->flush_pending(false); + } + HEX_VERBOSE("ggml-hex: %s opqueue-wait end: seq %llu, current rsp-seq %llu, pending %d\n", + this->name.c_str(), (unsigned long long)seq, (unsigned long long)op_queue->rsp_seq, (int)this->op_pending); +} + +uint64_t ggml_hexagon_session::record_event() { flush_batch(); - flush_pending(all); + return op_queue->req_seq; +} + +bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer *sbuf) +{ + if (this->cloned_buffers.find(sbuf->fd()) != this->cloned_buffers.end()) return true; + + HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(), + sbuf->c_name(), sbuf->base(), sbuf->size(), sbuf->fd()); + + auto clone = std::make_unique(this, *sbuf); + try { + clone->mmap(); + } catch (const std::exception & exc) { + GGML_LOG_ERROR("ggml-hex: %s lazy mapping of buffer context failed: %s\n", this->c_name(), exc.what()); + return false; + } + + this->cloned_buffers[sbuf->fd()] = std::move(clone); + return true; } static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) { @@ -1668,37 +2812,44 @@ static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) { } void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { + const auto & config = opt_device_configs[dev_id]; + int phys_idx = config.physical_idx; + int virt_idx = config.virtual_idx; + this->valid_session = false; this->valid_handle = false; this->valid_queue = false; this->valid_iface = false; - this->domain_id = 3; // Default for CDSP, updated after the session is created - this->session_id = 0; // Default for CDSP, updated after the session is created + this->phys_idx = phys_idx; + this->virt_idx = virt_idx; + this->domain_id = get_domain_id(phys_idx); + this->session_id = 0; this->dev_id = dev_id; - this->name = std::string("HTP") + std::to_string(dev_id); - - this->op_pending = 0; + this->name = config.name; + this->op_pending = 0; GGML_LOG_DEBUG("ggml-hex: %s allocating new session\n", this->name.c_str()); domain * my_domain = htpdrv_get_domain(this->domain_id); if (my_domain == NULL) { - GGML_LOG_ERROR("ggml-hex: unable to get domain struct for CDSP\n"); + GGML_LOG_ERROR("ggml-hex: unable to get domain struct for CDSP (domain_id %d)\n", this->domain_id); throw std::runtime_error("ggml-hex: failed to get CDSP domain (see log for details)"); } - // Create new session - if (dev_id != 0) { + std::string dom_name = get_domain_name(phys_idx); + + // Create new session if virtual_idx > 0 + if (virt_idx > 0) { struct remote_rpc_reserve_new_session n; - n.domain_name_len = strlen(CDSP_DOMAIN_NAME); - n.domain_name = const_cast(CDSP_DOMAIN_NAME); + n.domain_name_len = dom_name.size(); + n.domain_name = const_cast(dom_name.c_str()); n.session_name = const_cast(this->name.c_str()); n.session_name_len = this->name.size(); int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n)); if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: failed to reserve new session %d : error 0x%x\n", dev_id, err); + GGML_LOG_ERROR("ggml-hex: failed to reserve new session %d (physical %d, virtual %d) : error 0x%x\n", dev_id, phys_idx, virt_idx, err); throw std::runtime_error("ggml-hex: remote_session_control(new-sess) failed (see log for details)"); } @@ -1717,8 +2868,8 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { struct remote_rpc_get_uri u = {}; u.session_id = this->session_id; - u.domain_name = const_cast(CDSP_DOMAIN_NAME); - u.domain_name_len = strlen(CDSP_DOMAIN_NAME); + u.domain_name = const_cast(dom_name.c_str()); + u.domain_name_len = dom_name.size(); u.module_uri = const_cast(htp_uri); u.module_uri_len = strlen(htp_uri); u.uri = session_uri; @@ -1731,7 +2882,7 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { snprintf(session_uri, htp_URI_domain_len, "%s%s", htp_uri, my_domain->uri); - GGML_LOG_WARN("ggml-hex: failed to get URI for session %d : error 0x%x. Falling back to single session URI: %s\n", dev_id, err, session_uri); + GGML_LOG_WARN("ggml-hex: failed to get URI for session %d (physical %d, virtual %d) : error 0x%x. Falling back to single session URI: %s\n", dev_id, phys_idx, virt_idx, err, session_uri); } } @@ -1899,14 +3050,17 @@ void ggml_hexagon_session::release() noexcept(true) { if (this->valid_handle) { htp_iface_close(this->handle); } + + this->cloned_buffers.clear(); } ggml_hexagon_session::ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) noexcept(false) { - buffer_type.device = dev; - repack_buffer_type.device = dev; + buffer_type.device = dev; + host_buffer_type.device = dev; op_batch = nullptr; op_queue = nullptr; + fence_seq = ((uintptr_t)this) & 0xFFFF; try { allocate(dev_id); @@ -1914,8 +3068,8 @@ ggml_hexagon_session::ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) n buffer_type.iface = ggml_backend_hexagon_buffer_type_interface; buffer_type.context = new ggml_backend_hexagon_buffer_type_context(this->name, this); - repack_buffer_type.iface = ggml_backend_hexagon_repack_buffer_type_interface; - repack_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(this->name + "-REPACK", this); + host_buffer_type.iface = ggml_backend_hexagon_host_buffer_type_interface; + host_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(this->name + "-HOST", this); } catch (const std::exception & exc) { release(); throw; @@ -1926,7 +3080,7 @@ ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) { release(); delete static_cast(buffer_type.context); - delete static_cast(repack_buffer_type.context); + delete static_cast(host_buffer_type.context); } // ** backend interface @@ -1946,7 +3100,8 @@ static bool ggml_hexagon_flash_attn_is_hmx_eligible( return false; } - if (k->type != GGML_TYPE_F16 || v->type != GGML_TYPE_F16) { + if ((k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0) || + (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_Q8_0)) { return false; } @@ -2098,8 +3253,10 @@ static bool ggml_hexagon_supported_flash_attn_ext(const struct ggml_hexagon_sess const struct ggml_tensor * src4 = op->src[4]; const struct ggml_tensor * dst = op; - // Check for F16 support only as requested - if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) || src1->type != GGML_TYPE_F16 || src2->type != GGML_TYPE_F16) { + // Check for F16/Q8_0 support + if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) || + (src1->type != GGML_TYPE_F16 && src1->type != GGML_TYPE_Q8_0) || + (src2->type != GGML_TYPE_F16 && src2->type != GGML_TYPE_Q8_0)) { return false; } @@ -2352,7 +3509,7 @@ static void ggml_hexagon_precompute_hvx_mm_params( for (uint32_t d = max_prefetch; d >= 2; d /= 2) { htp_mm_hvx_vtcm_layout_build( &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - 0, src0->nb[1], 0, src2_row_size, d, true, false, false + 0, src0->nb[1], 0, src2_row_size, d, true, false ); if (L.total_bytes <= vtcm_budget) { best_n_prefetch = d; @@ -2362,7 +3519,7 @@ static void ggml_hexagon_precompute_hvx_mm_params( if (best_n_prefetch == 2 && L.total_bytes > vtcm_budget) { htp_mm_hvx_vtcm_layout_build( &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - 0, src0->nb[1], 0, src2_row_size, 2, true, false, false + 0, src0->nb[1], 0, src2_row_size, 2, true, false ); } kparams->n_prefetch = best_n_prefetch; @@ -2386,7 +3543,7 @@ static void ggml_hexagon_precompute_hvx_mm_params( for (uint32_t d = max_prefetch; d >= 2; d /= 2) { htp_mm_hvx_vtcm_layout_build( &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, d, false, false, false + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, d, false, false ); if (L.total_bytes <= vtcm_budget) { best_n_prefetch = d; @@ -2396,7 +3553,7 @@ static void ggml_hexagon_precompute_hvx_mm_params( if (best_n_prefetch == 2 && L.total_bytes > vtcm_budget) { htp_mm_hvx_vtcm_layout_build( &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 2, false, false, false + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 2, false, false ); } @@ -2420,7 +3577,7 @@ static void ggml_hexagon_precompute_hvx_mm_params( struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build( &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false, false + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false ); kparams->n_prefetch = 16; @@ -2440,7 +3597,7 @@ static void ggml_hexagon_precompute_hvx_mm_params( struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build( &L, HTP_MM_KERNEL_HVX_F16_F16_VTCM, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false, false + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false ); if (!is_batched && !is_permuted && L.total_bytes <= vtcm_budget) { @@ -2460,7 +3617,7 @@ static void ggml_hexagon_precompute_hvx_mm_params( kparams->src1_row_size = src1->nb[1]; htp_mm_hvx_vtcm_layout_build( &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false, false + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false ); kparams->vtcm_size = L.total_bytes; kparams->vtcm_src0_size = L.src0_bytes; @@ -2476,7 +3633,7 @@ static void ggml_hexagon_precompute_hvx_mm_params( struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build( &L, HTP_MM_KERNEL_HVX_F32_F32_VTCM, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false, false + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false ); if (!is_batched && !is_permuted && L.total_bytes <= vtcm_budget) { @@ -2492,7 +3649,7 @@ static void ggml_hexagon_precompute_hvx_mm_params( kparams->src1_row_size = src1->nb[1]; htp_mm_hvx_vtcm_layout_build( &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false, false + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false ); kparams->vtcm_size = L.total_bytes; kparams->vtcm_src0_size = L.src0_bytes; @@ -2642,10 +3799,99 @@ static void ggml_hexagon_precompute_unary_params( kparams->div_tpr = init_fastdiv_values(tiles_per_row); } -static void ggml_hexagon_precompute_fused_qkv_params( +static void ggml_hexagon_precompute_get_rows_params( const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, // Wk + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_get_rows_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const uint32_t ne00 = src0->ne[0]; + const uint32_t ne02 = src0->ne[2]; + const uint32_t ne03 = src0->ne[3]; + + const uint32_t ne10 = src1->ne[0]; + const uint32_t ne11 = src1->ne[1]; + const uint32_t ne12 = src1->ne[2]; + const uint32_t nr = ne10 * ne11 * ne12; + + const size_t nb01 = src0->nb[1]; + const size_t nb1 = dst->nb[1]; + + const bool can_use_dma = (src0->type == dst->type) && (nb01 == nb1); + const bool use_dma = can_use_dma && (ne00 >= 2048); + + kparams->use_dma = use_dma ? 1 : 0; + + uint32_t chunks_per_row = 1; + uint32_t chunk_size = ne00; + uint32_t total_tasks = nr; + + if (use_dma) { + kparams->n_threads = (std::min)((uint32_t)sess->n_threads, nr); + kparams->tasks_per_thread = (nr + kparams->n_threads - 1) / kparams->n_threads; + } else { + if (src0->type == GGML_TYPE_F32 && nr < sess->n_threads) { + const uint32_t min_chunk_size = 1024; + uint32_t max_chunks = ne00 / min_chunk_size; + if (max_chunks == 0) { + max_chunks = 1; + } + chunks_per_row = (std::min)((sess->n_threads + nr - 1) / nr, max_chunks); + chunk_size = (ne00 + chunks_per_row - 1) / chunks_per_row; + total_tasks = nr * chunks_per_row; + } + kparams->n_threads = (std::min)(total_tasks, (uint32_t)sess->n_threads); + kparams->tasks_per_thread = (total_tasks + kparams->n_threads - 1) / kparams->n_threads; + } + + kparams->chunks_per_row = chunks_per_row; + kparams->chunk_size = chunk_size; + kparams->total_tasks = total_tasks; + + kparams->div_ne10 = init_fastdiv_values(ne10); + kparams->div_ne10_ne11 = init_fastdiv_values(ne10 * ne11); + kparams->div_chunks_per_row = init_fastdiv_values(chunks_per_row); + kparams->div_ne02 = init_fastdiv_values(ne02); + kparams->div_ne03 = init_fastdiv_values(ne03); + + struct htp_get_rows_vtcm_layout vtcm_layout; + htp_get_rows_vtcm_layout_build(&vtcm_layout, src0->type, ne00, kparams->n_threads); + kparams->vtcm_size = vtcm_layout.total_bytes; +} + +static void ggml_hexagon_precompute_set_rows_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, // values + const struct ggml_tensor * src1, // indices + const struct ggml_tensor * dst, // destination + struct htp_set_rows_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const uint32_t nr = src0->ne[1]; + + kparams->n_threads = (std::min)((uint32_t)sess->n_threads, nr); + kparams->tasks_per_thread = (nr + kparams->n_threads - 1) / kparams->n_threads; + kparams->total_tasks = nr; + + kparams->div_ne11 = init_fastdiv_values(src1->ne[1]); + kparams->div_ne12 = init_fastdiv_values(src1->ne[2]); + kparams->div_tasks_per_thread = init_fastdiv_values(kparams->tasks_per_thread); + kparams->div_ne02 = init_fastdiv_values(src0->ne[2]); + + struct htp_set_rows_vtcm_layout vtcm_layout; + htp_set_rows_vtcm_layout_build(&vtcm_layout, dst->type, src0->ne[0], kparams->n_threads); + kparams->vtcm_size = vtcm_layout.total_bytes; +} + +static void ggml_hexagon_precompute_fused_mmnx_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, // W0 const struct ggml_tensor * src1, // x + int32_t n_weights, struct htp_mm_kernel_params * kparams ) { memset(kparams, 0, sizeof(*kparams)); @@ -2667,7 +3913,7 @@ static void ggml_hexagon_precompute_fused_qkv_params( struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build( &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, src1_nrows, sess->n_threads, - 0, src0_row_size, src1_row_size, 0, d, false, true, false + 0, src0_row_size, src1_row_size, 0, d, false, true ); if (L.total_bytes <= sess->vtcm_size) { best_n_prefetch = d; @@ -2682,102 +3928,40 @@ static void ggml_hexagon_precompute_fused_qkv_params( // Test tiled first htp_mm_hvx_vtcm_layout_build( &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, src1_nrows, sess->n_threads, - 0, src0_row_size, src1_row_size, 0, best_n_prefetch, false, true, false + 0, src0_row_size, src1_row_size, 0, best_n_prefetch, false, true ); if (try_tiled && L.total_bytes <= sess->vtcm_size) { kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW; kparams->vtcm_src0_size = L.src0_bytes; kparams->vtcm_src1_size = L.src1_bytes; - kparams->vtcm_src2_size = L.src2_bytes; - kparams->vtcm_src3_size = L.src3_bytes; kparams->vtcm_dst_size = L.dst_bytes; kparams->vtcm_size = L.total_bytes; kparams->n_prefetch = best_n_prefetch; + kparams->n_weights = n_weights; } else { kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT; size_t flat_src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); htp_mm_hvx_vtcm_layout_build( &L, HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT, wtype, ne10, src1_nrows, sess->n_threads, - 0, src0_row_size, flat_src1_row_size, 0, best_n_prefetch, false, true, false + 0, src0_row_size, flat_src1_row_size, 0, best_n_prefetch, false, true ); kparams->vtcm_src0_size = L.src0_bytes; kparams->vtcm_src1_size = L.src1_bytes; - kparams->vtcm_src2_size = L.src2_bytes; - kparams->vtcm_src3_size = L.src3_bytes; kparams->vtcm_dst_size = L.dst_bytes; kparams->vtcm_size = L.total_bytes; kparams->n_prefetch = best_n_prefetch; + kparams->n_weights = n_weights; } } -static void ggml_hexagon_precompute_fused_ffn_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, // Wgate - const struct ggml_tensor * src1, // y - struct htp_mm_kernel_params * kparams -) { - memset(kparams, 0, sizeof(*kparams)); +static bool ggml_hexagon_tensor_is_host(const struct ggml_hexagon_session * sess, const struct ggml_tensor * t) { + return t && t->buffer && t->buffer->buft == &sess->host_buffer_type; +} - const int wtype = src0->type; - const bool is_repack = ggml_hexagon_is_repack_type((ggml_type) wtype); - - const int ne10 = src1->ne[0]; - const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; - const size_t src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); - const size_t src0_row_size = src0->nb[1]; - - uint32_t best_n_prefetch = 16; - - if (is_repack) { - const uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; - best_n_prefetch = 2; - for (uint32_t d = max_prefetch; d >= 2; d /= 2) { - struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build( - &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, src1_nrows, sess->n_threads, - 0, src0_row_size, src1_row_size, 0, d, false, false, true - ); - if (L.total_bytes <= sess->vtcm_size) { - best_n_prefetch = d; - break; - } - } - } - - struct htp_mm_hvx_vtcm_layout L; - bool try_tiled = (opt_mm_select >= 2); - - // Test tiled first - htp_mm_hvx_vtcm_layout_build( - &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, src1_nrows, sess->n_threads, - 0, src0_row_size, src1_row_size, 0, best_n_prefetch, false, false, true - ); - - if (try_tiled && L.total_bytes <= sess->vtcm_size) { - kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW; - kparams->vtcm_src0_size = L.src0_bytes; - kparams->vtcm_src1_size = L.src1_bytes; - kparams->vtcm_src2_size = L.src2_bytes; - kparams->vtcm_dst_size = L.dst_bytes; - kparams->vtcm_size = L.total_bytes; - kparams->n_prefetch = best_n_prefetch; - } else { - kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT; - size_t flat_src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); - - htp_mm_hvx_vtcm_layout_build( - &L, HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT, wtype, ne10, src1_nrows, sess->n_threads, - 0, src0_row_size, flat_src1_row_size, 0, best_n_prefetch, false, false, true - ); - kparams->vtcm_src0_size = L.src0_bytes; - kparams->vtcm_src1_size = L.src1_bytes; - kparams->vtcm_src2_size = L.src2_bytes; - kparams->vtcm_dst_size = L.dst_bytes; - kparams->vtcm_size = L.total_bytes; - kparams->n_prefetch = best_n_prefetch; - } +static bool ggml_hexagon_tensor_is_non_host(const struct ggml_hexagon_session * sess, const struct ggml_tensor * t) { + return t && t->buffer && t->buffer->buft != &sess->host_buffer_type; } static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) { @@ -2811,9 +3995,8 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s return false; // no broadcasting (for now) } - // src0 (weights) must be repacked - if (src0->buffer && !ggml_backend_buffer_is_hexagon_repack(src0->buffer)) { - return false; + if (!src0->buffer) { + sess->needs_repack.insert(src0); } break; @@ -2872,9 +4055,8 @@ static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session return false; } - // src0 (weights) must be repacked - if (src0->buffer && !ggml_backend_buffer_is_hexagon_repack(src0->buffer)) { - return false; + if (!src0->buffer) { + sess->needs_repack.insert(src0); } break; @@ -2970,7 +4152,7 @@ static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * ses if (dst->type != GGML_TYPE_F32) { return false; } - if (ggml_is_permuted(src0)) { + if (!ggml_is_contiguous_rows(src0)) { return false; } if (!ggml_are_same_shape(src0, dst)) { @@ -3114,7 +4296,11 @@ static bool ggml_hexagon_supported_softmax(const struct ggml_hexagon_session * s static bool ggml_hexagon_supported_set_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { const struct ggml_tensor * src0 = op->src[0]; // values const struct ggml_tensor * src1 = op->src[1]; // indices - const struct ggml_tensor * dst = op; + const struct ggml_tensor * dst = op->src[2] ? op->src[2] : op; + + if (dst->type == GGML_TYPE_Q8_0 && src0->ne[0] < 32) { + return false; + } if (src0->type != GGML_TYPE_F32) { return false; @@ -3124,7 +4310,7 @@ static bool ggml_hexagon_supported_set_rows(const struct ggml_hexagon_session * return false; } - if (dst->type != GGML_TYPE_F16) { + if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_Q8_0) { return false; } @@ -3138,7 +4324,11 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * const struct ggml_tensor * src1 = op->src[1]; // indices const struct ggml_tensor * dst = op; - if (src0->type != GGML_TYPE_F32) { + if (src0->type != GGML_TYPE_F32 && src0->ne[0] < 32) { + return false; + } + + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_Q8_0) { return false; } @@ -3530,8 +4720,7 @@ static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor if (n1->src[1] != n2->src[1]) { return false; } - if (n1->src[0]->ne[0] != n2->src[0]->ne[0] || - n1->src[0]->ne[1] != n2->src[0]->ne[1]) { + if (n1->src[0]->ne[0] != n2->src[0]->ne[0]) { return false; } if (n1->src[0]->type != n2->src[0]->type) { @@ -3540,116 +4729,6 @@ static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor return true; } -static bool is_qkv_mergeable(const ggml_tensor * n_q, const ggml_tensor * n_k, const ggml_tensor * n_v) { - if (!is_mergeable_mul_mat(n_q) || !is_mergeable_mul_mat(n_k) || !is_mergeable_mul_mat(n_v)) { - return false; - } - if (n_q->src[1] != n_k->src[1] || n_q->src[1] != n_v->src[1]) { - return false; - } - if (n_q->src[0]->type != n_k->src[0]->type || n_q->src[0]->type != n_v->src[0]->type) { - return false; - } - if (n_k->src[0]->ne[0] != n_v->src[0]->ne[0] || - n_k->src[0]->ne[1] != n_v->src[0]->ne[1]) { - return false; - } - if (n_q->src[0]->ne[0] != n_k->src[0]->ne[0]) { - return false; - } - return true; -} - -static bool try_fuse_node(const ggml_hexagon_session * sess, const ggml_cgraph * graph, int & i, std::vector & nodes) { - if (!opt_opfusion) { - return false; - } - - ggml_tensor * n = graph->nodes[i]; - ggml_tensor * next_node = (i + 1 < graph->n_nodes) ? graph->nodes[i + 1] : nullptr; - - if (n->op == GGML_OP_RMS_NORM && next_node) { - if (next_node->op == GGML_OP_MUL && op_is_compute(next_node) && ggml_can_fuse(graph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { - htp_opnode node(n, {}, HTP_OP_RMS_NORM_MUL); - node.add_fused(next_node); - - auto inputs = node.get_inputs(); - const struct ggml_tensor * src0 = inputs[0]; - const struct ggml_tensor * src1 = inputs.size() > 1 ? inputs[1] : nullptr; - ggml_hexagon_precompute_unary_params(sess, - node.opcode, src0, src1, node.dst(), - (struct htp_unary_kernel_params *)node.kernel_params - ); - - nodes.push_back(std::move(node)); - i++; // skip the fused MUL node - return true; - } - } - - if (is_mergeable_mul_mat(n)) { - ggml_tensor * n1 = (i + 1 < graph->n_nodes) ? graph->nodes[i + 1] : nullptr; - ggml_tensor * n2 = (i + 2 < graph->n_nodes) ? graph->nodes[i + 2] : nullptr; - if (is_qkv_mergeable(n, n1, n2)) { - struct htp_mm_kernel_params kparams; - ggml_hexagon_precompute_fused_qkv_params(sess, n1->src[0], n1->src[1], &kparams); - if ((size_t)kparams.vtcm_size <= sess->vtcm_size) { - // Reorder to KVQ: K (n1), V (n2), Q (n) - htp_opnode node(n1, {}, HTP_OP_MUL_MAT_QKV); - node.add_fused(n2, true); - node.add_fused(n, true); - memcpy(node.kernel_params, &kparams, sizeof(kparams)); - nodes.push_back(std::move(node)); - i += 2; - return true; - } else { - HEX_VERBOSE("ggml-hex: skip QKV fusion because VTCM needed (%d) > budget (%zu)\n", - kparams.vtcm_size, sess->vtcm_size); - } - } - if (is_mergeable_mul_mat_pair(n, n1)) { - struct htp_mm_kernel_params kparams; - ggml_hexagon_precompute_fused_ffn_params(sess, n->src[0], n->src[1], &kparams); - if ((size_t)kparams.vtcm_size <= sess->vtcm_size) { - htp_opnode node(n, {}, HTP_OP_MUL_MAT_FFN); - node.add_fused(n1, true); - memcpy(node.kernel_params, &kparams, sizeof(kparams)); - nodes.push_back(std::move(node)); - i += 1; - return true; - } else { - HEX_VERBOSE("ggml-hex: skip FFN fusion because VTCM needed (%d) > budget (%zu)\n", - kparams.vtcm_size, sess->vtcm_size); - } - } - } - - if (n->op == GGML_OP_MUL_MAT && next_node) { - if (next_node->op == GGML_OP_ADD && op_is_compute(next_node) && ggml_can_fuse(graph, i, { GGML_OP_MUL_MAT, GGML_OP_ADD })) { - if (next_node->src[0] == n || next_node->src[1] == n) { - const struct ggml_tensor * src2 = (next_node->src[0] == n) ? next_node->src[1] : next_node->src[0]; - struct htp_mm_kernel_params kparams; - ggml_hexagon_precompute_fused_matmul_add_params(sess, n->src[0], n->src[1], src2, next_node, &kparams); - const int src1_nrows = n->src[1]->ne[1] * n->src[1]->ne[2] * n->src[1]->ne[3]; - const bool can_fuse = (kparams.n_hmx > 0) || (src1_nrows == 1); - if (can_fuse && (size_t)kparams.vtcm_size <= sess->vtcm_size) { - htp_opnode node(n, {}, HTP_OP_MUL_MAT_ADD); - node.add_fused(next_node); - memcpy(node.kernel_params, &kparams, sizeof(kparams)); - nodes.push_back(std::move(node)); - i += 1; - return true; - } else if (can_fuse) { - HEX_VERBOSE("ggml-hex: skip MUL_MAT_ADD fusion because VTCM needed (%d) > budget (%zu)\n", - kparams.vtcm_size, sess->vtcm_size); - } - } - } - } - - return false; -} - static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) { auto sess = static_cast(backend->context); @@ -3659,24 +4738,34 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg std::vector computed_nodes; // Check for cache hit - bool cache_hit = (graph->uid != 0 && sess->cached_graph.uid == graph->uid); + bool cache_hit = (graph->uid != 0 && sess->cached_uid == graph->uid); if (cache_hit) { - nodes_ptr = &sess->cached_graph.htp_nodes; + nodes_ptr = &sess->cached_nodes; } else { + // Tag fusable tensors in graph + for (int i = 0; i < graph->n_nodes; i++) { + auto * extra = (ggml_hexagon_tensor_extra *) graph->nodes[i]->extra; + if (!extra) continue; + + if (graph->nodes[i]->op == GGML_OP_RMS_NORM && ggml_can_fuse(graph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { + extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE; + } else if (graph->nodes[i]->op == GGML_OP_MUL_MAT) { + if ((i + 1 < graph->n_nodes && graph->nodes[i + 1]->op == GGML_OP_ADD && ggml_can_fuse(graph, i, { GGML_OP_MUL_MAT, GGML_OP_ADD })) || + ggml_node_has_n_uses(graph, i, 1)) { + extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE; + } + } + } + computed_nodes.reserve(graph->n_nodes); - // Fuse and finalize for (int i = 0; i < graph->n_nodes; ++i) { ggml_tensor * n = graph->nodes[i]; if (!op_is_compute(n)) { continue; } - if (try_fuse_node(sess, graph, i, computed_nodes)) { - continue; - } - - htp_opnode node(n, {}, HTP_OP_INVALID); + htp_opnode node(HTP_OP_INVALID, n); node.opcode = op_remap_to_htp(n); if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID) { ggml_hexagon_precompute_matmul_params(sess, @@ -3696,29 +4785,34 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg node.opcode, src0, src1, node.dst(), (struct htp_unary_kernel_params *)node.kernel_params ); + } else if (node.opcode == HTP_OP_GET_ROWS) { + ggml_hexagon_precompute_get_rows_params(sess, + node.node->src[0], node.node->src[1], node.dst(), + (struct htp_get_rows_kernel_params *)node.kernel_params + ); + } else if (node.opcode == HTP_OP_SET_ROWS) { + ggml_hexagon_precompute_set_rows_params(sess, + node.node->src[0], node.node->src[1], node.dst(), + (struct htp_set_rows_kernel_params *)node.kernel_params + ); } computed_nodes.push_back(std::move(node)); } if (graph->uid != 0) { - sess->cached_graph.uid = graph->uid; - sess->cached_graph.htp_nodes = std::move(computed_nodes); - nodes_ptr = &sess->cached_graph.htp_nodes; + sess->cached_uid = graph->uid; + sess->cached_nodes = std::move(computed_nodes); + nodes_ptr = &sess->cached_nodes; } else { nodes_ptr = &computed_nodes; } } // Queue and execute - if (opt_opstage & HTP_OPSTAGE_QUEUE) { - for (const auto & node : *nodes_ptr) { - sess->enqueue_op(node); - } + for (const auto & node : *nodes_ptr) { + sess->enqueue_op(node); } - // Wait until all pending ops complete - sess->flush(); - return GGML_STATUS_SUCCESS; } @@ -3731,6 +4825,106 @@ static void ggml_backend_hexagon_synchronize(ggml_backend_t backend) { sess->flush(); } +enum ggml_hexagon_mem_range_type { + HEXAGON_MEM_RANGE_TYPE_SRC, + HEXAGON_MEM_RANGE_TYPE_DST, +}; + +struct ggml_hexagon_mem_range { + uint64_t pb; + uint64_t p0; + uint64_t p1; + ggml_hexagon_mem_range_type pt; +}; + +struct ggml_hexagon_mem_ranges { + std::vector ranges; + + void reset() { + ranges.clear(); + } + + void add(const ggml_hexagon_mem_range & mr) { + ranges.push_back(mr); + } + + bool check(const ggml_hexagon_mem_range & mr) const { + for (const auto & cmp : ranges) { + if (mr.pb != cmp.pb) { + continue; + } + if (mr.pt == HEXAGON_MEM_RANGE_TYPE_SRC && cmp.pt == HEXAGON_MEM_RANGE_TYPE_SRC) { + continue; + } + if (mr.p0 < cmp.p1 && mr.p1 > cmp.p0) { + return false; + } + } + return true; + } +}; + +static ggml_hexagon_mem_range ggml_hexagon_mem_range_from_tensor(const ggml_tensor * tensor, ggml_hexagon_mem_range_type pt) { + const ggml_tensor * base = tensor->view_src ? tensor->view_src : tensor; + ggml_hexagon_mem_range mr; + if (tensor->buffer) { + mr = { + /*.pb =*/ (uint64_t) tensor->buffer, + /*.p0 =*/ (uint64_t) tensor->data, + /*.p1 =*/ (uint64_t) tensor->data + ggml_backend_buft_get_alloc_size(tensor->buffer->buft, tensor), + /*.pt =*/ pt, + }; + } else { + mr = { + /*.pb =*/ (uint64_t) base, + /*.p0 =*/ 0, + /*.p1 =*/ 1024, + /*.pt =*/ pt, + }; + } + return mr; +} + +static void ggml_hexagon_mem_ranges_add_node(ggml_hexagon_mem_ranges & mrs, const htp_opnode & node) { + if (node.is_empty()) return; + + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (node.node->src[i]) { + mrs.add(ggml_hexagon_mem_range_from_tensor(node.node->src[i], HEXAGON_MEM_RANGE_TYPE_SRC)); + } + } + for (const auto * fused : node.fused) { + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (fused->src[i]) { + mrs.add(ggml_hexagon_mem_range_from_tensor(fused->src[i], HEXAGON_MEM_RANGE_TYPE_SRC)); + } + } + } + mrs.add(ggml_hexagon_mem_range_from_tensor(node.dst(), HEXAGON_MEM_RANGE_TYPE_DST)); +} + +static bool ggml_hexagon_mem_ranges_check_node(const ggml_hexagon_mem_ranges & mrs, const htp_opnode & node) { + if (node.is_empty()) return true; + + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (node.node->src[i]) { + if (!mrs.check(ggml_hexagon_mem_range_from_tensor(node.node->src[i], HEXAGON_MEM_RANGE_TYPE_SRC))) { + return false; + } + } + } + for (const auto * fused : node.fused) { + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (fused->src[i]) { + if (!mrs.check(ggml_hexagon_mem_range_from_tensor(fused->src[i], HEXAGON_MEM_RANGE_TYPE_SRC))) { + return false; + } + } + } + } + return mrs.check(ggml_hexagon_mem_range_from_tensor(node.dst(), HEXAGON_MEM_RANGE_TYPE_DST)); +} + static std::vector ggml_hexagon_graph_optimize_reorder(const std::vector & nodes) { const int n = nodes.size(); @@ -3739,28 +4933,32 @@ static std::vector ggml_hexagon_graph_optimize_reorder(const std::vector used(n, false); - // The main goal here is to stack the MUL_MAT ops with the same src1 input. - // This allows use to reuse dynamically quantized src1 in VTCM. + ggml_hexagon_mem_ranges mrs; - // TODO: the current version might do incorrect reordering in cases where quantized src0 - // input is an output of another Op. + // The main goal here is to stack the MUL_MAT ops with the same src1 input. + // This allows us to reuse dynamically quantized src1 in VTCM. for (int i0 = 0; i0 < n; i0++) { if (used[i0]) { continue; } - res.push_back(i0); - const auto & node0 = nodes[i0]; if (!node0.stackable()) { + res.push_back(i0); + used[i0] = true; continue; } // that many nodes forward to search for stackable nodes that can reuse VTCM constexpr int N_FORWARD = 16; + std::vector stack; + stack.push_back(i0); + + mrs.reset(); + for (int i1 = i0 + 1; i1 < i0 + N_FORWARD && i1 < n; i1++) { if (used[i1]) { continue; @@ -3768,11 +4966,17 @@ static std::vector ggml_hexagon_graph_optimize_reorder(const std::vector nodes; nodes.reserve(gf->n_nodes); - // fuse nodes: - // we don't want to make reorders that break fusing, so we first pack all fusable tensors - // and perform the reorder over the fused nodes. after the reorder is done, we unfuse + // Pack nodes for reordering for (int i = 0; i < n; i++) { - htp_opnode node = { - /*.node =*/gf->nodes[i], - /*.fused =*/{}, - }; + htp_opnode node(HTP_OP_INVALID, gf->nodes[i]); // fuse only ops that start with these operations // can be expanded when needed @@ -3856,22 +5055,200 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgr GGML_UNUSED(backend); } +static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) { + auto sess_src = static_cast(backend_src->context); + auto sess_dst = static_cast(backend_dst->context); + auto sbuf_dst = (ggml_hexagon_shared_buffer *) dst->buffer->context; + + if (sess_dst->fence_seq == 0) sess_dst->fence_seq = 1; + uint32_t fence_seq = sess_dst->fence_seq++; + if (sess_dst->fence_seq == 0) sess_dst->fence_seq = 1; + + volatile uint32_t * fence = (volatile uint32_t *) sbuf_dst->alloc_fence(); + + HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq %u\n", + sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src), fence_seq); + + // dummy extra (must be static) + static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; + + ggml_tensor fence_tensor {}; + fence_tensor.buffer = dst->buffer; + fence_tensor.extra = &fence_extra; + fence_tensor.data = (void *) fence; + fence_tensor.type = GGML_TYPE_I32; + fence_tensor.ne[0] = 1; + fence_tensor.ne[1] = 1; + fence_tensor.ne[2] = 1; + fence_tensor.ne[3] = 1; + fence_tensor.nb[0] = sizeof(int32_t); + fence_tensor.nb[1] = sizeof(int32_t); + fence_tensor.nb[2] = sizeof(int32_t); + fence_tensor.nb[3] = sizeof(int32_t); + fence_tensor.op = GGML_OP_NONE; + + sess_src->enqueue_cpy(src, dst, &fence_tensor, fence_seq); + sess_dst->enqueue_fence(&fence_tensor, fence_seq); + + sess_dst->add_sync_peer(sess_src); + + return true; +} + +static bool ggml_hexagon_cpy_tensor_async_virt(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) { + auto sess_src = static_cast(backend_src->context); + auto sess_dst = static_cast(backend_dst->context); + auto sbuf_dst = (ggml_hexagon_shared_buffer *) dst->buffer->context; + + if (!sess_src->clone_buffer(sbuf_dst)) { return false; } + + HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n", + sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src)); + + sess_src->enqueue_cpy(src, dst); + sess_src->flush(true); + + return true; +} + +static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) { + if (!ggml_backend_is_hexagon(backend_src) || !ggml_backend_is_hexagon(backend_dst)) { + return false; + } + + *(ggml_hexagon_tensor_extra *) dst->extra = *(const ggml_hexagon_tensor_extra *) src->extra; + + auto sess_src = static_cast(backend_src->context); + auto sess_dst = static_cast(backend_dst->context); + + if (sess_src == sess_dst) { + HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n", sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src)); + sess_src->enqueue_cpy(src, dst); + sess_src->flush_batch(); + return true; + } + + if (sess_src->phys_idx != sess_dst->phys_idx) + return ggml_hexagon_cpy_tensor_async_phys(backend_src, backend_dst, src, dst); + + return ggml_hexagon_cpy_tensor_async_virt(backend_src, backend_dst, src, dst); +} + +static ggml_backend_event_t ggml_backend_hexagon_device_event_new(ggml_backend_dev_t dev) { + ggml_hexagon_event * hex_event = new ggml_hexagon_event(); + HEX_VERBOSE("ggml-hex: %s event-new : event %p\n", ggml_backend_dev_name(dev), (void *)hex_event); + + return new ggml_backend_event { + /* .device = */ dev, + /* .context = */ hex_event, + }; +} + +static void ggml_backend_hexagon_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) { + GGML_UNUSED(dev); + + if (event == nullptr) { + return; + } + + ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; + HEX_VERBOSE("ggml-hex: %s event-free : event %p\n", ggml_backend_dev_name(dev), (void *)hex_event); + delete hex_event; + delete event; +} + +static void ggml_backend_hexagon_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) { + GGML_UNUSED(dev); + + ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; + HEX_VERBOSE("ggml-hex: %s event-synchronize : event %p seq %llu\n", + ggml_backend_dev_name(dev), (void *)hex_event, (unsigned long long)hex_event->seq); + if (hex_event->sess != nullptr) { + hex_event->sess->wait_event(hex_event->seq); + } +} + +static void ggml_backend_hexagon_event_record(ggml_backend_t backend, ggml_backend_event_t event) { + auto sess = static_cast(backend->context); + ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; + + hex_event->sess = sess; + hex_event->seq = sess->record_event(); + HEX_VERBOSE("ggml-hex: %s event-record : event %p seq %llu\n", + sess->c_name(), (void *)hex_event, (unsigned long long)hex_event->seq); +} + +static void ggml_backend_hexagon_event_wait(ggml_backend_t backend, ggml_backend_event_t event) { + GGML_UNUSED(backend); + + ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; + if (hex_event->sess != nullptr) { + HEX_VERBOSE("ggml-hex: %s event-wait : event %p seq %llu\n", + hex_event->sess->c_name(), (void *)hex_event, (unsigned long long)hex_event->seq); + hex_event->sess->wait_event(hex_event->seq); + } +} + +static void ggml_backend_hexagon_set_tensor_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) { + auto sess = static_cast(backend->context); + HEX_VERBOSE("ggml-hex: %s set-tensor-async %s : data %p offset %zu size %zu usage %d\n", + sess->c_name(), tensor->name, data, offset, size, tensor->buffer ? (int) tensor->buffer->usage : -1); + ggml_backend_tensor_set(tensor, data, offset, size); +} + +static void ggml_backend_hexagon_get_tensor_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) { + auto sess = static_cast(backend->context); + HEX_VERBOSE("ggml-hex: %s get-tensor-async %s : data %p offset %zu size %zu usage %d\n", + sess->c_name(), tensor->name, data, offset, size, tensor->buffer ? (int) tensor->buffer->usage : -1); + sess->flush(true); + ggml_backend_tensor_get(tensor, data, offset, size); +} + +static void ggml_backend_hexagon_set_tensor_2d_async(ggml_backend_t backend, + struct ggml_tensor * tensor, + const void * data, + size_t offset, + size_t size, + size_t n_copies, + size_t stride_tensor, + size_t stride_data) { + auto sess = static_cast(backend->context); + HEX_VERBOSE("ggml-hex: %s set-tensor-2d-async %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n", + sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, tensor->buffer ? (int) tensor->buffer->usage : -1); + ggml_backend_tensor_set_2d(tensor, data, offset, size, n_copies, stride_tensor, stride_data); +} + +static void ggml_backend_hexagon_get_tensor_2d_async(ggml_backend_t backend, + const struct ggml_tensor * tensor, + void * data, + size_t offset, + size_t size, + size_t n_copies, + size_t stride_tensor, + size_t stride_data) { + auto sess = static_cast(backend->context); + HEX_VERBOSE("ggml-hex: %s get-tensor-2d-async %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n", + sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, tensor->buffer ? (int) tensor->buffer->usage : -1); + sess->flush(true); + ggml_backend_tensor_get_2d(tensor, data, offset, size, n_copies, stride_tensor, stride_data); +} + static struct ggml_backend_i hexagon_backend_i = { /* .get_name = */ ggml_backend_hexagon_name, /* .free = */ ggml_backend_hexagon_free, - /* .set_tensor_async = */ NULL, - /* .get_tensor_async = */ NULL, - /* .set_tensor_2d_async = */ NULL, - /* .get_tensor_2d_async = */ NULL, - /* .cpy_tensor_async = */ NULL, + /* .set_tensor_async = */ ggml_backend_hexagon_set_tensor_async, + /* .get_tensor_async = */ ggml_backend_hexagon_get_tensor_async, + /* .set_tensor_2d_async = */ ggml_backend_hexagon_set_tensor_2d_async, + /* .get_tensor_2d_async = */ ggml_backend_hexagon_get_tensor_2d_async, + /* .cpy_tensor_async = */ ggml_backend_hexagon_cpy_tensor_async, /* .synchronize = */ ggml_backend_hexagon_synchronize, /* .graph_plan_create = */ NULL, /* .graph_plan_free = */ NULL, /* .graph_plan_update = */ NULL, /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_hexagon_graph_compute, - /* .event_record = */ NULL, - /* .event_wait = */ NULL, + /* .event_record = */ ggml_backend_hexagon_event_record, + /* .event_wait = */ ggml_backend_hexagon_event_wait, /* .graph_optimize = */ ggml_backend_hexagon_graph_optimize, }; @@ -3932,9 +5309,9 @@ static void ggml_backend_hexagon_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_hexagon_device_get_memory(dev, &props->memory_free, &props->memory_total); props->caps = { /* .async = */ true, - /* .host_buffer = */ (bool) opt_hostbuf, + /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ false, - /* .events = */ false, + /* .events = */ true, /* .mmap_support = */ false, }; } @@ -3944,32 +5321,12 @@ static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_buffer_type(gg return &sess->buffer_type; } -static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_repack_buffer_type(ggml_backend_dev_t dev) { +static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_host_buffer_type(ggml_backend_dev_t dev) { + if (!opt_hostbuf) { + return NULL; + } auto sess = static_cast(dev->context); - return &sess->repack_buffer_type; -} - -static bool ggml_hexagon_supported_buffer(ggml_hexagon_session *sess, const struct ggml_tensor * t) { - if (t && t->buffer) { - if (ggml_backend_buffer_is_hexagon(t->buffer) == false) return false; // not our buffer - if (ggml_backend_hexagon_buffer_get_sess(t->buffer) != sess) return false; // wrong session - } - return true; -} - -static bool ggml_hexagon_supported_buffers(ggml_hexagon_session *sess, const struct ggml_tensor * t) { - // all srcs & dsts must be mapped to the same session - if (!ggml_hexagon_supported_buffer(sess, t)) { - return false; - } - - for (int i = 0; i < GGML_MAX_SRC; i++) { - if (!ggml_hexagon_supported_buffer(sess, t->src[i])) { - return false; - } - } - - return true; + return &sess->host_buffer_type; } static bool ggml_hexagon_supported_cpy(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -4067,12 +5424,6 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons return false; } - // all srcs & dsts must be mapped to the same session - if (!ggml_hexagon_supported_buffers(sess, op)) { - ggml_hexagon_dump_op_supp(sess->name, op, false); - return false; - } - bool supp = false; switch (op->op) { case GGML_OP_NONE: @@ -4233,31 +5584,20 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons } static bool ggml_backend_hexagon_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { - if (buft->iface.get_alignment != ggml_backend_hexagon_buffer_type_get_alignment) { - return false; - } + auto sess = static_cast(dev->context); - auto s0 = static_cast(dev->context); - auto s1 = static_cast(buft->context)->sess; - - // Need session/domain-id for buffers to be compatible - bool supp = (s0->session_id == s1->session_id); - - HEX_VERBOSE("ggml-hex: %s device-supports-buft %s (%d)\n", s0->name.c_str(), s1->name.c_str(), (int) supp); + // Technically we can clone hexagon buffers from any session but for some reason the output is garbled with layer-split, + // tensor-split works correctly, so it needs mode debugging and investigation. For now accept only our own buffers. +#if 0 + bool supp = (buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment); +#else + bool supp = (buft == &sess->host_buffer_type) || (buft == &sess->buffer_type); +#endif + HEX_VERBOSE("ggml-hex: %s device-supports-buft %s %s\n", sess->name.c_str(), ggml_backend_buft_name(buft), supp ? "yes" : "no"); return supp; } -static ggml_backend_buffer_type_t * ggml_backend_hexagon_device_get_extra_buffers_type(ggml_backend_dev_t dev) { - auto s0 = static_cast(dev->context); - HEX_VERBOSE("ggml-hex: device-get-extra-buft : %s \n", s0->name.c_str()); - - static ggml_backend_buffer_type_t bufts[2]; - bufts[0] = ggml_backend_hexagon_device_get_repack_buffer_type(dev); - bufts[1] = NULL; - return bufts; -} - static const struct ggml_backend_device_i ggml_backend_hexagon_device_i = { /* .get_name = */ ggml_backend_hexagon_device_get_name, /* .get_description = */ ggml_backend_hexagon_device_get_description, @@ -4266,27 +5606,18 @@ static const struct ggml_backend_device_i ggml_backend_hexagon_device_i = { /* .get_props = */ ggml_backend_hexagon_device_get_props, /* .init_backend = */ ggml_backend_hexagon_device_init, /* .get_buffer_type = */ ggml_backend_hexagon_device_get_buffer_type, - /* .get_host_buffer_type = */ NULL, // ggml_backend_hexagon_device_get_host_buffer_type, + /* .get_host_buffer_type = */ ggml_backend_hexagon_device_get_host_buffer_type, /* .buffer_from_host_ptr = */ NULL, // ggml_backend_hexagon_device_buffer_from_ptr, /* .supports_op = */ ggml_backend_hexagon_device_supports_op, /* .supports_buft = */ ggml_backend_hexagon_device_supports_buft, /* .offload_op = */ NULL, // ggml_backend_hexagon_device_offload_op, - /* .event_new = */ NULL, - /* .event_free = */ NULL, - /* .event_synchronize = */ NULL, + /* .event_new = */ ggml_backend_hexagon_device_event_new, + /* .event_free = */ ggml_backend_hexagon_device_event_free, + /* .event_synchronize = */ ggml_backend_hexagon_device_event_synchronize, }; //** backend registry -#define GGML_HEXAGON_MAX_SESSIONS 16 - -struct ggml_hexagon_registry { - ggml_hexagon_registry(ggml_backend_reg_t reg); - ~ggml_hexagon_registry(); - - ggml_backend_device devices[GGML_HEXAGON_MAX_SESSIONS]; -}; - ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) { GGML_LOG_INFO("ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev %zu\n", opt_ndev); @@ -4303,6 +5634,7 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) { devices[i].context = nullptr; } } + } ggml_hexagon_registry::~ggml_hexagon_registry() { @@ -4335,14 +5667,129 @@ static ggml_backend_dev_t ggml_backend_hexagon_reg_get_device(ggml_backend_reg_t return &hreg->devices[index]; } -static void * ggml_backend_hexagon_get_proc_address(ggml_backend_reg_t reg, const char * name) { - if (strcmp(name, "ggml_backend_dev_get_extra_bufts") == 0 && opt_hostbuf) { - ggml_backend_dev_get_extra_bufts_t fct = ggml_backend_hexagon_device_get_extra_buffers_type; - return (void *) fct; +// ** communication context for tensor-split allreduce + +static void * ggml_backend_hexagon_comm_init(ggml_backend_t * backends, size_t n_backends) { + if (n_backends < 2 || n_backends > 4) { + return nullptr; } - return NULL; + for (size_t i = 0; i < n_backends; ++i) { + if (!ggml_backend_is_hexagon(backends[i])) { + return nullptr; + } + } + + auto * ctx = new ggml_backend_hexagon_comm_context(); + ctx->backends.assign(backends, backends + n_backends); + ctx->n_backends = n_backends; + ctx->fence_seq = (((uintptr_t) ctx) & 0xFFFF) | 1; + + return ctx; +} + +static void ggml_backend_hexagon_comm_free(void * comm_ctx_v) { + if (!comm_ctx_v) return; + delete static_cast(comm_ctx_v); +} + +static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tensor ** tensors) { + if (opt_ar_select == 0 || !comm_ctx_v) return false; + auto * comm_ctx = static_cast(comm_ctx_v); + const size_t n_backends = comm_ctx->n_backends; + + if (n_backends < 2 || n_backends > 4) return false; + + for (size_t i = 0; i < n_backends; i++) { + if (!tensors[i] || !tensors[i]->buffer || !ggml_backend_buffer_is_hexagon(tensors[i]->buffer)) { + return false; + } + if (tensors[i]->type != tensors[0]->type) { + return false; + } + if (!ggml_is_contiguous(tensors[i])) { + return false; + } + if (ggml_nelements(tensors[i]) != ggml_nelements(tensors[0])) { + return false; + } + } + + if (tensors[0]->type != GGML_TYPE_F16 && tensors[0]->type != GGML_TYPE_F32) { + return false; + } + + for (size_t r = 0; r < n_backends; r++) { + auto sess = static_cast(comm_ctx->backends[r]->context); + struct htp_allreduce_kernel_params kparams; + if (!ggml_hexagon_precompute_allreduce_params(sess, tensors[r], (uint32_t) r, (uint32_t) n_backends, false, false, &kparams)) { + return false; + } + } + + if (comm_ctx->fence_seq == 0) comm_ctx->fence_seq = 1; + uint32_t fence_seq_entry = comm_ctx->fence_seq++; + if (comm_ctx->fence_seq == 0) comm_ctx->fence_seq = 1; + uint32_t fence_seq_exit = comm_ctx->fence_seq++; + if (comm_ctx->fence_seq == 0) comm_ctx->fence_seq = 1; + + volatile uint32_t * fences[GGML_HEXAGON_MAX_SESSIONS]; + for (size_t i = 0; i < n_backends; i++) { + auto sbuf = (ggml_hexagon_shared_buffer *) tensors[i]->buffer->context; + fences[i] = (volatile uint32_t *) sbuf->alloc_fence(); + } + + static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; + ggml_tensor fence_tensors[GGML_HEXAGON_MAX_SESSIONS]; + for (size_t i = 0; i < n_backends; i++) { + fence_tensors[i] = {}; + fence_tensors[i].buffer = tensors[i]->buffer; + fence_tensors[i].extra = &fence_extra; + fence_tensors[i].data = (void *) fences[i]; + fence_tensors[i].type = GGML_TYPE_I32; + fence_tensors[i].ne[0] = 4; + fence_tensors[i].ne[1] = 1; + fence_tensors[i].ne[2] = 1; + fence_tensors[i].ne[3] = 1; + fence_tensors[i].nb[0] = sizeof(int32_t); + fence_tensors[i].nb[1] = sizeof(int32_t); + fence_tensors[i].nb[2] = sizeof(int32_t); + fence_tensors[i].nb[3] = sizeof(int32_t); + fence_tensors[i].op = GGML_OP_NONE; + } + + std::vector data_tensors(n_backends); + std::vector sync_tensors(n_backends); + for (size_t i = 0; i < n_backends; i++) { + data_tensors[i] = tensors[i]; + sync_tensors[i] = &fence_tensors[i]; + } + + for (size_t r = 0; r < n_backends; r++) { + auto sess = static_cast(comm_ctx->backends[r]->context); + sess->enqueue_allreduce(tensors[r], data_tensors, sync_tensors, (uint32_t) r, (uint32_t) n_backends, fence_seq_entry, fence_seq_exit); + for (size_t j = 0; j < n_backends; j++) { + if (r != j) { + sess->add_sync_peer(static_cast(comm_ctx->backends[j]->context)); + } + } + } + + return true; +} + +static void * ggml_backend_hexagon_get_proc_address(ggml_backend_reg_t reg, const char * name) { GGML_UNUSED(reg); + if (strcmp(name, "ggml_backend_comm_init") == 0) { + return (void *) ggml_backend_hexagon_comm_init; + } + if (strcmp(name, "ggml_backend_comm_free") == 0) { + return (void *) ggml_backend_hexagon_comm_free; + } + if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) { + return (void *) ggml_backend_hexagon_comm_allreduce_tensor; + } + return NULL; } template std::vector str_to_vec(const char* str) { @@ -4379,8 +5826,6 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { "please update hexagon_type to match ggml_type"); const char * str_verbose = getenv("GGML_HEXAGON_VERBOSE"); - const char * str_hostbuf = getenv("GGML_HEXAGON_HOSTBUF"); - const char * str_opstage = getenv("GGML_HEXAGON_OPSTAGE"); const char * str_opbatch = getenv("GGML_HEXAGON_OPBATCH"); const char * str_opqueue = getenv("GGML_HEXAGON_OPQUEUE"); const char * str_oppoll = getenv("GGML_HEXAGON_OPPOLL"); @@ -4389,15 +5834,16 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { const char * str_profile = getenv("GGML_HEXAGON_PROFILE"); const char * str_etm = getenv("GGML_HEXAGON_ETM"); const char * str_nhvx = getenv("GGML_HEXAGON_NHVX"); - const char * str_use_hmx = getenv("GGML_HEXAGON_USE_HMX"); const char * str_nhmx = getenv("GGML_HEXAGON_NHMX"); const char * str_mm_select = getenv("GGML_HEXAGON_MM_SELECT"); const char * str_fa_select = getenv("GGML_HEXAGON_FA_SELECT"); + const char * str_ar_select = getenv("GGML_HEXAGON_AR_SELECT"); const char * str_ndev = getenv("GGML_HEXAGON_NDEV"); const char * str_arch = getenv("GGML_HEXAGON_ARCH"); const char * str_vmem = getenv("GGML_HEXAGON_VMEM"); const char * str_mbuf = getenv("GGML_HEXAGON_MBUF"); const char * str_optrace = getenv("GGML_HEXAGON_OPTRACE"); + const char * str_hostbuf = getenv("GGML_HEXAGON_HOSTBUF"); // Init Arch first since it affects other defaults if (!str_arch) { @@ -4430,8 +5876,6 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_opfilter = str_opfilter ? new std::regex(str_opfilter, RE_ICASE) : NULL; opt_verbose = str_verbose ? atoi(str_verbose) : 0; - opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) : opt_hostbuf; - opt_opstage = str_opstage ? strtoul(str_opstage, NULL, 0) : opt_opstage; opt_opbatch = str_opbatch ? strtoul(str_opbatch, NULL, 0) : opt_opbatch; opt_opqueue = str_opqueue ? strtoul(str_opqueue, NULL, 0) : opt_opqueue; opt_optrace = str_optrace ? strtoul(str_optrace, NULL, 0) : (opt_opbatch * 256); @@ -4440,16 +5884,90 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_profile = str_profile ? atoi(str_profile) : 0; opt_etm = str_etm ? atoi(str_etm) : 0; opt_nhvx = str_nhvx ? strtoul(str_nhvx, NULL, 0) : opt_nhvx; - opt_nhmx = str_nhmx ? atoi(str_nhmx) : (str_use_hmx ? atoi(str_use_hmx) : opt_nhmx); + opt_nhmx = str_nhmx ? atoi(str_nhmx) : opt_nhmx; opt_mm_select = str_mm_select ? atoi(str_mm_select) : opt_mm_select; opt_fa_select = str_fa_select ? atoi(str_fa_select) : opt_fa_select; - opt_ndev = str_ndev ? strtoul(str_ndev, NULL, 0) : opt_ndev; - opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) : opt_hostbuf; + opt_ar_select = str_ar_select ? atoi(str_ar_select) : opt_ar_select; opt_mbuf = str_mbuf ? strtoul(str_mbuf, NULL, 0) * MiB : opt_mbuf; opt_vmem = str_vmem ? strtoul(str_vmem, NULL, 0) * MiB : opt_vmem; + opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) != 0 : opt_hostbuf; - if (opt_ndev > GGML_HEXAGON_MAX_SESSIONS) { - opt_ndev = GGML_HEXAGON_MAX_SESSIONS; + // Parse device configuration + const char * str_devices = getenv("GGML_HEXAGON_DEVICES"); + if (!str_devices && str_ndev && str_ndev[0] != '\0') { + GGML_LOG_WARN("DEPRECATED: GGML_HEXAGON_NDEV is deprecated. use GGML_HEXAGON_DEVICES instead\n"); + str_devices = str_ndev; + } + + if (str_devices && str_devices[0] != '\0') { + bool is_single_number = true; + for (int i = 0; str_devices[i] != '\0'; i++) { + if (!isdigit((unsigned char)str_devices[i])) { + is_single_number = false; + break; + } + } + if (is_single_number) { + int n = atoi(str_devices); + if (n < 1) n = 1; + if (n > GGML_HEXAGON_MAX_SESSIONS) n = GGML_HEXAGON_MAX_SESSIONS; + opt_ndev = n; + for (size_t i = 0; i < opt_ndev; i++) { + opt_device_configs[i].physical_idx = 0; + opt_device_configs[i].virtual_idx = (int)i; + opt_device_configs[i].name = "HTP" + std::to_string(i); + } + } else { + std::string s_devices(str_devices); + std::stringstream ss(s_devices); + std::string item; + opt_ndev = 0; + while (std::getline(ss, item, ',')) { + size_t start = item.find_first_not_of(" \t\r\n"); + size_t end = item.find_last_not_of(" \t\r\n"); + if (start == std::string::npos) { + continue; + } + item = item.substr(start, end - start + 1); + + if (item.rfind("HTP", 0) == 0) { + std::string rest = item.substr(3); + size_t colon_pos = rest.find(':'); + int phys = 0; + int virt = 0; + try { + if (colon_pos == std::string::npos) { + phys = std::stoi(rest); + virt = 0; + } else { + phys = std::stoi(rest.substr(0, colon_pos)); + virt = std::stoi(rest.substr(colon_pos + 1)); + } + } catch (...) { + GGML_LOG_WARN("ggml-hex: failed to parse device index in '%s'\n", item.c_str()); + continue; + } + + if (opt_ndev < GGML_HEXAGON_MAX_SESSIONS) { + opt_device_configs[opt_ndev].physical_idx = phys; + opt_device_configs[opt_ndev].virtual_idx = virt; + opt_device_configs[opt_ndev].name = colon_pos == std::string::npos + ? "HTP" + std::to_string(phys) + : "HTP" + std::to_string(phys) + ":" + std::to_string(virt); + opt_ndev++; + } else { + GGML_LOG_WARN("ggml-hex: max sessions limit reached (%d), ignoring device %s\n", GGML_HEXAGON_MAX_SESSIONS, item.c_str()); + } + } else { + GGML_LOG_WARN("ggml-hex: invalid device name format '%s', must start with HTP\n", item.c_str()); + } + } + } + } else { + opt_ndev = 1; + opt_device_configs[0].physical_idx = 0; + opt_device_configs[0].virtual_idx = 0; + opt_device_configs[0].name = "HTP0"; } #if defined(__ANDROID__) diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.h index b0c859dac..741b5e04e 100644 --- a/ggml/src/ggml-hexagon/htp-opnode.h +++ b/ggml/src/ggml-hexagon/htp-opnode.h @@ -8,60 +8,107 @@ #include #include #include +#include #include #include "htp-ops.h" #include "htp/matmul-ops.h" #include "htp/flash-attn-ops.h" #include "htp/unary-ops.h" +#include "htp/allreduce-ops.h" struct htp_opnode { - ggml_tensor * node = nullptr; + ggml_tensor * node { nullptr }; + htp_op_code opcode { HTP_OP_INVALID }; + int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS] {0}; - std::vector fused; + std::vector fused; + std::vector> dummy; - htp_op_code opcode = HTP_OP_INVALID; + std::vector inputs; + std::vector outputs; + std::string name; - std::vector extra_dsts; - - int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS] = {0}; - - htp_opnode(ggml_tensor * node = nullptr, std::vector fused = {}, htp_op_code opcode = HTP_OP_INVALID, std::vector extra_dsts = {}) - : node(node), fused(std::move(fused)), opcode(opcode), extra_dsts(std::move(extra_dsts)) {} - - ggml_op op() const { - return node->op; + int n_active_src(const ggml_tensor * t) const { + if (!t) return 0; + for (int i = GGML_MAX_SRC - 1; i >= 0; i--) { + if (t->src[i]) { + return i + 1; + } + } + return 0; } - const ggml_tensor * dst() const { - return fused.empty() ? node : fused.back(); + void init(ggml_tensor * node) { + this->node = node; + if (this->node) { + this->name = ggml_op_desc(this->node); + + // Build inputs (preserving optional nullptrs) + int n_inputs = n_active_src(this->node); + this->inputs.resize(n_inputs, nullptr); + for (int i = 0; i < n_inputs; i++) { + this->inputs[i] = this->node->src[i]; + } + + // Build outputs + this->outputs.push_back(this->dst()); + } + } + + htp_opnode(htp_op_code opcode = HTP_OP_INVALID, ggml_tensor * node = nullptr) : opcode(opcode) { + init(node); + } + + ggml_op op() const { return node->op; } + const ggml_tensor * src0() const { return node->src[0]; } + const ggml_tensor * src1() const { return node->src[1]; } + const ggml_tensor * dst() const { return outputs.empty() ? node : outputs.back(); } + + ggml_tensor * add_dummy(const ggml_tensor & t) { + dummy.push_back(std::make_shared(t)); + return dummy.back().get(); } void add_fused(ggml_tensor * t, bool extra_dst = false) { fused.push_back(t); - if (extra_dst) { - extra_dsts.push_back(t); - } - } - std::vector get_outputs() const { - std::vector res; - if (extra_dsts.empty()) { - res.push_back(dst()); + name += "+"; + name += ggml_op_desc(t); + + if (extra_dst) { + outputs.push_back(t); } else { - res.push_back(node); - for (const auto * x : extra_dsts) { - res.push_back(x); + outputs.clear(); + outputs.push_back(t); + } + + // Remove the newly fused intermediate output tensor t from inputs (if it was there) + inputs.erase(std::remove(inputs.begin(), inputs.end(), t), inputs.end()); + + // Append new inputs from t, preserving middle nullptrs + int n_inputs = n_active_src(t); + for (int i = 0; i < n_inputs; i++) { + const auto * src = t->src[i]; + if (!src) { + inputs.push_back(nullptr); + } else if (src != node && + std::find(fused.begin(), fused.end(), src) == fused.end() && + std::find(inputs.begin(), inputs.end(), src) == inputs.end()) { + inputs.push_back(src); } } - return res; } - const ggml_tensor * src0() const { - return node->src[0]; + const std::vector & get_inputs() const { + return inputs; } - const ggml_tensor * src1() const { - return node->src[1]; + const std::vector & get_outputs() const { + return outputs; + } + + std::string op_name() const { + return name; } bool is_empty() const { @@ -81,75 +128,6 @@ struct htp_opnode { bool same_input(const htp_opnode& n) const { return n.src1() == this->src1(); } - - std::vector get_inputs() const { - if (fused.empty()) { - int last_non_null = -1; - for (int i = 0; i < GGML_MAX_SRC; i++) { - if (node->src[i]) { - last_non_null = i; - } - } - std::vector inputs(last_non_null + 1, nullptr); - for (int i = 0; i <= last_non_null; i++) { - inputs[i] = node->src[i]; - } - return inputs; - } - - std::vector inputs(GGML_MAX_SRC, nullptr); - std::vector outputs; - outputs.push_back(node); - for (const auto * f : fused) { - outputs.push_back(f); - } - - auto contains = [&](const std::vector & vec, const ggml_tensor * t) { - for (const auto * x : vec) { - if (x == t) return true; - } - return false; - }; - - int count = 0; - auto add_input = [&](const ggml_tensor * t) { - if (t && !contains(outputs, t) && !contains(inputs, t)) { - if (count < (int)inputs.size()) { - inputs[count++] = t; - } else { - inputs.push_back(t); - } - } - }; - - for (int i = 0; i < GGML_MAX_SRC; i++) { - if (node->src[i]) { - add_input(node->src[i]); - } - } - for (const auto * f : fused) { - for (int i = 0; i < GGML_MAX_SRC; i++) { - if (f->src[i]) { - add_input(f->src[i]); - } - } - } - - inputs.resize(count); - return inputs; - } - - std::string op_name() const { - if (fused.empty()) { - return ggml_op_desc(node); - } - std::string name = ggml_op_desc(node); - for (const auto * f : fused) { - name += "+"; - name += ggml_op_desc(f); - } - return name; - } }; struct htp_opformat { @@ -337,8 +315,7 @@ struct htp_opformat { } void format_kernel_params(char * str, size_t max_size, const htp_opnode & node) { if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID || - node.opcode == HTP_OP_MUL_MAT_QKV || node.opcode == HTP_OP_MUL_MAT_FFN || - node.opcode == HTP_OP_MUL_MAT_ADD) { + node.opcode == HTP_OP_MUL_MAT_NX || node.opcode == HTP_OP_MUL_MAT_ADD) { const auto * kparams = (const struct htp_mm_kernel_params *) node.kernel_params; const char * path = "unknown"; int32_t type = kparams->kernel_type; diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt index b00aa2bc9..77f3ee39d 100644 --- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt @@ -43,6 +43,7 @@ add_library(${HTP_LIB} SHARED pad-ops.c argsort-ops.c im2col-ops.c + allreduce-ops.c ) target_compile_definitions(${HTP_LIB} PRIVATE diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.c index 9973c088d..0a8bf84e3 100644 --- a/ggml/src/ggml-hexagon/htp/act-ops.c +++ b/ggml/src/ggml-hexagon/htp/act-ops.c @@ -183,6 +183,53 @@ static void swiglu_oai_f32(const float * restrict src0, static const float GELU_COEF_A = 0.044715f; static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f; +static inline HVX_Vector hvx_vec_fast_sigmoid_f32_2it(HVX_Vector v) { + v = Q6_Vqf32_vmpy_VsfVsf(v, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); + v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), Q6_V_vsplat_R(FAST_SIGMOID_C3)); + + HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v)); + HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int)); + HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x); + + HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2)); + v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); + + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1)); + v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx); + v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x); + + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1)); + v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24); + + HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1)); + HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4)); + + // Newton-Raphson with 2 iterations + HVX_Vector two_sf = hvx_vec_splat_f32(2.0f); + HVX_Vector i_sf = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5); + HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf( + i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5))))); + r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32( + r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5)))); + HVX_Vector res = Q6_Vsf_equals_Vqf32(r_qf); + + res = Q6_Vqf32_vmpy_VsfVsf(v3, res); + + return Q6_Vsf_equals_Vqf32(res); +} + +static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard_2it(HVX_Vector v, + HVX_Vector one, + HVX_Vector max_exp, + HVX_Vector min_exp) { + const HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(max_exp, v); + const HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(v, min_exp); + + HVX_Vector out = hvx_vec_fast_sigmoid_f32_2it(v); + out = Q6_V_vmux_QVV(pred_max, out, one); + return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero()); +} + static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { assert((unsigned long) dst % 128 == 0); assert((unsigned long) src0 % 128 == 0); @@ -200,20 +247,13 @@ static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * rest const HVX_Vector v_coef_a_times_sqrt = hvx_vec_splat_f32(GELU_COEF_A_TIMES_SQRT); const HVX_Vector v_sqrt_2_pi = hvx_vec_splat_f32(SQRT_2_OVER_PI); - const HVX_Vector v_half = hvx_vec_splat_f32(0.5f); const HVX_Vector v_one = hvx_vec_splat_f32(1.0f); - const HVX_Vector v_two = hvx_vec_splat_f32(2.0f); - - // Hoisted fast sigmoid / inverse constants to avoid loop-internal overhead - const HVX_Vector v_log2f = Q6_V_vsplat_R(FAST_SIGMOID_LOG2F); - const HVX_Vector v_c1 = Q6_V_vsplat_R(FAST_SIGMOID_C1); - const HVX_Vector v_c2 = Q6_V_vsplat_R(FAST_SIGMOID_C2); - const HVX_Vector v_inv_aprox = Q6_V_vsplat_R(0x7EEEEBB3); const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f); const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f); uint32_t i = 0; + _Pragma("unroll(4)") for (; i < nvec; i++) { HVX_Vector x = vsrc0[i]; HVX_Vector g = vsrc1[i]; @@ -223,56 +263,13 @@ static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * rest coef = hvx_vec_add_f32_f32(coef, v_sqrt_2_pi); HVX_Vector inner = hvx_vec_mul_f32_f32(x, coef); - // y2 = 2 * inner - HVX_Vector y2 = hvx_vec_mul_f32_f32(inner, v_two); + // y2 = 2 * inner = inner + inner + HVX_Vector y2 = hvx_vec_add_f32_f32(inner, inner); - // Sigmoid guard check predicates - HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(v_max_exp, y2); - HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(y2, v_min_exp); - - // Fast sigmoid approximation - HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(y2, v_log2f); - v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), v_half); - - HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v)); - HVX_Vector x_sig = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int)); - HVX_Vector xx_sig = Q6_Vqf32_vmpy_Vqf32Vqf32(x_sig, x_sig); - - HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx_sig), v_c2); - v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, v_log2f); - - HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x_sig), v_c1); - v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx_sig); - v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x_sig); - - HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1)); - v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24); - - HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1)); - HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4)); - - // Fast division (Newton-Raphson with 2 iterations) - HVX_Vector i_sf = Q6_Vw_vsub_VwVw(v_inv_aprox, v5); - HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf( - i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5))))); - r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32( - r_qf, Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5)))); - HVX_Vector res_inv = Q6_Vsf_equals_Vqf32(r_qf); - - HVX_Vector sig2y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, res_inv)); - - // Sigmoid guards - sig2y = Q6_V_vmux_QVV(pred_max, sig2y, v_one); - sig2y = Q6_V_vmux_QVV(pred_min, sig2y, Q6_V_vzero()); - - // tanh(inner) = 2 * sigmoid(2 * inner) - 1 - HVX_Vector tanh_val = hvx_vec_mul_f32_f32(sig2y, v_two); - tanh_val = hvx_vec_sub_f32_f32(tanh_val, v_one); - - HVX_Vector tanh_plus_one = hvx_vec_add_f32_f32(tanh_val, v_one); - HVX_Vector half_x = hvx_vec_mul_f32_f32(x, v_half); - HVX_Vector gelu_x = hvx_vec_mul_f32_f32(half_x, tanh_plus_one); + // Fast sigmoid approximation (2 iterations) + HVX_Vector sig2y = hvx_vec_fast_sigmoid_f32_guard_2it(y2, v_one, v_max_exp, v_min_exp); + HVX_Vector gelu_x = hvx_vec_mul_f32_f32(x, sig2y); vdst[i] = hvx_vec_mul_f32_f32(gelu_x, g); } @@ -285,50 +282,11 @@ static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * rest coef = hvx_vec_add_f32_f32(coef, v_sqrt_2_pi); HVX_Vector inner = hvx_vec_mul_f32_f32(x, coef); - HVX_Vector y2 = hvx_vec_mul_f32_f32(inner, v_two); + HVX_Vector y2 = hvx_vec_add_f32_f32(inner, inner); - HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(v_max_exp, y2); - HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(y2, v_min_exp); - - HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(y2, v_log2f); - v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), v_half); - - HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v)); - HVX_Vector x_sig = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int)); - HVX_Vector xx_sig = Q6_Vqf32_vmpy_Vqf32Vqf32(x_sig, x_sig); - - HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx_sig), v_c2); - v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, v_log2f); - - HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x_sig), v_c1); - v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx_sig); - v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x_sig); - - HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1)); - v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24); - - HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1)); - HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4)); - - HVX_Vector i_sf = Q6_Vw_vsub_VwVw(v_inv_aprox, v5); - HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf( - i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5))))); - r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32( - r_qf, Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5)))); - HVX_Vector res_inv = Q6_Vsf_equals_Vqf32(r_qf); - - HVX_Vector sig2y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, res_inv)); - - sig2y = Q6_V_vmux_QVV(pred_max, sig2y, v_one); - sig2y = Q6_V_vmux_QVV(pred_min, sig2y, Q6_V_vzero()); - - HVX_Vector tanh_val = hvx_vec_mul_f32_f32(sig2y, v_two); - tanh_val = hvx_vec_sub_f32_f32(tanh_val, v_one); - - HVX_Vector tanh_plus_one = hvx_vec_add_f32_f32(tanh_val, v_one); - HVX_Vector half_x = hvx_vec_mul_f32_f32(x, v_half); - HVX_Vector gelu_x = hvx_vec_mul_f32_f32(half_x, tanh_plus_one); + HVX_Vector sig2y = hvx_vec_fast_sigmoid_f32_guard_2it(y2, v_one, v_max_exp, v_min_exp); + HVX_Vector gelu_x = hvx_vec_mul_f32_f32(x, sig2y); HVX_Vector res = hvx_vec_mul_f32_f32(gelu_x, g); hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(float), res); } diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.c b/ggml/src/ggml-hexagon/htp/allreduce-ops.c new file mode 100644 index 000000000..d35f685a6 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.c @@ -0,0 +1,398 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include +#include +#include +#include +#include + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "hvx-utils.h" +#include "htp-tensor.h" +#include "hex-dma.h" +#include "hex-profile.h" +#include "allreduce-ops.h" + +struct htp_allreduce_context { + struct htp_ops_context * octx; + uint32_t n_ranks; + uint32_t n_dsts; + uint32_t nelem; + uint32_t ne0; + uint32_t ne1; + uint32_t row_size_aligned; + uint32_t rank_elem_start; + uint32_t rank_nelem; + uint32_t elems_per_thread; + uint32_t block_elems; + uint32_t vtcm_size_per_thread; + bool is_row_bcast; + uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS]; + uint8_t * dst_spad_base; + uint8_t * res_spad_base; +}; + +#define DEFINE_ALLREDUCE_THREAD_DMA_1D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD) \ +static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \ + struct htp_ops_context * octx = actx->octx; \ + \ + const uint32_t n_ranks = actx->n_ranks; \ + const uint32_t n_dsts = actx->n_dsts; \ + const uint32_t block_elems = actx->block_elems; \ + \ + const uint32_t dr = actx->elems_per_thread; \ + const uint32_t ir0 = actx->rank_elem_start + dr * ith; \ + const uint32_t ir1 = MIN(ir0 + dr, actx->rank_elem_start + actx->rank_nelem); \ + if (ir0 >= ir1) return; \ + \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + dma_queue * q = octx->ctx->dma[ith]; \ + \ + uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS]; \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + src_spad_base[s] = actx->src_spad_base[s] + (ith * actx->vtcm_size_per_thread); \ + } \ + uint8_t * dst_spad_base = actx->dst_spad_base + (ith * actx->vtcm_size_per_thread); \ + uint8_t * res_spad_base = HAS_ADD ? (actx->res_spad_base + (ith * actx->vtcm_size_per_thread)) : NULL; \ + \ + const size_t spad_half = actx->vtcm_size_per_thread / 2; \ + uint32_t ir_prefetch = ir0; \ + int spad_idx = 0; \ + \ + for (int k = 0; k < 2 && ir_prefetch < ir1; k++) { \ + uint32_t cur_elems = MIN(block_elems, ir1 - ir_prefetch); \ + size_t cur_bytes = cur_elems * sizeof(TYPE); \ + uint8_t * d_spad = dst_spad_base + spad_idx * spad_half; \ + for (uint32_t d = 0; d < n_dsts; d++) { \ + uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + ir_prefetch * sizeof(TYPE); \ + dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 0); \ + } \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + uint8_t * s_spad = src_spad_base[s] + spad_idx * spad_half; \ + const uint8_t * s_ddr = (const uint8_t *) octx->src[s]->data + ir_prefetch * sizeof(TYPE); \ + dma_queue_push(q, dma_make_ptr(s_spad, s_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \ + } \ + if (HAS_ADD) { \ + uint8_t * r_spad = res_spad_base + spad_idx * spad_half; \ + const uint8_t * r_ddr = (const uint8_t *) octx->src[2 * n_ranks]->data + ir_prefetch * sizeof(TYPE); \ + dma_queue_push(q, dma_make_ptr(r_spad, r_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \ + } \ + ir_prefetch += cur_elems; \ + spad_idx ^= 1; \ + } \ + \ + for (uint32_t ir = ir0; ir < ir1; ) { \ + uint32_t cur_elems = MIN(block_elems, ir1 - ir); \ + size_t cur_bytes = cur_elems * sizeof(TYPE); \ + uint8_t * d_spad = NULL; \ + for (uint32_t d = 0; d < n_dsts; d++) { \ + d_spad = (uint8_t *) dma_queue_pop(q).src; \ + } \ + uint8_t * s_spad[HTP_ALLREDUCE_MAX_RANKS]; \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + s_spad[s] = (uint8_t *) dma_queue_pop(q).dst; \ + } \ + uint8_t * r_spad = HAS_ADD ? (uint8_t *) dma_queue_pop(q).dst : NULL; \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \ + HVX_ADD_FN(d_spad, s_spad[0], s_spad[1], cur_elems); \ + for (uint32_t s = 2; s < n_ranks; s++) { \ + HVX_ADD_FN(d_spad, d_spad, s_spad[s], cur_elems); \ + } \ + if (HAS_ADD) { \ + HVX_ADD_FN(d_spad, d_spad, r_spad, cur_elems); \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \ + for (uint32_t d = 0; d < n_dsts; d++) { \ + uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + ir * sizeof(TYPE); \ + dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 1); \ + } \ + if (ir_prefetch < ir1) { \ + uint32_t next_elems = MIN(block_elems, ir1 - ir_prefetch); \ + size_t next_bytes = next_elems * sizeof(TYPE); \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + const uint8_t * s_next = (const uint8_t *) octx->src[s]->data + ir_prefetch * sizeof(TYPE); \ + dma_queue_push(q, dma_make_ptr(s_spad[s], s_next), next_bytes, next_bytes, next_bytes, 1); \ + } \ + if (HAS_ADD) { \ + const uint8_t * r_next = (const uint8_t *) octx->src[2 * n_ranks]->data + ir_prefetch * sizeof(TYPE); \ + dma_queue_push(q, dma_make_ptr(r_spad, r_next), next_bytes, next_bytes, next_bytes, 1); \ + } \ + ir_prefetch += next_elems; \ + } \ + ir += cur_elems; \ + } \ + dma_queue_flush(q); \ +} + +DEFINE_ALLREDUCE_THREAD_DMA_1D(f16, __fp16, hvx_add_f16_aaa, 0) +DEFINE_ALLREDUCE_THREAD_DMA_1D(f32, float, hvx_add_f32_aaa, 0) +DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f16, __fp16, hvx_add_f16_aaa, 1) +DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f32, float, hvx_add_f32_aaa, 1) + +#define DEFINE_ALLREDUCE_THREAD_DMA_2D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD, IS_ROW_BCAST) \ +static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \ + struct htp_ops_context * octx = actx->octx; \ + \ + const uint32_t n_ranks = actx->n_ranks; \ + const uint32_t n_dsts = actx->n_dsts; \ + const uint32_t ne0 = actx->ne0; \ + const uint32_t block_rows = actx->block_elems; \ + const uint32_t row_size_aligned = actx->row_size_aligned; \ + const uint32_t row_bytes = ne0 * sizeof(TYPE); \ + \ + const uint32_t dr = actx->elems_per_thread; \ + const uint32_t r0 = actx->rank_elem_start + dr * ith; \ + const uint32_t r1 = MIN(r0 + dr, actx->rank_elem_start + actx->rank_nelem); \ + if (r0 >= r1) return; \ + \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + dma_queue * q = octx->ctx->dma[ith]; \ + \ + uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS]; \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + src_spad_base[s] = actx->src_spad_base[s] + (ith * actx->vtcm_size_per_thread); \ + } \ + uint8_t * dst_spad_base = actx->dst_spad_base + (ith * actx->vtcm_size_per_thread); \ + uint8_t * res_spad_base = HAS_ADD ? (IS_ROW_BCAST ? actx->res_spad_base : (actx->res_spad_base + (ith * actx->vtcm_size_per_thread))) : NULL; \ + \ + const size_t spad_half = actx->vtcm_size_per_thread / 2; \ + uint32_t r_prefetch = r0; \ + int spad_idx = 0; \ + \ + for (int k = 0; k < 2 && r_prefetch < r1; k++) { \ + uint32_t cur_rows = MIN(block_rows, r1 - r_prefetch); \ + uint8_t * d_spad = dst_spad_base + spad_idx * spad_half; \ + for (uint32_t d = 0; d < n_dsts; d++) { \ + uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + r_prefetch * octx->dsts[d]->nb[1]; \ + dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, 0); \ + } \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + uint8_t * s_spad = src_spad_base[s] + spad_idx * spad_half; \ + const uint8_t * s_ddr = (const uint8_t *) octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \ + dma_queue_push(q, dma_make_ptr(s_spad, s_ddr), row_size_aligned, octx->src[s]->nb[1], row_bytes, cur_rows); \ + } \ + if (HAS_ADD && !IS_ROW_BCAST) { \ + uint8_t * r_spad = res_spad_base + spad_idx * spad_half; \ + const uint8_t * r_ddr = (const uint8_t *) octx->src[2 * n_ranks]->data + r_prefetch * octx->src[2 * n_ranks]->nb[1]; \ + dma_queue_push(q, dma_make_ptr(r_spad, r_ddr), row_size_aligned, octx->src[2 * n_ranks]->nb[1], row_bytes, cur_rows); \ + } \ + r_prefetch += cur_rows; \ + spad_idx ^= 1; \ + } \ + \ + for (uint32_t r = r0; r < r1; ) { \ + uint32_t cur_rows = MIN(block_rows, r1 - r); \ + uint8_t * d_spad = NULL; \ + for (uint32_t d = 0; d < n_dsts; d++) { \ + d_spad = (uint8_t *) dma_queue_pop(q).src; \ + } \ + uint8_t * s_spad[HTP_ALLREDUCE_MAX_RANKS]; \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + s_spad[s] = (uint8_t *) dma_queue_pop(q).dst; \ + } \ + uint8_t * r_spad = (HAS_ADD && !IS_ROW_BCAST) ? (uint8_t *) dma_queue_pop(q).dst : NULL; \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \ + for (uint32_t row = 0; row < cur_rows; row++) { \ + uint8_t * d_row = d_spad + row * row_size_aligned; \ + const uint8_t * s0_row = s_spad[0] + row * row_size_aligned; \ + const uint8_t * s1_row = s_spad[1] + row * row_size_aligned; \ + HVX_ADD_FN(d_row, s0_row, s1_row, ne0); \ + for (uint32_t s = 2; s < n_ranks; s++) { \ + const uint8_t * ss_row = s_spad[s] + row * row_size_aligned; \ + HVX_ADD_FN(d_row, d_row, ss_row, ne0); \ + } \ + if (HAS_ADD) { \ + const uint8_t * res_row = IS_ROW_BCAST ? res_spad_base : (r_spad + row * row_size_aligned); \ + HVX_ADD_FN(d_row, d_row, res_row, ne0); \ + } \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \ + for (uint32_t d = 0; d < n_dsts; d++) { \ + uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + r * octx->dsts[d]->nb[1]; \ + dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, cur_rows); \ + } \ + if (r_prefetch < r1) { \ + uint32_t next_rows = MIN(block_rows, r1 - r_prefetch); \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + const uint8_t * s_next = (const uint8_t *) octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \ + dma_queue_push(q, dma_make_ptr(s_spad[s], s_next), row_size_aligned, octx->src[s]->nb[1], row_bytes, next_rows); \ + } \ + if (HAS_ADD && !IS_ROW_BCAST) { \ + const uint8_t * r_next = (const uint8_t *) octx->src[2 * n_ranks]->data + r_prefetch * octx->src[2 * n_ranks]->nb[1]; \ + dma_queue_push(q, dma_make_ptr(r_spad, r_next), row_size_aligned, octx->src[2 * n_ranks]->nb[1], row_bytes, next_rows); \ + } \ + r_prefetch += next_rows; \ + } \ + r += cur_rows; \ + } \ + dma_queue_flush(q); \ +} + +DEFINE_ALLREDUCE_THREAD_DMA_2D(f16, __fp16, hvx_add_f16_aaa, 0, 0) +DEFINE_ALLREDUCE_THREAD_DMA_2D(f32, float, hvx_add_f32_aaa, 0, 0) +DEFINE_ALLREDUCE_THREAD_DMA_2D(add_f16, __fp16, hvx_add_f16_aaa, 1, 0) +DEFINE_ALLREDUCE_THREAD_DMA_2D(add_f32, float, hvx_add_f32_aaa, 1, 0) +DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f16, __fp16, hvx_add_f16_aaa, 1, 1) +DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f32, float, hvx_add_f32_aaa, 1, 1) + +int op_allreduce(struct htp_ops_context * octx) { + const struct htp_allreduce_kernel_params * kparams = (const struct htp_allreduce_kernel_params *) octx->kernel_params; + const struct htp_tensor * dst = octx->dst; + + const uint32_t rank = (uint32_t) kparams->rank; + const uint32_t n_ranks = (uint32_t) kparams->n_ranks; + + if (n_ranks < 2 || n_ranks > HTP_ALLREDUCE_MAX_RANKS || rank >= n_ranks) { + return HTP_STATUS_INVAL_PARAMS; + } + + if (dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_F32) { + return HTP_STATUS_NO_SUPPORT; + } + + const uint32_t nelem = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; + const uint32_t fence_seq_entry = (uint32_t) octx->op_params[0]; + const uint32_t fence_seq_exit = (uint32_t) octx->op_params[1]; + + // 1. Entry Barrier: Synchronize all ranks before reading + struct htp_thread_trace * tr0 = &octx->ctx->trace[0]; + htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry); + + const struct htp_tensor * my_sync = octx->src[n_ranks + rank]; + atomic_uint * my_fence = (atomic_uint *) my_sync->data; + + atomic_store(&my_fence[0], fence_seq_entry); + asm volatile ("syncht" : : : "memory"); + Q6_dccleaninva_A((void *) my_fence); + + for (uint32_t j = 0; j < n_ranks; j++) { + if (j == rank) continue; + const struct htp_tensor * peer_sync = octx->src[n_ranks + j]; + atomic_uint * peer_fence = (atomic_uint *) peer_sync->data; + uint64_t spins = 0; + while (1) { + Q6_dccleaninva_A((void *) peer_fence); + uint32_t val = atomic_load(&peer_fence[0]); + if (val == fence_seq_entry || val == fence_seq_exit) { + break; + } + if (++spins > HTP_FENCE_TIMEOUT) { + FARF(ERROR, "ggml-hex: allreduce entry fence-wait TIMEOUT: rank %u waiting on %u (fence %p seq %u)\n", rank, j, peer_fence, fence_seq_entry); + return HTP_STATUS_INTERNAL_ERR; + } + hex_pause(); + } + } + asm volatile ("syncht" : : : "memory"); + + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry); + + // 2. Multi-threaded Reduction across assigned rank chunk + if (nelem > 0) { + const uint32_t n_threads = (uint32_t) kparams->n_threads; + const uint32_t block_elems = (uint32_t) kparams->block_elems; + const uint32_t elems_per_thread = (uint32_t) kparams->elems_per_thread; + const uint32_t vtcm_size_per_thread = (uint32_t) kparams->vtcm_size_per_thread; + + const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD); + + struct htp_allreduce_context actx; + actx.octx = octx; + actx.n_ranks = n_ranks; + actx.n_dsts = (uint32_t) kparams->n_dsts ? (uint32_t) kparams->n_dsts : n_ranks; + actx.nelem = nelem; + actx.ne0 = (uint32_t) kparams->ne0; + actx.ne1 = (uint32_t) kparams->ne1; + actx.row_size_aligned = (uint32_t) kparams->row_size_aligned; + actx.rank_elem_start = (uint32_t) kparams->rank_elem_start; + actx.rank_nelem = (uint32_t) kparams->rank_nelem; + actx.elems_per_thread = elems_per_thread; + actx.block_elems = block_elems; + actx.vtcm_size_per_thread = vtcm_size_per_thread; + actx.is_row_bcast = (kparams->is_row_bcast != 0); + + work_queue_func_t reduce_fun = NULL; + switch (kparams->kernel_type) { + case HTP_ALLREDUCE_KERNEL_DMA_1D: + if (has_add) { + reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_1d_add_f16 : allreduce_thread_dma_1d_add_f32; + } else { + reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_1d_f16 : allreduce_thread_dma_1d_f32; + } + break; + case HTP_ALLREDUCE_KERNEL_DMA_2D: + if (has_add) { + if (kparams->is_row_bcast) { + reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_2d_add_bcast_f16 : allreduce_thread_dma_2d_add_bcast_f32; + } else { + reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_2d_add_f16 : allreduce_thread_dma_2d_add_f32; + } + } else { + reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_2d_f16 : allreduce_thread_dma_2d_f32; + } + break; + default: + return HTP_STATUS_NO_SUPPORT; + } + + uint8_t * vtcm_ptr = (uint8_t *) octx->ctx->vtcm_base; + for (uint32_t s = 0; s < n_ranks; s++) { + actx.src_spad_base[s] = vtcm_ptr; + vtcm_ptr += n_threads * vtcm_size_per_thread; + } + actx.dst_spad_base = vtcm_ptr; + vtcm_ptr += n_threads * vtcm_size_per_thread; + if (has_add) { + actx.res_spad_base = vtcm_ptr; + vtcm_ptr += (actx.is_row_bcast ? 1 : n_threads) * vtcm_size_per_thread; + } + + if (has_add && actx.is_row_bcast) { + const uint8_t * r_ddr = (const uint8_t *) octx->src[2 * n_ranks]->data; + const uint32_t row_bytes = actx.ne0 * (dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float)); + dma_queue * q = octx->ctx->dma[0]; + dma_queue_push(q, dma_make_ptr(actx.res_spad_base, r_ddr), actx.row_size_aligned, 0, row_bytes, 1); + dma_queue_pop(q); + } + + work_queue_run(octx->ctx->work_queue, reduce_fun, &actx, n_threads); + } + + // 4. Exit Barrier: Synchronize all ranks after writing + htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); + + atomic_store(&my_fence[0], fence_seq_exit); + asm volatile ("syncht" : : : "memory"); + Q6_dccleaninva_A((void *) my_fence); + + for (uint32_t j = 0; j < n_ranks; j++) { + if (j == rank) continue; + const struct htp_tensor * peer_sync = octx->src[n_ranks + j]; + atomic_uint * peer_fence = (atomic_uint *) peer_sync->data; + uint64_t spins = 0; + while (1) { + Q6_dccleaninva_A((void *) peer_fence); + uint32_t val = atomic_load(&peer_fence[0]); + if (val == fence_seq_exit) { + break; + } + if (++spins > HTP_FENCE_TIMEOUT) { + FARF(ERROR, "ggml-hex: allreduce exit fence-wait TIMEOUT: rank %u waiting on %u (fence %p seq %u)\n", rank, j, peer_fence, fence_seq_exit); + return HTP_STATUS_INTERNAL_ERR; + } + hex_pause(); + } + } + asm volatile ("syncht" : : : "memory"); + + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); + + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.h b/ggml/src/ggml-hexagon/htp/allreduce-ops.h new file mode 100644 index 000000000..de447d87e --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.h @@ -0,0 +1,40 @@ +#ifndef ALLREDUCE_OPS_H +#define ALLREDUCE_OPS_H + +#include + +#define HTP_ALLREDUCE_MAX_RANKS 4 + +#ifdef __cplusplus +extern "C" { +#endif + +enum htp_allreduce_kernel_type { + HTP_ALLREDUCE_KERNEL_UNSUPPORTED = 0, + HTP_ALLREDUCE_KERNEL_DMA_1D, + HTP_ALLREDUCE_KERNEL_DMA_2D, +}; + +struct htp_allreduce_kernel_params { + int32_t rank; + int32_t n_ranks; + int32_t n_threads; + int32_t block_elems; // 1D: block_elems, 2D: block_rows + int32_t elems_per_thread; // 1D: nelem_per_thread, 2D: nrows_per_thread + int32_t vtcm_size_per_thread; + int32_t vtcm_size; + int32_t kernel_type; + int32_t ne0; + int32_t ne1; + int32_t row_size_aligned; + int32_t rank_elem_start; + int32_t rank_nelem; + int32_t n_dsts; + int32_t is_row_bcast; +}; + +#ifdef __cplusplus +} +#endif + +#endif /* ALLREDUCE_OPS_H */ diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c index ae507effa..15bc8dc24 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.c +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -14,6 +15,7 @@ #include "htp-ops.h" #include "htp-ops.h" #include "hvx-utils.h" +#include "htp-tensor.h" struct htp_copy_context { struct htp_ops_context * octx; @@ -78,7 +80,7 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo } \ } -DEFINE_CPY_SAMESHAPE(f32, float, 4) +DEFINE_CPY_SAMESHAPE(f32, float, 4) DEFINE_CPY_SAMESHAPE(f16, __fp16, 2) #define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ @@ -179,7 +181,7 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void } \ } -DEFINE_CPY_RESHAPE(f32, float, 4) +DEFINE_CPY_RESHAPE(f32, float, 4) DEFINE_CPY_RESHAPE(f16, __fp16, 2) static void cpy_thread_f16_f32_sameshape(unsigned int nth, unsigned int ith, void * data) { @@ -232,6 +234,41 @@ static void cpy_thread_f32_f16_sameshape(unsigned int nth, unsigned int ith, voi } } +static inline void cpy_dma_sametype_sameshape( + struct htp_ops_context * octx, + const struct htp_tensor * dst, + const struct htp_tensor * src0, + uint32_t elem_size, + uint32_t ne00, uint32_t ne01, uint32_t ne02, uint32_t ne03, + uint32_t nb01, uint32_t nb02, uint32_t nb03, + uint32_t nb1, uint32_t nb2, uint32_t nb3 +) { + const bool contiguous_outer = + (ne02 == 1 || (nb02 == ne01 * nb01 && nb2 == ne01 * nb1)) && + (ne03 == 1 || (nb03 == ne02 * nb02 && nb3 == ne02 * nb2)); + + dma_queue * q = octx->ctx->dma[0]; + + if (contiguous_outer) { + dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data), nb1, nb01, ne00 * elem_size, ne01 * ne02 * ne03); + dma_queue_pop(q); + return; + } + + for (uint32_t i03 = 0; i03 < ne03; i03++) { + for (uint32_t i02 = 0; i02 < ne02; i02++) { + uint8_t* dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3; + uint8_t* src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03; + if (!dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01)) { + dma_queue_flush(q); + dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01); + } + } + } + + dma_queue_flush(q); +} + int op_cpy(struct htp_ops_context * octx) { cpy_preamble; @@ -264,14 +301,11 @@ int op_cpy(struct htp_ops_context * octx) { ct.src0_nrows_per_thread = (nr + n_threads - 1) / n_threads; - worker_callback_t copy_fun; + worker_callback_t copy_fun = NULL; + bool use_dma = false; if (sametype && sameshape) { - if (src0->type == HTP_TYPE_F32) { - copy_fun = cpy_thread_f32_sameshape; - } else { - copy_fun = cpy_thread_f16_sameshape; - } + use_dma = true; } else if (sameshape) { /**/ if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) copy_fun = cpy_thread_f16_f32_sameshape; @@ -289,7 +323,28 @@ int op_cpy(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - worker_pool_run_func(octx->ctx->worker_pool, copy_fun, &ct, n_threads); + if (use_dma) { + cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3); + } else { + worker_pool_run_func(octx->ctx->worker_pool, copy_fun, &ct, n_threads); + } + + const struct htp_tensor *sync = octx->src[1]; + if (sync) { + if (!use_dma) { + // htp_tensor_flush_all(octx->ctx, octx->dsts, 1); + qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); + } + + atomic_uint * sync_fence = (atomic_uint *) sync->data; + const uint32_t seq = (uint32_t) octx->op_params[0]; + + atomic_store(&sync_fence[0], seq); + asm volatile ("syncht" : : : "memory"); + Q6_dccleaninva_A((void *) sync_fence); + + FARF(HIGH, "ggml-hex: sync-release : fence %p seq %u\n", sync_fence, seq); + } return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.h index 264284bda..190ca3a9b 100644 --- a/ggml/src/ggml-hexagon/htp/dma-queue.h +++ b/ggml/src/ggml-hexagon/htp/dma-queue.h @@ -244,17 +244,18 @@ static inline dma_ptr dma_queue_pop(dma_queue * q) { return dptr; } - dma_descriptor_2d * desc = &r->desc[r->pop_idx]; + dptr = r->dptr[r->pop_idx]; + + volatile dma_descriptor_2d * desc = &r->desc[r->pop_idx]; // Wait for desc to complete if (!desc->done) { + // FARF(ALWAYS, "dma-poll: idx %u dst %p src %p", r->pop_idx, dptr.dst, dptr.src); while (!desc->done) { dmpoll(); } } - dptr = r->dptr[r->pop_idx]; - htp_trace_event_stop(r->trace, HTP_TRACE_EVT_DMA, r->pop_idx); r->pop_idx = (r->pop_idx + 1) & r->idx_mask; diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c index 817656290..c76b4d3a3 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c @@ -30,6 +30,8 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" +#include "hvx-quant.h" #include "flash-attn-ops.h" #include "hvx-fa-kernels.h" @@ -85,12 +87,17 @@ struct htp_fa_context { uint8_t * spad_m; uint8_t * spad_a; + const struct htp_tensor * k; + const struct htp_tensor * v; + uint64_t t_start; }; struct hmx_fa_context { const struct htp_ops_context * octx; const struct htp_tensor * sinks; // attention sinks (src[4]), NULL if absent + const struct htp_tensor * k; + const struct htp_tensor * v; bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_threads >= 2 uint32_t n_threads; @@ -214,8 +221,8 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * const uint32_t DV = nev0; const size_t size_q_row = DK * ((q->type == HTP_TYPE_F32) ? 4 : 2); - const size_t size_k_row = DK * sizeof(__fp16); - const size_t size_v_row = DV * sizeof(__fp16); + const size_t size_k_row = htp_tensor_get_row_size(k->type, DK); + const size_t size_v_row = htp_tensor_get_row_size(v->type, DV); // Scratchpad buffers for Q, K, V, Mask, and VKQ32 accumulator uint8_t * spad_q = factx->spad_q + factx->size_q_block * ith; @@ -364,6 +371,23 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * uint8_t * v_base = dma_queue_pop(dma).dst; // V __fp16 * m_base = mask ? dma_queue_pop(dma).dst : NULL; // M + if (factx->k->type == HTP_TYPE_Q8_0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir); + for (uint32_t r = 0; r < current_block_size; ++r) { + __fp16 * row_k = (__fp16 *)(k_base + r * factx->size_k_row_padded); + hvx_dequantize_row_q8_0_f16(row_k, row_k, DK); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir); + } + if (factx->v->type == HTP_TYPE_Q8_0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir); + for (uint32_t r = 0; r < current_block_size; ++r) { + __fp16 * row_v = (__fp16 *)(v_base + r * factx->size_v_row_padded); + hvx_dequantize_row_q8_0_f16(row_v, row_v, DV); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir); + } + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_QK, ir); // Inner loop processing the block from VTCM @@ -625,6 +649,12 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start)); + if (factx->k->type == HTP_TYPE_Q8_0) { + for (uint32_t r = start; r < end; ++r) { + __fp16 * row_k = (__fp16 *)((char *)args->curr_k + r * args->src_stride * sizeof(__fp16)); + hvx_dequantize_row_q8_0_f16(row_k, row_k, factx->DK); + } + } hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles[args->buf_idx], (const __fp16 *) args->curr_k, total_rows, factx->DK, args->src_stride, start, end); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start)); @@ -673,6 +703,12 @@ static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start)); + if (factx->v->type == HTP_TYPE_Q8_0) { + for (uint32_t r = start; r < end; ++r) { + __fp16 * row_v = (__fp16 *)((char *)args->v_src + r * args->src_stride * sizeof(__fp16)); + hvx_dequantize_row_q8_0_f16(row_v, row_v, factx->DV); + } + } hmx_interleave_cols_to_tiles(v_tiles_dst, (const __fp16 *) args->v_src, total_rows, factx->DV, args->src_stride, (uint32_t) args->n_col_tiles, start, end); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start)); @@ -1809,6 +1845,8 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { memset(&factx, 0, sizeof(factx)); factx.octx = octx; factx.sinks = octx->src[4]; // NULL if this op has no attention sinks + factx.k = k; + factx.v = v; factx.n_threads = kparams->n_threads; factx.DK = DK; factx.DV = DV; @@ -1853,10 +1891,10 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { // ======== VTCM allocation (GQA-aware) ======== // K/V row sizes drive the DMA descriptors (not the VTCM layout) and are used // throughout the KV loop below. - const size_t size_k_row = DK * sizeof(__fp16); - const size_t size_v_row = DV * sizeof(__fp16); - const size_t size_k_row_padded = hex_round_up(size_k_row, 128); - const size_t size_v_row_padded = hex_round_up(size_v_row, 128); + const size_t size_k_row = htp_tensor_get_row_size(k->type, DK); + const size_t size_v_row = htp_tensor_get_row_size(v->type, DV); + const size_t size_k_row_padded = hex_round_up(DK * sizeof(__fp16), 128); + const size_t size_v_row_padded = hex_round_up(DV * sizeof(__fp16), 128); // Build the VTCM layout once (shared with the host estimator) and place every // scratch buffer at its computed offset. @@ -2348,7 +2386,9 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { const struct htp_tensor * dst = octx->dst; // Check support - if ((q->type != HTP_TYPE_F16 && q->type != HTP_TYPE_F32) || k->type != HTP_TYPE_F16 || v->type != HTP_TYPE_F16) { + if ((q->type != HTP_TYPE_F16 && q->type != HTP_TYPE_F32) || + (k->type != HTP_TYPE_F16 && k->type != HTP_TYPE_Q8_0) || + (v->type != HTP_TYPE_F16 && v->type != HTP_TYPE_Q8_0)) { return HTP_STATUS_NO_SUPPORT; } @@ -2364,6 +2404,8 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { struct htp_fa_context factx; factx.octx = octx; + factx.k = k; + factx.v = v; factx.t_start = HAP_perf_get_qtimer_count(); diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c index bf7063e98..05769d17f 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c @@ -12,18 +12,17 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #include "hvx-utils.h" +#include "hvx-quant.h" +#include "get-rows-ops.h" +#include "work-queue.h" struct get_rows_context { struct htp_ops_context * octx; - uint32_t tasks_per_thread; - uint32_t total_tasks; - uint32_t chunks_per_row; - uint32_t chunk_size; - struct fastdiv_values get_rows_div_ne10; - struct fastdiv_values get_rows_div_ne10_ne11; - struct fastdiv_values get_rows_div_chunks_per_row; + const struct htp_get_rows_kernel_params * kparams; + struct htp_get_rows_vtcm_layout vtcm_layout; + uint8_t * vtcm_base; }; #define get_rows_preamble \ @@ -56,102 +55,161 @@ struct get_rows_context { \ const uint32_t nr = ne10 * ne11 * ne12; -static void get_rows_thread_f32_f32_dma(unsigned int nth, unsigned int ith, void *data) { - struct get_rows_context * grctx = (struct get_rows_context *)data; - struct htp_ops_context * octx = grctx->octx; - get_rows_preamble; - - uint64_t qt = HAP_perf_get_qtimer_count(); - - const uint32_t dr = grctx->tasks_per_thread; - const uint32_t ir0 = dr * ith; - if (ir0 >= grctx->total_tasks) { - return; - } - const uint32_t ir1 = MIN(ir0 + dr, grctx->total_tasks); - - const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32); - - dma_queue * dma_queue = octx->ctx->dma[ith]; - for (uint32_t i = ir0; i < ir1; ++i) { - const uint32_t i12 = fastdiv(i, &grctx->get_rows_div_ne10_ne11); - const uint32_t rem = i - i12 * ne11 * ne10; - const uint32_t i11 = fastdiv(rem, &grctx->get_rows_div_ne10); - const uint32_t i10 = rem - i11 * ne10; - - const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12; - uint32_t i01 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr; - - if (i01 >= ne01) { - continue; - } - - const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i11*nb02 + i12*nb03; - const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; - - while (!dma_queue_push(dma_queue, dma_make_ptr((void *)dst_ptr, (const void *)src0_ptr), nb1, nb01, ne00 * sizeof(float), 1)) { - dma_queue_pop(dma_queue); - } - } - dma_queue_flush(dma_queue); - - qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt); - FARF(HIGH, "get-rows-f32-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, - ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt); +#define GET_ROWS_THREAD_ST_FN(IDX_TYPE) \ +static void get_rows_thread_st_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ + struct get_rows_context * grctx = (struct get_rows_context *)data; \ + struct htp_ops_context * octx = grctx->octx; \ + const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \ + get_rows_preamble; \ + const uint32_t dr = kparams->tasks_per_thread; \ + const uint32_t ir0 = dr * ith; \ + if (ir0 >= kparams->total_tasks) { \ + return; \ + } \ + const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \ + const uint32_t row_size_bytes = htp_tensor_get_row_size(octx->src[0]->type, ne00); \ + dma_queue * dma_queue = octx->ctx->dma[ith]; \ + for (uint32_t i = ir0; i < ir1; ++i) { \ + const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); \ + const uint32_t rem = i - i12 * ne11 * ne10; \ + const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \ + const uint32_t i10 = rem - i11 * ne10; \ + const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \ + const uint32_t i01 = (uint32_t)*src1_ptr; \ + assert(i01 < ne01); \ + const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \ + const uint32_t i02 = i11 - q02 * ne02; \ + const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \ + const uint32_t i03 = i12 - q03 * ne03; \ + const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03; \ + const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; \ + while (!dma_queue_push(dma_queue, dma_make_ptr((void *)dst_ptr, (const void *)src0_ptr), nb1, nb01, \ + row_size_bytes, 1)) { \ + dma_queue_pop(dma_queue); \ + } \ + } \ + dma_queue_flush(dma_queue); \ } -static void get_rows_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void *data) { - struct get_rows_context * grctx = (struct get_rows_context *)data; - struct htp_ops_context * octx = grctx->octx; - get_rows_preamble; +GET_ROWS_THREAD_ST_FN(int32_t) +GET_ROWS_THREAD_ST_FN(int64_t) - uint64_t qt = HAP_perf_get_qtimer_count(); - - const uint32_t dr = grctx->tasks_per_thread; - const uint32_t ir0 = dr * ith; - if (ir0 >= grctx->total_tasks) { - return; - } - const uint32_t ir1 = MIN(ir0 + dr, grctx->total_tasks); - - const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32); - - const uint32_t chunks_per_row = grctx->chunks_per_row; - const uint32_t chunk_size = grctx->chunk_size; - for (uint32_t i = ir0; i < ir1; ++i) { - const uint32_t row_idx = fastdiv(i, &grctx->get_rows_div_chunks_per_row); - const uint32_t chunk_idx = i - row_idx * chunks_per_row; - - const uint32_t i12 = fastdiv(row_idx, &grctx->get_rows_div_ne10_ne11); - const uint32_t rem = row_idx - i12 * ne11 * ne10; - const uint32_t i11 = fastdiv(rem, &grctx->get_rows_div_ne10); - const uint32_t i10 = rem - i11 * ne10; - - const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12; - uint32_t i01 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr; - - if (i01 >= ne01) { - continue; - } - - const uint32_t offset = chunk_idx * chunk_size; - if (offset < ne00) { - const uint32_t copy_size = MIN(chunk_size, ne00 - offset); - const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i11*nb02 + i12*nb03 + offset * sizeof(float); - const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3 + offset * sizeof(float); - hvx_copy_f32_uu((uint8_t *)dst_ptr, (const uint8_t *)src0_ptr, copy_size); - } - } - - qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt); - FARF(HIGH, "get-rows-f32-f32-hvx %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, - ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt); +#define GET_ROWS_THREAD_DT_FN(TYPE_NAME, SRC0_SIZE_EXPR, IDX_TYPE, COMPUTE_EXPR) \ +static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ + struct get_rows_context * grctx = (struct get_rows_context *)data; \ + struct htp_ops_context * octx = grctx->octx; \ + const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \ + get_rows_preamble; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + const uint32_t dr = kparams->tasks_per_thread; \ + const uint32_t ir0 = dr * ith; \ + if (ir0 >= kparams->total_tasks) { \ + return; \ + } \ + const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \ + const uint32_t chunks_per_row = kparams->chunks_per_row; \ + const uint32_t chunk_size = kparams->chunk_size; \ + dma_queue * dma_queue = octx->ctx->dma[ith]; \ + const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; \ + uint8_t * vtcm_src0 = grctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \ + uint8_t * vtcm_dst = grctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \ + for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, spad_idx++) { \ + const uint32_t i = ir0 + step; \ + const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \ + const uint32_t chunk_idx = i - row_idx * chunks_per_row; \ + const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \ + const uint32_t rem = row_idx - i12 * ne11 * ne10; \ + const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \ + const uint32_t i10 = rem - i11 * ne10; \ + const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \ + const uint32_t i01 = (uint32_t)*src1_ptr; \ + assert(i01 < ne01); \ + const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \ + const uint32_t i02 = i11 - q02 * ne02; \ + const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \ + const uint32_t i03 = i12 - q03 * ne03; \ + const uint32_t offset = chunk_idx * chunk_size; \ + const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \ + const uint32_t cur_src0_bytes = SRC0_SIZE_EXPR(cur_elems); \ + const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \ + const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03 + SRC0_SIZE_EXPR(offset); \ + dma_queue_push(dma_queue, \ + dma_make_ptr((void *)(uintptr_t)octx->dst->data, \ + vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \ + cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 0); \ + dma_queue_push(dma_queue, \ + dma_make_ptr((void *)(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size), \ + (const void *)src0_ptr), \ + vtcm_layout->src0_spad_half_size, cur_src0_bytes, cur_src0_bytes, 1); \ + } \ + for (uint32_t step = 0; step < ir1 - ir0; ++step) { \ + const uint32_t i = ir0 + step; \ + void * dst_spad = (void *) dma_queue_pop(dma_queue).src; \ + void * src_spad = (void *) dma_queue_pop(dma_queue).dst; \ + const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \ + const uint32_t chunk_idx = i - row_idx * chunks_per_row; \ + const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \ + const uint32_t rem = row_idx - i12 * ne11 * ne10; \ + const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \ + const uint32_t i10 = rem - i11 * ne10; \ + const uint32_t offset = chunk_idx * chunk_size; \ + const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \ + const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, i); \ + COMPUTE_EXPR; \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, i); \ + const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3 + offset * sizeof(float); \ + dma_queue_push(dma_queue, \ + dma_make_ptr((void *)dst_ptr, (const void *)dst_spad), \ + cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 1); \ + const uint32_t next_step = step + 2; \ + if (next_step < ir1 - ir0) { \ + const uint32_t pi = ir0 + next_step; \ + const uint32_t prow_idx = fastdiv(pi, &kparams->div_chunks_per_row); \ + const uint32_t pchunk_idx = pi - prow_idx * chunks_per_row; \ + const uint32_t pi12 = fastdiv(prow_idx, &kparams->div_ne10_ne11); \ + const uint32_t prem = prow_idx - pi12 * ne11 * ne10; \ + const uint32_t pi11 = fastdiv(prem, &kparams->div_ne10); \ + const uint32_t pi10 = prem - pi11 * ne10; \ + const IDX_TYPE * psrc1_ptr = (const IDX_TYPE *)(octx->src[1]->data + pi10*nb10 + pi11*nb11 + pi12*nb12); \ + const uint32_t pi01 = (uint32_t)*psrc1_ptr; \ + assert(pi01 < ne01); \ + const uint32_t pq02 = fastdiv(pi11, &kparams->div_ne02); \ + const uint32_t pi02 = pi11 - pq02 * ne02; \ + const uint32_t pq03 = fastdiv(pi12, &kparams->div_ne03); \ + const uint32_t pi03 = pi12 - pq03 * ne03; \ + const uint32_t poffset = pchunk_idx * chunk_size; \ + const uint32_t pcur_elems = (poffset < ne00) ? MIN(chunk_size, ne00 - poffset) : 0; \ + const uint32_t pcur_src0_bytes = SRC0_SIZE_EXPR(pcur_elems); \ + const uintptr_t psrc0_ptr = \ + octx->src[0]->data + pi01*nb01 + pi02*nb02 + pi03*nb03 + SRC0_SIZE_EXPR(poffset); \ + dma_queue_push(dma_queue, \ + dma_make_ptr((void *)src_spad, (const void *)psrc0_ptr), \ + vtcm_layout->src0_spad_half_size, pcur_src0_bytes, pcur_src0_bytes, 1); \ + } \ + } \ + dma_queue_flush(dma_queue); \ } +#define F32_BYTES(n) ((n) * sizeof(float)) +#define F16_BYTES(n) ((n) * sizeof(__fp16)) +#define Q8_0_BYTES(n) (((n) / 32) * sizeof(block_q8_0)) + +GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int32_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); }) +GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int64_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); }) + +GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int32_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); }) +GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int64_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); }) + +GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); }) +GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); }) + int op_get_rows(struct htp_ops_context * octx) { - get_rows_preamble; + const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params *) octx->kernel_params; - if (octx->src[0]->type != HTP_TYPE_F32) { + if (octx->src[0]->type != HTP_TYPE_F32 && + octx->src[0]->type != HTP_TYPE_F16 && + octx->src[0]->type != HTP_TYPE_Q8_0) { return HTP_STATUS_NO_SUPPORT; } @@ -167,52 +225,28 @@ int op_get_rows(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t nb00 = octx->src[0]->nb[0]; - const uint32_t nb0 = octx->dst->nb[0]; - - const bool can_use_dma = (nb00 == sizeof(float)) && (nb0 == sizeof(float)); - const bool use_dma = can_use_dma && (ne00 >= 2048); - struct get_rows_context grctx; grctx.octx = octx; - grctx.get_rows_div_ne10 = init_fastdiv_values(octx->src[1]->ne[0]); - grctx.get_rows_div_ne10_ne11 = init_fastdiv_values(octx->src[1]->ne[0] * octx->src[1]->ne[1]); + grctx.kparams = kparams; + grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base; - if (use_dma) { - grctx.chunks_per_row = 1; - grctx.chunk_size = ne00; - grctx.total_tasks = nr; - grctx.get_rows_div_chunks_per_row = init_fastdiv_values(1); + const uint32_t ne00 = octx->src[0]->ne[0]; + htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, kparams->n_threads); - const uint32_t n_threads = MIN(nr, octx->n_threads); - grctx.tasks_per_thread = (nr + n_threads - 1) / n_threads; + const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32); - worker_pool_run_func(octx->ctx->worker_pool, get_rows_thread_f32_f32_dma, &grctx, n_threads); + work_queue_func_t q_func = NULL; + if (kparams->use_dma) { + q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t); } else { - uint32_t chunks_per_row = 1; - uint32_t chunk_size = ne00; - uint32_t total_tasks = nr; - - if (nr < octx->n_threads) { - const uint32_t min_chunk_size = 1024; - uint32_t max_chunks = ne00 / min_chunk_size; - if (max_chunks == 0) { - max_chunks = 1; - } - chunks_per_row = MIN((octx->n_threads + nr - 1) / nr, max_chunks); - chunk_size = (ne00 + chunks_per_row - 1) / chunks_per_row; - total_tasks = nr * chunks_per_row; + switch (octx->src[0]->type) { + case HTP_TYPE_F32: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f32_int32_t : get_rows_thread_f32_int64_t); break; + case HTP_TYPE_F16: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f16_int32_t : get_rows_thread_f16_int64_t); break; + case HTP_TYPE_Q8_0: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_q8_0_int32_t : get_rows_thread_q8_0_int64_t); break; + default: return HTP_STATUS_NO_SUPPORT; } - - grctx.chunks_per_row = chunks_per_row; - grctx.chunk_size = chunk_size; - grctx.total_tasks = total_tasks; - grctx.get_rows_div_chunks_per_row = init_fastdiv_values(chunks_per_row); - - const uint32_t n_threads = MIN(total_tasks, octx->n_threads); - grctx.tasks_per_thread = (total_tasks + n_threads - 1) / n_threads; - - worker_pool_run_func(octx->ctx->worker_pool, get_rows_thread_f32_f32_hvx, &grctx, n_threads); } + + work_queue_run(octx->ctx->work_queue, q_func, &grctx, kparams->n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.h new file mode 100644 index 000000000..0e7c2ca8c --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h @@ -0,0 +1,77 @@ +#ifndef HTP_GET_ROWS_OPS_H +#define HTP_GET_ROWS_OPS_H + +#include "hex-fastdiv.h" + +struct htp_get_rows_kernel_params { + int32_t n_threads; + int32_t use_dma; + int32_t chunks_per_row; + int32_t chunk_size; + int32_t total_tasks; + int32_t tasks_per_thread; + int32_t vtcm_size; + + // Fastdiv helpers + struct fastdiv_values div_ne10; + struct fastdiv_values div_ne10_ne11; + struct fastdiv_values div_chunks_per_row; + struct fastdiv_values div_ne02; + struct fastdiv_values div_ne03; +}; + +struct htp_get_rows_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_dst; + + size_t src0_bytes_per_thread; + size_t dst_bytes_per_thread; + + size_t src0_spad_half_size; + size_t dst_spad_half_size; +}; + +static inline void htp_get_rows_vtcm_layout_build( + struct htp_get_rows_vtcm_layout * vtcm_layout, + int type, + uint32_t ne00, + uint32_t n_threads) { + + uint32_t src0_row_size = 0; + switch (type) { + case 0: // HTP_TYPE_F32 + src0_row_size = ne00 * 4; + break; + case 1: // HTP_TYPE_F16 + src0_row_size = ne00 * 2; + break; + case 8: // HTP_TYPE_Q8_0 + src0_row_size = (ne00 / 32) * 34; + break; + default: + src0_row_size = 0; + break; + } + + size_t src0_row_size_aligned = (src0_row_size + 255) & ~255; + size_t dst_row_size_aligned = (ne00 * sizeof(float) + 255) & ~255; + + vtcm_layout->src0_spad_half_size = src0_row_size_aligned; + vtcm_layout->dst_spad_half_size = dst_row_size_aligned; + + vtcm_layout->src0_bytes_per_thread = src0_row_size_aligned * 2; + vtcm_layout->dst_bytes_per_thread = dst_row_size_aligned * 2; + + vtcm_layout->off_src0 = 0; + vtcm_layout->off_dst = vtcm_layout->off_src0 + vtcm_layout->src0_bytes_per_thread * n_threads; + vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads; +} + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_get_rows_kernel_params) <= 128, "htp_get_rows_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_get_rows_kernel_params) <= 128, "htp_get_rows_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_GET_ROWS_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/hex-utils.h b/ggml/src/ggml-hexagon/htp/hex-utils.h index 93e87efcb..1b3965030 100644 --- a/ggml/src/ggml-hexagon/htp/hex-utils.h +++ b/ggml/src/ggml-hexagon/htp/hex-utils.h @@ -39,17 +39,22 @@ static inline void hex_l2fetch_block(const void * addr, size_t size) { #define HEX_L2_LINE_SIZE 128 #define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (lines per loop iteration) +#define HEX_L2_FLUSH_IL_THRESHOLD 1024 // inline flush threshold #define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024) #define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024) static inline void hex_l2flush(void * addr, size_t size) { const uint32_t s = ((uint32_t) addr) & ~(HEX_L2_LINE_SIZE - 1); const uint32_t e = (((uint32_t) addr) + size + HEX_L2_LINE_SIZE - 1) & ~(HEX_L2_LINE_SIZE - 1); - for (uint32_t i = s; i < e; i += HEX_L2_BLOCK_SIZE) { - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 0); - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 1); - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 2); - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 3); + const uint32_t eb = s + ((e - s) & ~(HEX_L2_BLOCK_SIZE - 1)); + for (uint32_t i = s; i < eb; i += HEX_L2_BLOCK_SIZE) { + Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 0)); + Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 1)); + Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 2)); + Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 3)); + } + for (uint32_t i = eb; i < e; i += HEX_L2_LINE_SIZE) { + Q6_dccleaninva_A((void *) i); } } diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h index e0f9a0c40..88ecf144b 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ctx.h +++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h @@ -117,8 +117,7 @@ struct htp_context { int op_matmul(struct htp_ops_context * octx); int op_matmul_id(struct htp_ops_context * octx); -int op_matmul_qkv(struct htp_ops_context * octx); -int op_matmul_ffn(struct htp_ops_context * octx); +int op_matmul_nx(struct htp_ops_context * octx); int op_binary(struct htp_ops_context * octx); int op_unary(struct htp_ops_context * octx); int op_sum_rows(struct htp_ops_context * octx); @@ -141,5 +140,6 @@ int op_solve_tri(struct htp_ops_context * octx); int op_gated_delta_net(struct htp_ops_context * octx); int op_pad(struct htp_ops_context * octx); int op_im2col(struct htp_ops_context * octx); +int op_allreduce(struct htp_ops_context * octx); #endif /* HTP_CTX_H */ diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index a138f062a..b4023b34d 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -43,13 +43,6 @@ enum htp_data_type { -// Mask to enable various stages of the Ops. -// Used for debugging and profiling. -enum htp_op_stage { - HTP_OPSTAGE_QUEUE = (1 << 0), // Enable Queueing (ie calls into NPU) - HTP_OPSTAGE_COMPUTE = (1 << 1), // Enable Compute -}; - // Do not reorder first 4 (used as an index) enum htp_op_code { HTP_OP_MUL = 0, @@ -58,8 +51,7 @@ enum htp_op_code { HTP_OP_DIV = 3, HTP_OP_MUL_MAT, HTP_OP_MUL_MAT_ID, - HTP_OP_MUL_MAT_QKV, - HTP_OP_MUL_MAT_FFN, + HTP_OP_MUL_MAT_NX, HTP_OP_MUL_MAT_ADD, HTP_OP_RMS_NORM, HTP_OP_RMS_NORM_MUL, @@ -99,12 +91,15 @@ enum htp_op_code { HTP_OP_CONCAT, HTP_OP_CLAMP, HTP_OP_IM2COL, + HTP_OP_FENCE, + HTP_OP_ALLREDUCE, + HTP_OP_ALLREDUCE_ADD, HTP_OP_INVALID }; #define HTP_OP_MAX_DIMS 4 // aka GGML_MAX_DIMS -#define HTP_OP_MAX_INPUTS 6 // aka GGML_MAX_SRCS +#define HTP_OP_MAX_INPUTS 10 // aka GGML_MAX_SRCS #define HTP_OP_MAX_OUTPUTS 4 #define HTP_OP_MAX_PARAMS 16 // aka GGML_MAX_OP_PARAMS #define HTP_OP_MAX_KERN_PARAMS 32 @@ -112,13 +107,16 @@ enum htp_op_code { #define HTP_OP_MAX_BUFS 16 #define HTP_OP_MAX_TENSORS 8192 // must stay under 64K (uint16) +#define HTP_FENCE_TIMEOUT (1000000000ULL) + #define HTP_OP_MAX_VMEM_DEFAULT (3355443200u) #define HTP_MMAP_MAX_VMEM (2147483648u) enum htp_tensor_flags { - HTP_TENSOR_COMPUTE = (1U << 0), // Tensor buffer temporal compute data (not weights) - HTP_TENSOR_DIRTY = (1U << 1) // Tensor buffer is dirty and needs to be flushed + HTP_TENSOR_WEIGHT = (1U << 0), // Tensor buffer model weight data (not compute) + HTP_TENSOR_REPACK = (1U << 1), // Tensor is in repacked tiled format + HTP_TENSOR_FENCE = (1U << 2) // Tensor is synchronization fence (explicitly managed) }; // Tensor descriptor @@ -175,6 +173,7 @@ enum htp_trace_event_id { HTP_TRACE_EVT_L2FLUSH = 1, HTP_TRACE_EVT_INIT = 2, HTP_TRACE_EVT_BUFF = 3, + HTP_TRACE_EVT_FENCE = 4, HTP_TRACE_EVT_HVX_COMP = 20, HTP_TRACE_EVT_HVX_A_QUANT = 21, @@ -215,6 +214,7 @@ struct htp_opbatch_req { uint32_t n_ops; // Number of ops uint32_t n_traces; // Number of trace descriptors per thread uint32_t pad; // unused + uint64_t seq; // Sequence number // struct htp_buf_desc bufs[]; -- dspqueue buf 0 // struct htp_tensor tensors[]; -- dspqueue buf 0 // struct htp_op_desc ops[]; -- dspqueue buf 0 @@ -231,6 +231,7 @@ struct htp_opbatch_rsp { uint32_t pad; // align to 8 bytes uint64_t cycles_start; // Start cycle counter uint64_t cycles_stop; // Stop cycle counter + uint64_t seq; // Sequence number // struct htp_prof_desc profs[]; -- dspqueue buf 0 }; diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.c b/ggml/src/ggml-hexagon/htp/htp-tensor.c index 39436e26d..ae377c922 100644 --- a/ggml/src/ggml-hexagon/htp/htp-tensor.c +++ b/ggml/src/ggml-hexagon/htp/htp-tensor.c @@ -79,7 +79,14 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co for (uint32_t i = 0; i < n; i++) { const struct htp_tensor * t = tensors[i]; - if (!t) continue; + if (!t || (t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE))) { + continue; + } + + if (t->size <= HEX_L2_FLUSH_IL_THRESHOLD) { + hex_l2flush((void *) (uintptr_t) t->data, t->size); + continue; + } uint32_t t_start = t->data; uint32_t t_end = t_start + t->size; @@ -242,7 +249,7 @@ void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * co for (uint32_t i = 0; i < n; i++) { const struct htp_tensor * t = tensors[i]; - if (t && (t->flags & HTP_TENSOR_COMPUTE) && is_tensor_dirty(ctx, t)) { + if (t && !(t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE)) && is_tensor_dirty(ctx, t)) { dirty_tensors[n_dirty++] = t; total_dirty += t->size; } diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.h b/ggml/src/ggml-hexagon/htp/htp-tensor.h index 2c3fc54c7..c9cadbae3 100644 --- a/ggml/src/ggml-hexagon/htp/htp-tensor.h +++ b/ggml/src/ggml-hexagon/htp/htp-tensor.h @@ -13,6 +13,15 @@ static inline uint32_t * htp_tensor_flags(const struct htp_tensor * t) { return (uint32_t *) &t->flags; } +static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) { + switch (type) { + case HTP_TYPE_F32: return ne00 * 4; + case HTP_TYPE_F16: return ne00 * 2; + case HTP_TYPE_Q8_0: return (ne00 / 32) * 34; + default: return 0; + } +} + struct htp_context; void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h index 82e341697..c8d0003ab 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-arith.h +++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h @@ -17,9 +17,9 @@ #define hvx_arith_loop_body(dst_type, src0_type, src1_type, elem_size, vec_store, vec_op) \ do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src0_type * restrict vsrc0 = (src0_type *) src0; \ - src1_type * restrict vsrc1 = (src1_type *) src1; \ + dst_type * vdst = (dst_type *) dst; \ + src0_type * vsrc0 = (src0_type *) src0; \ + src1_type * vsrc1 = (src1_type *) src1; \ \ const uint32_t epv = 128 / (elem_size); \ const uint32_t nvec = n / epv; \ @@ -57,40 +57,40 @@ // Generic macro to define alignment permutations for an op #define DEFINE_HVX_BINARY_OP_VARIANTS(OP_NAME, OP_MACRO, ELEM_TYPE) \ -static inline void OP_NAME##_aaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ +static inline void OP_NAME##_aaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ assert((uintptr_t) dst % 128 == 0); \ assert((uintptr_t) src0 % 128 == 0); \ assert((uintptr_t) src1 % 128 == 0); \ hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ } \ -static inline void OP_NAME##_aau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ +static inline void OP_NAME##_aau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ assert((uintptr_t) dst % 128 == 0); \ assert((uintptr_t) src0 % 128 == 0); \ hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ } \ -static inline void OP_NAME##_aua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ +static inline void OP_NAME##_aua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ assert((uintptr_t) dst % 128 == 0); \ assert((uintptr_t) src1 % 128 == 0); \ hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ } \ -static inline void OP_NAME##_auu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ +static inline void OP_NAME##_auu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ assert((uintptr_t) dst % 128 == 0); \ hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ } \ -static inline void OP_NAME##_uaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ +static inline void OP_NAME##_uaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ assert((uintptr_t) src0 % 128 == 0); \ assert((uintptr_t) src1 % 128 == 0); \ hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ } \ -static inline void OP_NAME##_uau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ +static inline void OP_NAME##_uau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ assert((uintptr_t) src0 % 128 == 0); \ hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ } \ -static inline void OP_NAME##_uua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ +static inline void OP_NAME##_uua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ assert((uintptr_t) src1 % 128 == 0); \ hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ } \ -static inline void OP_NAME##_uuu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ +static inline void OP_NAME##_uuu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ } \ diff --git a/ggml/src/ggml-hexagon/htp/hvx-quant.h b/ggml/src/ggml-hexagon/htp/hvx-quant.h new file mode 100644 index 000000000..6b172cd63 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hvx-quant.h @@ -0,0 +1,165 @@ +#ifndef HVX_QUANT_H +#define HVX_QUANT_H + +#include +#include +#include + +#include "hvx-arith.h" +#include "hvx-base.h" +#include "hvx-reduce.h" +#include "hvx-repl.h" +#include "hvx-utils.h" + +#ifndef GGML_COMMON_DECL_C +#define GGML_COMMON_DECL_C +#endif +#include "ggml-common.h" +#include "ggml-impl.h" + +static inline void hvx_quantize_row_q8_0_f32(void * restrict dst_ptr, const float * restrict src_ptr, int n) { + const int nb = n / QK8_0; + block_q8_0 * dst = (block_q8_0 *) dst_ptr; + HVX_Vector zero = Q6_V_vzero(); + + int i = 0; + for (; i + 3 < nb; i += 4) { + HVX_Vector * vx = (HVX_Vector *) (src_ptr + i * QK8_0); + + HVX_Vector vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0])); + HVX_Vector vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1])); + HVX_Vector vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2])); + HVX_Vector vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3])); + + HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero); + HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero); + HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); + HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); + + HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero); + HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero); + HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero); + HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero); + + HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf))); + HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf))); + + HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf))); + HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf))); + + HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 + HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 + HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16); + HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16); + + HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf); + HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf); + vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf)); + vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf)); + + HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf); + HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf); + HVX_Vector vx_i8 = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16); + + hvx_vec_store_u(&dst[i + 0].d, 2, vd01_hf); + hvx_vec_store_u(dst[i + 0].qs, 32, vx_i8); + + hvx_vec_store_u(&dst[i + 1].d, 2, Q6_V_vror_VR(vd01_hf, 64)); + hvx_vec_store_u(dst[i + 1].qs, 32, Q6_V_vror_VR(vx_i8, 32)); + + hvx_vec_store_u(&dst[i + 2].d, 2, vd23_hf); + hvx_vec_store_u(dst[i + 2].qs, 32, Q6_V_vror_VR(vx_i8, 64)); + + hvx_vec_store_u(&dst[i + 3].d, 2, Q6_V_vror_VR(vd23_hf, 64)); + hvx_vec_store_u(dst[i + 3].qs, 32, Q6_V_vror_VR(vx_i8, 96)); + } + + for (; i < nb; i++) { + const float * block_src = src_ptr + i * QK8_0; + HVX_Vector vx = *(const HVX_UVector *) block_src; + HVX_Vector v_abs = hvx_vec_abs_f32(vx); + HVX_Vector v_max = hvx_vec_reduce_max_f32(v_abs); + float amax = hvx_vec_get_f32(v_max); + + const float d = amax / 127.0f; + const float id = d ? (1.0f / d) : 0.0f; + dst[i].d = GGML_FP32_TO_FP16(d); + + HVX_Vector vid = hvx_vec_splat_f32(id); + HVX_Vector v_scaled = hvx_vec_mul_f32_f32(vx, vid); + HVX_Vector v_scaled_qf = Q6_Vqf32_vsub_VsfVsf(v_scaled, zero); + HVX_Vector v_scaled_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(zero, v_scaled_qf))); + HVX_Vector v_i16 = hvx_vec_i16_from_hf_rnd_sat(v_scaled_hf); + HVX_Vector v_i8 = Q6_Vb_vpack_VhVh_sat(zero, v_i16); + + hvx_vec_store_u(dst[i].qs, 32, v_i8); + } +} + +static inline void hvx_dequantize_row_q8_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, int n) { + const int nb = n / QK8_0; + const block_q8_0 * src = (const block_q8_0 *) src_ptr; + + for (int i = 0; i < nb; i++) { + HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d); + HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16); + HVX_Vector vd = Q6_V_lo_W(vp_f32); + + HVX_Vector vq_i8 = *(const HVX_UVector *) src[i].qs; + + HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq_i8); + HVX_Vector v_i16 = Q6_V_lo_W(p16); + HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(v_i16); + HVX_Vector v_i32 = Q6_V_lo_W(p32); + + HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32); + HVX_Vector res = hvx_vec_mul_f32_f32(v_f32, vd); + + float * block_dst = dst_ptr + i * QK8_0; + hvx_vmem(block_dst) = res; + } +} + +static inline void hvx_dequantize_row_q8_0_f16(__fp16 * restrict dst_ptr, const void * restrict src_ptr, int n) { + const int nb = n / QK8_0; + const block_q8_0 * src = (const block_q8_0 *) src_ptr; + + for (int i = nb - 1; i >= 0; i--) { + HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d); + HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16); + HVX_Vector vd = Q6_V_lo_W(vp_f32); + + HVX_Vector vq_i8 = *(const HVX_UVector *) src[i].qs; + + HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq_i8); + HVX_Vector v_i16 = Q6_V_lo_W(p16); + HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(v_i16); + HVX_Vector v_i32 = Q6_V_lo_W(p32); + + HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32); + HVX_Vector res_f32 = hvx_vec_mul_f32_f32(v_f32, vd); + + HVX_Vector res_f16 = hvx_vec_f32_to_f16(res_f32, Q6_V_vzero()); + + __fp16 * block_dst = dst_ptr + i * QK8_0; + hvx_vec_store_u(block_dst, QK8_0 * sizeof(__fp16), res_f16); + } +} + +static inline void hvx_dequantize_row_f16_f32(float * restrict dst_ptr, const void * restrict src_ptr, int n) { + const int nb = n / 32; + const _Float16 * src = (const _Float16 *) src_ptr; + + for (int i = 0; i < nb; i++) { + HVX_Vector v_f16 = *(const HVX_UVector *) (src + i * 32); + HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(v_f16); + HVX_Vector res = Q6_V_lo_W(vp_f32); + + float * block_dst = dst_ptr + i * 32; + hvx_vmem(block_dst) = res; + } +} + + + +#endif // HVX_QUANT_H diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index 880e20c99..975ba0c7a 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "hex-utils.h" #include "hex-dma.h" @@ -32,6 +33,7 @@ #include "htp_iface.h" #include "work-queue.h" #include "hex-profile.h" +#include "allreduce-ops.h" #define HMX_QUEUE_CAPACITY 16 #define HMX_QUEUE_STACK_SIZE 16384 @@ -46,6 +48,36 @@ struct htp_handle { struct htp_context * ctx; }; +static inline void * htp_mmap(uint32_t fd, uint32_t size) { + void * va = (void *)-1; + for (int retry = 0; retry < 2; retry++) { +#if __HVX_ARCH__ > 73 + va = HAP_mmap2(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); +#else + if (size > HTP_MMAP_MAX_VMEM) { + FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) size); + abort(); + } + va = HAP_mmap(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); +#endif + if (va != (void *)-1 && va != NULL) { + return va; + } + if (retry == 0) { + FARF(HIGH, "mmap failed first try (va %p fd %u size %u), retrying...", va, fd, size); + } + } + return NULL; +} + +static inline void htp_munmap(void * va, uint32_t size) { +#if __HVX_ARCH__ > 73 + HAP_munmap2(va, size); +#else + HAP_munmap(va, size); +#endif +} + AEEResult htp_iface_open(const char * uri, remote_handle64 * handle) { (void) uri; struct htp_handle * h = calloc(1, sizeof(*h)); @@ -127,11 +159,7 @@ AEEResult htp_iface_close(remote_handle64 handle) { // release the mmaps (if any) for (uint32_t i=0; immap[i].size) { -#if __HVX_ARCH__ > 73 - HAP_munmap2((void *) ctx->mmap[i].base, ctx->mmap[i].size); -#else - HAP_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size); -#endif + htp_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size); ctx->mmap[i].size = 0; ctx->mmap[i].base = NULL; ctx->mmap[i].fd = -1; @@ -175,18 +203,9 @@ AEEResult htp_iface_mmap(remote_handle64 handle, uint32_t fd, uint32_t size) { struct htp_mmap *m = &ctx->mmap[i]; if (!m->size) { FARF(HIGH, "mmap : fd %u size %u", fd, size); -#if __HVX_ARCH__ > 73 - void *va = HAP_mmap2(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); -#else - if (size > HTP_MMAP_MAX_VMEM) { // HAP_mmap has a size limit of 2GB - FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) size); - abort(); // can't do much else at this point - } - - void *va = HAP_mmap(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); -#endif - if (va == (void*)-1) { - FARF(ERROR, "mmap failed : va %p fd %u size %u", va, fd, (uint32_t) size); + void *va = htp_mmap(fd, size); + if (va == NULL) { + FARF(ERROR, "mmap failed : fd %u size %u", fd, (uint32_t) size); return AEE_EFAILED; } @@ -212,11 +231,7 @@ AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) { struct htp_mmap *m = &ctx->mmap[i]; if (fd < 0 || m->fd == fd) { FARF(HIGH, "unmmap : base %p fd %u size %u", (void*) m->base, m->fd, (uint32_t) m->size); -#if __HVX_ARCH__ > 73 - HAP_munmap2((void *) m->base, m->size); -#else - HAP_munmap((void *) m->base, m->size); -#endif + htp_munmap((void *) m->base, m->size); m->size = 0; m->base = NULL; m->fd = -1; @@ -228,7 +243,7 @@ AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) { static void vtcm_acquire(struct htp_context * ctx) { if (!ctx->vtcm_valid) { - int err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 1000000u); + int err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 10000000u); if (err != 0) { FARF(ERROR, "ggml-hex: failed to acquire VTCM: 0x%08x", (unsigned)err); abort(); @@ -692,8 +707,45 @@ static inline void profile_stop(uint32_t mode, struct profile_data * d) { } } +static int op_fence(struct htp_ops_context * octx) { + struct htp_context *ctx = octx->ctx; + struct htp_thread_trace * tr = &ctx->trace[0]; + const uint32_t seq = (uint32_t) octx->op_params[0]; + + htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); + + const struct htp_tensor * sync = octx->src[0]; + atomic_uint * sync_fence = (atomic_uint *) sync->data; + uint64_t spins = 0; + while (1) { + Q6_dccleaninva_A((void *) sync_fence); + asm volatile ("syncht" : : : "memory"); + uint32_t val = atomic_load(&sync_fence[0]); + if ((int32_t)(val - seq) >= 0) { + break; + } + if (++spins > HTP_FENCE_TIMEOUT) { + FARF(ERROR, "ggml-hex: sync-wait TIMEOUT : fence %p spins %llu seq %u\n", sync_fence, spins, seq); + break; + } + hex_pause(); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); + + FARF(HIGH, "ggml-hex: sync-done : fence %p spins %llu seq %u\n", sync_fence, spins, seq); + return HTP_STATUS_OK; +} + static int execute_op(struct htp_ops_context * octx) { switch (octx->op) { + case HTP_OP_FENCE: + return op_fence(octx); + + case HTP_OP_ALLREDUCE: + case HTP_OP_ALLREDUCE_ADD: + return op_allreduce(octx); + case HTP_OP_MUL_MAT: case HTP_OP_MUL_MAT_ADD: return op_matmul(octx); @@ -701,11 +753,8 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_MUL_MAT_ID: return op_matmul_id(octx); - case HTP_OP_MUL_MAT_QKV: - return op_matmul_qkv(octx); - - case HTP_OP_MUL_MAT_FFN: - return op_matmul_ffn(octx); + case HTP_OP_MUL_MAT_NX: + return op_matmul_nx(octx); case HTP_OP_MUL: case HTP_OP_ADD: @@ -818,12 +867,8 @@ static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct static inline void drop_mmap(struct htp_context *ctx, struct htp_mmap *m) { if (m->size) { - FARF(HIGH, "unmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size); -#if __HVX_ARCH__ > 73 - HAP_munmap2((void *) m->base, m->size); -#else - HAP_munmap((void *) m->base, m->size); -#endif + FARF(ALWAYS, "unmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size); + htp_munmap((void *) m->base, m->size); m->size = 0; m->base = 0; m->fd = -1; @@ -837,18 +882,9 @@ static inline void mmap_buf(struct htp_context *ctx, struct htp_buf_desc *b) { for (uint32_t i=0; i < HTP_MAX_MMAPS; i++) { struct htp_mmap *m = &ctx->mmap[i]; if (!m->size) { -#if __HVX_ARCH__ > 73 - void *va = HAP_mmap2(NULL, b->size, HAP_PROT_READ | HAP_PROT_WRITE, 0, b->fd, 0); -#else - if (b->size > HTP_MMAP_MAX_VMEM) { // HAP_mmap has a size limit of 2GB - FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) b->size); - abort(); // can't do much else at this point - } - - void *va = HAP_mmap(NULL, b->size, HAP_PROT_READ | HAP_PROT_WRITE, 0, b->fd, 0); -#endif - if (va == (void*)-1) { - FARF(ERROR, "mmap failed : va %p fd %u size %u", va, b->fd, (uint32_t) b->size); + void *va = htp_mmap(b->fd, b->size); + if (va == NULL) { + FARF(ERROR, "mmap failed : fd %u size %u", b->fd, (uint32_t) b->size); abort(); // can't do much else at this point } @@ -856,10 +892,13 @@ static inline void mmap_buf(struct htp_context *ctx, struct htp_buf_desc *b) { m->fd = b->fd; m->size = b->size; - FARF(HIGH, "mmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size); + FARF(ALWAYS, "mmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size); return; } } + + FARF(ERROR, "mmap failed : exceeded mapping capacity limit of %u", HTP_MAX_MMAPS); + abort(); } static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uint32_t n_bufs) { @@ -1081,6 +1120,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r rsp.usecs = batch_prof.usecs; rsp.cycles_start = batch_prof.cycles_start; rsp.cycles_stop = batch_prof.cycles_stop; + rsp.seq = req->seq; if (ctx->profiler == HTP_PROF_TRACE) { for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c index 9d385469a..a6adc0e61 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c @@ -64,6 +64,7 @@ typedef struct { struct htp_mm_context { const char * type; struct htp_ops_context * octx; + const struct htp_tensor * act; void (*vec_dot_1x1)(const uint32_t n, float * restrict s0, const void * restrict vx0, @@ -478,7 +479,7 @@ static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows, NULL); \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ \ if (push_ct < ct_end) { \ dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile, src0_row + push_ct * tile_row_stride), \ @@ -502,150 +503,67 @@ static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void } \ } -#define MATMUL_QKV_2D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1) \ -static void hvx_mm_qkv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ +#define MATMUL_NX_2D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1) \ +static void hvx_mm_nx_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ struct htp_mm_context * mmctx = data; \ struct htp_ops_context * octx = mmctx->octx; \ + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ + const uint32_t n_weights = kparams->n_weights; \ \ - const struct htp_tensor * restrict src0 = octx->src[0]; /* Wk */ \ - const struct htp_tensor * restrict src1 = octx->src[1]; /* x */ \ - const struct htp_tensor * restrict src2 = octx->src[2]; /* Wv */ \ - const struct htp_tensor * restrict src3 = octx->src[3]; /* Wq */ \ - const struct htp_tensor * restrict dst_k = octx->dsts[0]; \ - const struct htp_tensor * restrict dst_v = octx->dsts[1]; \ - const struct htp_tensor * restrict dst_q = octx->dsts[2]; \ - \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne10 = src1->ne[0]; \ - const uint32_t src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; \ - \ - const size_t dst_k_row_size = dst_k->nb[1]; /* K and V share output width */ \ - const size_t dst_q_row_size = dst_q->nb[1]; /* Q may be wider (GQA) */ \ + const struct htp_tensor * restrict act = octx->src[n_weights]; /* x */ \ + const uint32_t ne10 = act->ne[0]; \ + const uint32_t src1_nrows = act->ne[1] * act->ne[2] * act->ne[3]; \ const size_t src1_stride = mmctx->vtcm_src1_stride; \ \ - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \ - uint8_t * restrict vtcm_src2_ptr = mmctx->vtcm_src2 + mmctx->vtcm_src2_size_per_thread * ith; \ - uint8_t * restrict vtcm_src3_ptr = mmctx->vtcm_src3 + mmctx->vtcm_src3_size_per_thread * ith; \ - uint8_t * restrict src1_data = mmctx->vtcm_src1; \ + uint8_t * restrict vtcm_weight_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \ + uint8_t * restrict src1_data = mmctx->vtcm_src1; \ \ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - \ - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ const uint32_t n_prefetch = kparams->n_prefetch; \ assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); \ \ - const uint8_t * restrict src0_row = (const uint8_t *) src0->data; \ - const uint8_t * restrict src2_row = (const uint8_t *) src2->data; \ - const uint8_t * restrict src3_row = (const uint8_t *) src3->data; \ - \ const uint32_t tile_size = TILE_SIZE; \ const uint32_t aligned_tile_size = hex_align_up(tile_size, 128); \ - \ - uint32_t n_k_tiles_w = ne00 / 32; \ uint32_t n_k_tiles_a = ne10 / 32; \ - uint32_t tile_row_stride = n_k_tiles_w * tile_size; \ uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; \ \ dma_queue * dma_queue = octx->ctx->dma[ith]; \ \ - /* 1. Process K and V together */ \ - const uint32_t src0_nrows_kv = src0->ne[1] * src0->ne[2] * src0->ne[3]; /* src0 is Wk */ \ - uint32_t src0_nrows_per_thread_kv = (src0_nrows_kv + nth - 1) / nth; \ - src0_nrows_per_thread_kv = hex_round_up(src0_nrows_per_thread_kv, 32); \ - \ - const uint32_t start_row_kv = src0_nrows_per_thread_kv * ith; \ - const uint32_t end_row_kv = MIN(start_row_kv + src0_nrows_per_thread_kv, src0_nrows_kv); \ - \ - uint32_t ct_start_kv = start_row_kv / 32; \ - uint32_t ct_end_kv = (end_row_kv + 31) / 32; \ - \ - uint32_t push_ct = ct_start_kv; \ - if (start_row_kv < end_row_kv) { \ - for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end_kv; d++, push_ct++) { \ - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ - src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + d * tile_row_transfer_size_aligned, \ - src2_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - } \ - } \ - \ hvx_mm_run_quant_task(mmctx, ith); \ \ - if (start_row_kv < end_row_kv) { \ + for (uint32_t widx = 0; widx < n_weights; widx++) { \ + const struct htp_tensor * restrict src_w = octx->src[widx]; \ + const struct htp_tensor * restrict dst = octx->dsts[widx]; \ + if (!src_w || !dst) continue; \ \ - for (uint32_t ct = ct_start_kv; ct < ct_end_kv; ct++) { \ - const uint8_t * w_tile_k = dma_queue_pop(dma_queue).dst; \ - const uint8_t * w_tile_v = dma_queue_pop(dma_queue).dst; \ + const uint32_t ne00 = src_w->ne[0]; \ + const uint32_t ne01 = src_w->ne[1]; \ + const size_t dst_row_size = dst->nb[1]; \ + const uint8_t * restrict src_w_row = (const uint8_t *) src_w->data; \ \ - int valid_rows = (int)src0->ne[1] - (int)(ct * 32); \ - valid_rows = MIN(32, MAX(0, valid_rows)); \ + uint32_t n_k_tiles_w = ne00 / 32; \ + uint32_t tile_row_stride = n_k_tiles_w * tile_size; \ \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ith); \ - uint32_t ir1 = 0; \ - for (; ir1 + 1 < src1_nrows; ir1 += 2) { \ - const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); \ - const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); \ + const uint32_t src0_nrows = ne01 * src_w->ne[2] * src_w->ne[3]; \ + uint32_t src0_nrows_per_thread = (src0_nrows + nth - 1) / nth; \ + src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); \ \ - float * restrict dst_row0_k = (float *) (dst_k->data + ((ir1+0) * dst_k_row_size)); \ - float * restrict dst_row1_k = (float *) (dst_k->data + ((ir1+1) * dst_k_row_size)); \ - float * dst_ptr0_k = &dst_row0_k[ct * 32]; \ - float * dst_ptr1_k = &dst_row1_k[ct * 32]; \ + const uint32_t start_row = src0_nrows_per_thread * ith; \ + const uint32_t end_row = MIN(start_row + src0_nrows_per_thread, src0_nrows); \ + if (start_row >= end_row) continue; \ \ - float * restrict dst_row0_v = (float *) (dst_v->data + ((ir1+0) * dst_k_row_size)); \ - float * restrict dst_row1_v = (float *) (dst_v->data + ((ir1+1) * dst_k_row_size)); \ - float * dst_ptr0_v = &dst_row0_v[ct * 32]; \ - float * dst_ptr1_v = &dst_row1_v[ct * 32]; \ + uint32_t ct_start = start_row / 32; \ + uint32_t ct_end = (end_row + 31) / 32; \ \ - DOT_2X2(ne10, dst_ptr0_k, dst_ptr1_k, w_tile_k, src1_col0, src1_col1, valid_rows, NULL, NULL); \ - DOT_2X2(ne10, dst_ptr0_v, dst_ptr1_v, w_tile_v, src1_col0, src1_col1, valid_rows, NULL, NULL); \ - } \ - \ - for (; ir1 < src1_nrows; ++ir1) { \ - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); \ - \ - float * restrict dst_row_k = (float *) (dst_k->data + (ir1 * dst_k_row_size)); \ - float * dst_ptr_k = &dst_row_k[ct * 32]; \ - \ - float * restrict dst_row_v = (float *) (dst_v->data + (ir1 * dst_k_row_size)); \ - float * dst_ptr_v = &dst_row_v[ct * 32]; \ - \ - DOT_2X1(ne10, dst_ptr_k, w_tile_k, src1_col, valid_rows, NULL); \ - DOT_2X1(ne10, dst_ptr_v, w_tile_v, src1_col, valid_rows, NULL); \ - } \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ith); \ - \ - if (push_ct < ct_end_kv) { \ - dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile_k, src0_row + push_ct * tile_row_stride), \ - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile_v, src2_row + push_ct * tile_row_stride), \ - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - push_ct++; \ - } \ - } \ - } \ - \ - /* 2. Process Q separately */ \ - const uint32_t src0_nrows_q = src3->ne[1] * src3->ne[2] * src3->ne[3]; /* src3 is Wq */ \ - uint32_t src0_nrows_per_thread_q = (src0_nrows_q + nth - 1) / nth; \ - src0_nrows_per_thread_q = hex_round_up(src0_nrows_per_thread_q, 32); \ - \ - const uint32_t start_row_q = src0_nrows_per_thread_q * ith; \ - const uint32_t end_row_q = MIN(start_row_q + src0_nrows_per_thread_q, src0_nrows_q); \ - \ - if (start_row_q < end_row_q) { \ - uint32_t ct_start_q = start_row_q / 32; \ - uint32_t ct_end_q = (end_row_q + 31) / 32; \ - \ - uint32_t push_ct = ct_start_q; \ - for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end_q; d++, push_ct++) { \ - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src3_ptr + d * tile_row_transfer_size_aligned, \ - src3_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ + uint32_t push_ct = ct_start; \ + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ + dma_queue_push(dma_queue, dma_make_ptr(vtcm_weight_ptr + d * tile_row_transfer_size_aligned, \ + src_w_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ } \ \ - for (uint32_t ct = ct_start_q; ct < ct_end_q; ct++) { \ - const uint8_t * w_tile_q = dma_queue_pop(dma_queue).dst; \ - \ - int valid_rows = (int)src3->ne[1] - (int)(ct * 32); \ + for (uint32_t ct = ct_start; ct < ct_end; ct++) { \ + const uint8_t * w_tile = dma_queue_pop(dma_queue).dst; \ + int valid_rows = (int)ne01 - (int)(ct * 32); \ valid_rows = MIN(32, MAX(0, valid_rows)); \ \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ @@ -654,26 +572,24 @@ static void hvx_mm_qkv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); \ const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); \ \ - float * restrict dst_row0_q = (float *) (dst_q->data + ((ir1+0) * dst_q_row_size)); \ - float * restrict dst_row1_q = (float *) (dst_q->data + ((ir1+1) * dst_q_row_size)); \ - float * dst_ptr0_q = &dst_row0_q[ct * 32]; \ - float * dst_ptr1_q = &dst_row1_q[ct * 32]; \ + float * restrict dst_row0 = (float *) (dst->data + ((ir1+0) * dst_row_size)); \ + float * restrict dst_row1 = (float *) (dst->data + ((ir1+1) * dst_row_size)); \ + float * dst_ptr0 = &dst_row0[ct * 32]; \ + float * dst_ptr1 = &dst_row1[ct * 32]; \ \ - DOT_2X2(ne10, dst_ptr0_q, dst_ptr1_q, w_tile_q, src1_col0, src1_col1, valid_rows, NULL, NULL); \ + DOT_2X2(ne10, dst_ptr0, dst_ptr1, w_tile, src1_col0, src1_col1, valid_rows, NULL, NULL); \ } \ \ for (; ir1 < src1_nrows; ++ir1) { \ const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); \ - \ - float * restrict dst_row_q = (float *) (dst_q->data + (ir1 * dst_q_row_size)); \ - float * dst_ptr_q = &dst_row_q[ct * 32]; \ - \ - DOT_2X1(ne10, dst_ptr_q, w_tile_q, src1_col, valid_rows, NULL); \ + float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); \ + float * dst_ptr = &dst_row[ct * 32]; \ + DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows, NULL); \ } \ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ \ - if (push_ct < ct_end_q) { \ - dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile_q, src3_row + push_ct * tile_row_stride), \ + if (push_ct < ct_end) { \ + dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile, src_w_row + push_ct * tile_row_stride), \ aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ push_ct++; \ } \ @@ -681,121 +597,6 @@ static void hvx_mm_qkv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, } \ } -#define MATMUL_FFN_2D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1) \ -static void hvx_mm_ffn_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_mm_context * mmctx = data; \ - struct htp_ops_context * octx = mmctx->octx; \ - \ - const struct htp_tensor * restrict src0 = octx->src[0]; /* Wgate */ \ - const struct htp_tensor * restrict src1 = octx->src[1]; /* y */ \ - const struct htp_tensor * restrict src2 = octx->src[2]; /* Wup */ \ - const struct htp_tensor * restrict dst_gate = octx->dsts[0]; \ - const struct htp_tensor * restrict dst_up = octx->dsts[1]; \ - \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne10 = src1->ne[0]; \ - const uint32_t src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; \ - \ - const size_t dst_row_size = dst_gate->nb[1]; \ - const size_t src1_stride = mmctx->vtcm_src1_stride; \ - \ - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \ - uint8_t * restrict vtcm_src2_ptr = mmctx->vtcm_src2 + mmctx->vtcm_src2_size_per_thread * ith; \ - uint8_t * restrict src1_data = mmctx->vtcm_src1; \ - \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - \ - const uint8_t * restrict src0_row = (const uint8_t *) src0->data; \ - const uint8_t * restrict src2_row = (const uint8_t *) src2->data; \ - \ - const uint32_t tile_size = TILE_SIZE; \ - const uint32_t aligned_tile_size = hex_align_up(tile_size, 128); \ - \ - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ - const uint32_t n_prefetch = kparams->n_prefetch; \ - assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); \ - \ - uint32_t n_k_tiles_w = ne00 / 32; \ - uint32_t n_k_tiles_a = ne10 / 32; \ - uint32_t tile_row_stride = n_k_tiles_w * tile_size; \ - uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; \ - dma_queue * dma_queue = octx->ctx->dma[ith]; \ - \ - const uint32_t src0_nrows = ne01 * src0->ne[2] * src0->ne[3]; \ - const uint32_t src0_start_row = mmctx->src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + mmctx->src0_nrows_per_thread, src0_nrows); \ - \ - uint32_t ct_start = src0_start_row / 32; \ - uint32_t ct_end = (src0_end_row + 31) / 32; \ - \ - uint32_t push_ct = ct_start; \ - if (src0_start_row < src0_end_row) { \ - for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ - src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + d * tile_row_transfer_size_aligned, \ - src2_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - } \ - } \ - \ - hvx_mm_run_quant_task(mmctx, ith); \ - \ - if (src0_start_row >= src0_end_row) { \ - return; \ - } \ - \ - for (uint32_t ct = ct_start; ct < ct_end; ct++) { \ - const uint8_t * w_tile_gate = dma_queue_pop(dma_queue).dst; \ - const uint8_t * w_tile_up = dma_queue_pop(dma_queue).dst; \ - \ - int valid_rows = (int)ne01 - (int)(ct * 32); \ - valid_rows = MIN(32, MAX(0, valid_rows)); \ - \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ - uint32_t ir1 = 0; \ - for (; ir1 + 1 < src1_nrows; ir1 += 2) { \ - const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); \ - const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); \ - \ - float * restrict dst_row0_gate = (float *) (dst_gate->data + ((ir1+0) * dst_row_size)); \ - float * restrict dst_row1_gate = (float *) (dst_gate->data + ((ir1+1) * dst_row_size)); \ - float * dst_ptr0_gate = &dst_row0_gate[ct * 32]; \ - float * dst_ptr1_gate = &dst_row1_gate[ct * 32]; \ - \ - float * restrict dst_row0_up = (float *) (dst_up->data + ((ir1+0) * dst_row_size)); \ - float * restrict dst_row1_up = (float *) (dst_up->data + ((ir1+1) * dst_row_size)); \ - float * dst_ptr0_up = &dst_row0_up[ct * 32]; \ - float * dst_ptr1_up = &dst_row1_up[ct * 32]; \ - \ - DOT_2X2(ne10, dst_ptr0_gate, dst_ptr1_gate, w_tile_gate, src1_col0, src1_col1, valid_rows, NULL, NULL); \ - DOT_2X2(ne10, dst_ptr0_up, dst_ptr1_up, w_tile_up, src1_col0, src1_col1, valid_rows, NULL, NULL); \ - } \ - \ - for (; ir1 < src1_nrows; ++ir1) { \ - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); \ - \ - float * restrict dst_row_gate = (float *) (dst_gate->data + (ir1 * dst_row_size)); \ - float * dst_ptr_gate = &dst_row_gate[ct * 32]; \ - \ - float * restrict dst_row_up = (float *) (dst_up->data + (ir1 * dst_row_size)); \ - float * dst_ptr_up = &dst_row_up[ct * 32]; \ - \ - DOT_2X1(ne10, dst_ptr_gate, w_tile_gate, src1_col, valid_rows, NULL); \ - DOT_2X1(ne10, dst_ptr_up, w_tile_up, src1_col, valid_rows, NULL); \ - } \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ - \ - if (push_ct < ct_end) { \ - dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile_gate, src0_row + push_ct * tile_row_stride), \ - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile_up, src2_row + push_ct * tile_row_stride), \ - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - push_ct++; \ - } \ - } \ -} - MATMUL_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1) MATMUL_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1) MATMUL_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1) @@ -812,7 +613,7 @@ MATMUL_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x2, flat_vec_dot static void name(unsigned int nth, unsigned int ith, void * data) { \ struct htp_mm_context * mmctx = data; \ struct htp_ops_context * octx = mmctx->octx; \ - const struct htp_tensor * src = octx->src[1]; \ + const struct htp_tensor * src = mmctx->act; \ const uint32_t ne0 = src->ne[0]; \ const uint32_t ne1 = src->ne[1]; \ const uint32_t ne2 = src->ne[2]; \ @@ -854,7 +655,7 @@ static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, vo struct htp_thread_trace * tr = &octx->ctx->trace[ith]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); - const struct htp_tensor * src = octx->src[1]; + const struct htp_tensor * src = mmctx->act; quantize_f32_q8_0_tiled_block_kernel( (const float *) src->data, @@ -878,7 +679,7 @@ static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, vo struct htp_thread_trace * tr = &octx->ctx->trace[ith]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); - const struct htp_tensor * src = octx->src[1]; + const struct htp_tensor * src = mmctx->act; quantize_f32_q8_1_tiled_block_kernel( (const float *) src->data, @@ -909,30 +710,17 @@ MATVEC_2D_REPACKED_IMPL(iq4nl_flat, 576, flat_vec_dot_iq4nl_32x1) MATVEC_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x1) -MATMUL_QKV_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1) -MATMUL_QKV_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1) -MATMUL_QKV_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1) -MATMUL_QKV_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1) -MATMUL_QKV_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1) +MATMUL_NX_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1) +MATMUL_NX_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1) +MATMUL_NX_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1) +MATMUL_NX_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1) +MATMUL_NX_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1) -MATMUL_QKV_2D_REPACKED_IMPL(q4_0_flat, 576, flat_vec_dot_q4_0_32x2, flat_vec_dot_q4_0_32x1) -MATMUL_QKV_2D_REPACKED_IMPL(q4_1_flat, 640, flat_vec_dot_q4_1_32x2, flat_vec_dot_q4_1_32x1) -MATMUL_QKV_2D_REPACKED_IMPL(q8_0_flat, 1088, flat_vec_dot_q8_0_32x2, flat_vec_dot_q8_0_32x1) -MATMUL_QKV_2D_REPACKED_IMPL(iq4nl_flat, 576, flat_vec_dot_iq4nl_32x2, flat_vec_dot_iq4nl_32x1) -MATMUL_QKV_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x2, flat_vec_dot_mxfp4_32x1) - - -MATMUL_FFN_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1) -MATMUL_FFN_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1) -MATMUL_FFN_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1) -MATMUL_FFN_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1) -MATMUL_FFN_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1) - -MATMUL_FFN_2D_REPACKED_IMPL(q4_0_flat, 576, flat_vec_dot_q4_0_32x2, flat_vec_dot_q4_0_32x1) -MATMUL_FFN_2D_REPACKED_IMPL(q4_1_flat, 640, flat_vec_dot_q4_1_32x2, flat_vec_dot_q4_1_32x1) -MATMUL_FFN_2D_REPACKED_IMPL(q8_0_flat, 1088, flat_vec_dot_q8_0_32x2, flat_vec_dot_q8_0_32x1) -MATMUL_FFN_2D_REPACKED_IMPL(iq4nl_flat, 576, flat_vec_dot_iq4nl_32x2, flat_vec_dot_iq4nl_32x1) -MATMUL_FFN_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x2, flat_vec_dot_mxfp4_32x1) +MATMUL_NX_2D_REPACKED_IMPL(q4_0_flat, 576, flat_vec_dot_q4_0_32x2, flat_vec_dot_q4_0_32x1) +MATMUL_NX_2D_REPACKED_IMPL(q4_1_flat, 640, flat_vec_dot_q4_1_32x2, flat_vec_dot_q4_1_32x1) +MATMUL_NX_2D_REPACKED_IMPL(q8_0_flat, 1088, flat_vec_dot_q8_0_32x2, flat_vec_dot_q8_0_32x1) +MATMUL_NX_2D_REPACKED_IMPL(iq4nl_flat, 576, flat_vec_dot_iq4nl_32x2, flat_vec_dot_iq4nl_32x1) +MATMUL_NX_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x2, flat_vec_dot_mxfp4_32x1) static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; @@ -1353,6 +1141,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { struct htp_mm_context mmctx_struct = {0}; struct htp_mm_context * mmctx = &mmctx_struct; mmctx->octx = octx; + mmctx->act = src1; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -1528,7 +1317,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, src1_nrows, octx->n_threads, - dst_row_size, src0_row_size, src1_row_size, src2 ? src2->nb[1] : 0, kparams->n_prefetch, false, false, false); + dst_row_size, src0_row_size, src1_row_size, src2 ? src2->nb[1] : 0, kparams->n_prefetch, false, false); if (kparams->kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || kparams->kernel_type == HTP_MM_KERNEL_HVX_F32_F32_VTCM || @@ -1587,297 +1376,97 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { return HTP_STATUS_OK; } -static void hvx_mm_qkv_2d(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_nx_2d(unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = data; struct htp_ops_context * octx = mmctx->octx; + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_weights = kparams->n_weights; - const struct htp_tensor * restrict src0 = octx->src[0]; // Wk - const struct htp_tensor * restrict src1 = octx->src[1]; // x - const struct htp_tensor * restrict src2 = octx->src[2]; // Wv - const struct htp_tensor * restrict src3 = octx->src[3]; // Wq - const struct htp_tensor * restrict dst_k = octx->dsts[0]; - const struct htp_tensor * restrict dst_v = octx->dsts[1]; - const struct htp_tensor * restrict dst_q = octx->dsts[2]; - - const uint32_t ne00 = src0->ne[0]; - const uint32_t ne01 = src0->ne[1]; - const uint32_t ne02 = src0->ne[2]; - const uint32_t ne03 = src0->ne[3]; - - const uint32_t ne11 = src1->ne[1]; - const uint32_t ne12 = src1->ne[2]; - const uint32_t ne13 = src1->ne[3]; - - const uint32_t src0_nrows = ne01 * ne02 * ne03; - const uint32_t src1_nrows = ne11 * ne12 * ne13; - - const uint32_t src0_nrows_per_thread = mmctx->src0_nrows_per_thread; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - - const size_t dst_k_row_size = dst_k->nb[1]; // K and V share output width - const size_t dst_q_row_size = dst_q->nb[1]; // Q may be wider (GQA) - const size_t src0_row_size = src0->nb[1]; - const size_t src2_row_size = src2->nb[1]; - const size_t src3_row_size = src3->nb[1]; - - const size_t src0_stride = mmctx->vtcm_src0_stride; - const size_t src2_stride = mmctx->vtcm_src2_stride; - const size_t src3_stride = mmctx->vtcm_src3_stride; + const struct htp_tensor * restrict act = octx->src[n_weights]; + const uint32_t src1_nrows = act->ne[1] * act->ne[2] * act->ne[3]; const size_t src1_stride = mmctx->vtcm_src1_stride; uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; - uint8_t * restrict vtcm_src2_ptr = mmctx->vtcm_src2 + mmctx->vtcm_src2_size_per_thread * ith; - uint8_t * restrict vtcm_src3_ptr = mmctx->vtcm_src3 + mmctx->vtcm_src3_size_per_thread * ith; - uint8_t * restrict src1_data = mmctx->vtcm_src1; + uint8_t * restrict src1_data = mmctx->vtcm_src1; dma_queue * dma_queue = octx->ctx->dma[ith]; - - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); const uint32_t prefetch_mask = n_prefetch - 1; - const uint8_t * restrict src0_row = (const uint8_t *) src0->data; - const uint8_t * restrict src2_row = (const uint8_t *) src2->data; - const uint8_t * restrict src3_row = (const uint8_t *) src3->data; - - // Prefill spad with src0, src2, src3 rows - if (src0_start_row < src0_end_row) { - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const int is0 = (ir0 - src0_start_row); - if (is0 >= (int)n_prefetch) { - break; - } - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), - src0_stride, src0_row_size, src0_row_size, 2); - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + ir0 * src2_row_size), - src2_stride, src2_row_size, src2_row_size, 2); - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src3_ptr + is0 * src3_stride, src3_row + ir0 * src3_row_size), - src3_stride, src3_row_size, src3_row_size, 2); - } - } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; hvx_mm_run_quant_task(mmctx, ith); - if (src0_start_row >= src0_end_row) { - return; - } + for (uint32_t widx = 0; widx < n_weights; widx++) { + const struct htp_tensor * restrict src_w = octx->src[widx]; + const struct htp_tensor * restrict dst = octx->dsts[widx]; + if (!src_w || !dst) continue; - // Process rows - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; - const uint8_t * ss2 = dma_queue_pop(dma_queue).dst; - const uint8_t * ss3 = dma_queue_pop(dma_queue).dst; + const uint32_t ne00 = src_w->ne[0]; + const uint32_t ne01 = src_w->ne[1]; + const uint32_t src0_nrows = ne01 * src_w->ne[2] * src_w->ne[3]; - // Process src1 columns in pairs (2×2 tiling) - uint32_t ir1 = 0; - for (; ir1 + 1 < src1_nrows; ir1 += 2) { - const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); - const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); + uint32_t src0_nrows_per_thread = (src0_nrows + nth - 1) / nth; + src0_nrows_per_thread += (src0_nrows_per_thread & 1); - float * restrict dst_row0_k = (float *) (dst_k->data + ((ir1+0) * dst_k_row_size)); - float * restrict dst_row1_k = (float *) (dst_k->data + ((ir1+1) * dst_k_row_size)); - mmctx->vec_dot_2x2(ne00, &dst_row0_k[ir0], &dst_row1_k[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1); + const uint32_t src0_start_row = src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); + if (src0_start_row >= src0_end_row) continue; - float * restrict dst_row0_v = (float *) (dst_v->data + ((ir1+0) * dst_k_row_size)); - float * restrict dst_row1_v = (float *) (dst_v->data + ((ir1+1) * dst_k_row_size)); - mmctx->vec_dot_2x2(ne00, &dst_row0_v[ir0], &dst_row1_v[ir0], ss2, ss2 + src2_stride, src1_col0, src1_col1); + const size_t dst_row_size = dst->nb[1]; + const size_t src0_row_size = src_w->nb[1]; + const size_t src0_stride = hex_round_up(src0_row_size, 128); - float * restrict dst_row0_q = (float *) (dst_q->data + ((ir1+0) * dst_q_row_size)); - float * restrict dst_row1_q = (float *) (dst_q->data + ((ir1+1) * dst_q_row_size)); - mmctx->vec_dot_2x2(ne00, &dst_row0_q[ir0], &dst_row1_q[ir0], ss3, ss3 + src3_stride, src1_col0, src1_col1); - } + const uint8_t * restrict src0_row = (const uint8_t *) src_w->data; - // Handle remaining src1 rows (fallback to 2×1) - for (; ir1 < src1_nrows; ++ir1) { - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); - - float * restrict dst_row_k = (float *) (dst_k->data + (ir1 * dst_k_row_size)); - mmctx->vec_dot_2x1(ne00, &dst_row_k[ir0], ss0, ss0 + src0_stride, src1_col); - - float * restrict dst_row_v = (float *) (dst_v->data + (ir1 * dst_k_row_size)); - mmctx->vec_dot_2x1(ne00, &dst_row_v[ir0], ss2, ss2 + src2_stride, src1_col); - - float * restrict dst_row_q = (float *) (dst_q->data + (ir1 * dst_q_row_size)); - mmctx->vec_dot_2x1(ne00, &dst_row_q[ir0], ss3, ss3 + src3_stride, src1_col); - } - - // Prefetch next (n + vtcm_nrows) rows - const int pr0 = (ir0 + n_prefetch); - const int is0 = (pr0 - src0_start_row) & prefetch_mask; - if (pr0 < src0_end_row_x2) { - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + pr0 * src0_row_size), - src0_stride, src0_row_size, src0_row_size, 2); - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + pr0 * src2_row_size), - src2_stride, src2_row_size, src2_row_size, 2); - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src3_ptr + is0 * src3_stride, src3_row + pr0 * src3_row_size), - src3_stride, src3_row_size, src3_row_size, 2); - } - } - - // Process last row (if any) - if (src0_end_row != src0_end_row_x2) { - uint32_t ir0 = src0_end_row_x2; - const int is0 = (ir0 - src0_start_row) & prefetch_mask; - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), - src0_stride, src0_row_size, src0_row_size, 1); - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + ir0 * src2_row_size), - src2_stride, src2_row_size, src2_row_size, 1); - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src3_ptr + is0 * src3_stride, src3_row + ir0 * src3_row_size), - src3_stride, src3_row_size, src3_row_size, 1); - - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; - const uint8_t * ss2 = dma_queue_pop(dma_queue).dst; - const uint8_t * ss3 = dma_queue_pop(dma_queue).dst; - - for (uint32_t ir1 = 0; ir1 < src1_nrows; ++ir1) { - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); - - float * restrict dst_row_k = (float *) (dst_k->data + (ir1 * dst_k_row_size)); - mmctx->vec_dot_1x1(ne00, &dst_row_k[ir0], ss0, src1_col); - - float * restrict dst_row_v = (float *) (dst_v->data + (ir1 * dst_k_row_size)); - mmctx->vec_dot_1x1(ne00, &dst_row_v[ir0], ss2, src1_col); - - float * restrict dst_row_q = (float *) (dst_q->data + (ir1 * dst_q_row_size)); - mmctx->vec_dot_1x1(ne00, &dst_row_q[ir0], ss3, src1_col); - } - } -} - -static void hvx_mm_ffn_2d(unsigned int nth, unsigned int ith, void * data) { - struct htp_mm_context * mmctx = data; - struct htp_ops_context * octx = mmctx->octx; - - const struct htp_tensor * restrict src0 = octx->src[0]; // Wgate - const struct htp_tensor * restrict src1 = octx->src[1]; // y - const struct htp_tensor * restrict src2 = octx->src[2]; // Wup - const struct htp_tensor * restrict dst_gate = octx->dsts[0]; - const struct htp_tensor * restrict dst_up = octx->dsts[1]; - - const uint32_t ne00 = src0->ne[0]; - const uint32_t ne01 = src0->ne[1]; - const uint32_t ne02 = src0->ne[2]; - const uint32_t ne03 = src0->ne[3]; - - const uint32_t ne11 = src1->ne[1]; - const uint32_t ne12 = src1->ne[2]; - const uint32_t ne13 = src1->ne[3]; - - const uint32_t src0_nrows = ne01 * ne02 * ne03; - const uint32_t src1_nrows = ne11 * ne12 * ne13; - - const uint32_t src0_nrows_per_thread = mmctx->src0_nrows_per_thread; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - - const size_t dst_row_size = dst_gate->nb[1]; - const size_t src0_row_size = src0->nb[1]; - const size_t src2_row_size = src2->nb[1]; - - const size_t src0_stride = mmctx->vtcm_src0_stride; - const size_t src2_stride = mmctx->vtcm_src2_stride; - const size_t src1_stride = mmctx->vtcm_src1_stride; - - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; - uint8_t * restrict vtcm_src2_ptr = mmctx->vtcm_src2 + mmctx->vtcm_src2_size_per_thread * ith; - uint8_t * restrict src1_data = mmctx->vtcm_src1; - - dma_queue * dma_queue = octx->ctx->dma[ith]; - - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - const uint32_t n_prefetch = kparams->n_prefetch; - assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); - const uint32_t prefetch_mask = n_prefetch - 1; - - const uint8_t * restrict src0_row = (const uint8_t *) src0->data; - const uint8_t * restrict src2_row = (const uint8_t *) src2->data; - - // Prefill spad with src0, src2 rows - if (src0_start_row < src0_end_row) { for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { const int is0 = (ir0 - src0_start_row); - if (is0 >= (int)n_prefetch) { - break; - } + if (is0 >= (int)n_prefetch) break; dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), src0_stride, src0_row_size, src0_row_size, 2); - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + ir0 * src2_row_size), - src2_stride, src2_row_size, src2_row_size, 2); - } - } - - hvx_mm_run_quant_task(mmctx, ith); - - if (src0_start_row >= src0_end_row) { - return; - } - - // Process rows - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; - const uint8_t * ss2 = dma_queue_pop(dma_queue).dst; - - // Process src1 columns in pairs (2×2 tiling) - uint32_t ir1 = 0; - for (; ir1 + 1 < src1_nrows; ir1 += 2) { - const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); - const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); - - float * restrict dst_row0_gate = (float *) (dst_gate->data + ((ir1+0) * dst_row_size)); - float * restrict dst_row1_gate = (float *) (dst_gate->data + ((ir1+1) * dst_row_size)); - mmctx->vec_dot_2x2(ne00, &dst_row0_gate[ir0], &dst_row1_gate[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1); - - float * restrict dst_row0_up = (float *) (dst_up->data + ((ir1+0) * dst_row_size)); - float * restrict dst_row1_up = (float *) (dst_up->data + ((ir1+1) * dst_row_size)); - mmctx->vec_dot_2x2(ne00, &dst_row0_up[ir0], &dst_row1_up[ir0], ss2, ss2 + src2_stride, src1_col0, src1_col1); } - // Handle remaining src1 rows (fallback to 2×1) - for (; ir1 < src1_nrows; ++ir1) { - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); + for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { + const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); + uint32_t ir1 = 0; + for (; ir1 + 1 < src1_nrows; ir1 += 2) { + const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); + const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); + float * restrict dst_row0 = (float *) (dst->data + ((ir1+0) * dst_row_size)); + float * restrict dst_row1 = (float *) (dst->data + ((ir1+1) * dst_row_size)); + mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1); + } + for (; ir1 < src1_nrows; ++ir1) { + const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); + float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); + mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_stride, src1_col); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); - float * restrict dst_row_gate = (float *) (dst_gate->data + (ir1 * dst_row_size)); - mmctx->vec_dot_2x1(ne00, &dst_row_gate[ir0], ss0, ss0 + src0_stride, src1_col); - - float * restrict dst_row_up = (float *) (dst_up->data + (ir1 * dst_row_size)); - mmctx->vec_dot_2x1(ne00, &dst_row_up[ir0], ss2, ss2 + src2_stride, src1_col); + const int pr0 = (ir0 + n_prefetch); + const int is0 = (pr0 - src0_start_row) & prefetch_mask; + if (pr0 < src0_end_row_x2) { + dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + pr0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 2); + } } - // Prefetch next rows - const int pr0 = (ir0 + n_prefetch); - const int is0 = (pr0 - src0_start_row) & prefetch_mask; - if (pr0 < src0_end_row_x2) { - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + pr0 * src0_row_size), - src0_stride, src0_row_size, src0_row_size, 2); - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + pr0 * src2_row_size), - src2_stride, src2_row_size, src2_row_size, 2); - } - } - - // Process last row (if any) - if (src0_end_row != src0_end_row_x2) { - uint32_t ir0 = src0_end_row_x2; - const int is0 = (ir0 - src0_start_row) & prefetch_mask; - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), - src0_stride, src0_row_size, src0_row_size, 1); - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + ir0 * src2_row_size), - src2_stride, src2_row_size, src2_row_size, 1); - - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; - const uint8_t * ss2 = dma_queue_pop(dma_queue).dst; - - for (uint32_t ir1 = 0; ir1 < src1_nrows; ++ir1) { - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); - - float * restrict dst_row_gate = (float *) (dst_gate->data + (ir1 * dst_row_size)); - mmctx->vec_dot_1x1(ne00, &dst_row_gate[ir0], ss0, src1_col); - - float * restrict dst_row_up = (float *) (dst_up->data + (ir1 * dst_row_size)); - mmctx->vec_dot_1x1(ne00, &dst_row_up[ir0], ss2, src1_col); + if (src0_end_row != src0_end_row_x2) { + uint32_t ir0 = src0_end_row_x2; + const int is0 = (ir0 - src0_start_row) & prefetch_mask; + dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 1); + const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); + for (uint32_t ir1 = 0; ir1 < src1_nrows; ++ir1) { + const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); + float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); + mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, src1_col); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); } } } @@ -3485,7 +3074,7 @@ static int hvx_mm_matmul_id( struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, src1_nrows, octx->n_threads, - 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, true, false, false); + 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, true, false); size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; @@ -3610,6 +3199,7 @@ int op_matmul_id(struct htp_ops_context * octx) { struct htp_mm_context mmctx_struct = {0}; struct htp_mm_context * mmctx = &mmctx_struct; mmctx->octx = octx; + mmctx->act = src1; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -3690,18 +3280,15 @@ int op_matmul_id(struct htp_ops_context * octx) { return s; } - -int op_matmul_qkv(struct htp_ops_context * octx) { +int op_matmul_nx(struct htp_ops_context * octx) { struct htp_thread_trace * tr = &octx->ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - const struct htp_tensor * restrict src0 = octx->src[0]; // Wk - const struct htp_tensor * restrict src1 = octx->src[1]; // x - const struct htp_tensor * restrict src2 = octx->src[2]; // Wv - const struct htp_tensor * restrict src3 = octx->src[3]; // Wq - const struct htp_tensor * restrict dst_k = octx->dsts[0]; - const struct htp_tensor * restrict dst_v = octx->dsts[1]; - const struct htp_tensor * restrict dst_q = octx->dsts[2]; + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_weights = kparams->n_weights; + + const struct htp_tensor * restrict src0 = octx->src[0]; // first weight + const struct htp_tensor * restrict act = octx->src[n_weights]; // activation x bool is_repacked = (src0->type == HTP_TYPE_Q4_0 || src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q8_0 || src0->type == HTP_TYPE_IQ4_NL || @@ -3710,19 +3297,9 @@ int op_matmul_qkv(struct htp_ops_context * octx) { struct htp_mm_context mmctx_struct = {0}; struct htp_mm_context * mmctx = &mmctx_struct; mmctx->octx = octx; + mmctx->act = act; - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; - - // Compute src0_nrows_per_thread - mmctx->src0_nrows_per_thread = (src0_nrows + octx->n_threads - 1) / octx->n_threads; - if (is_repacked) { - mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); - } else { - mmctx->src0_nrows_per_thread += (mmctx->src0_nrows_per_thread & 1); // round up to even - } + const uint32_t src1_nrows = act->ne[1] * act->ne[2] * act->ne[3]; const size_t src0_row_size = src0->nb[1]; const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); @@ -3732,7 +3309,7 @@ int op_matmul_qkv(struct htp_ops_context * octx) { } const uint32_t qk = QK_Q8_0_TILED; - const uint32_t nb = (src1->ne[0] + qk - 1) / qk; + const uint32_t nb = (act->ne[0] + qk - 1) / qk; const uint32_t total_nb = src1_nrows * nb; worker_callback_t quant_task_func; @@ -3758,45 +3335,39 @@ int op_matmul_qkv(struct htp_ops_context * octx) { size_t src1_row_size; if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) { - src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(src1->ne[0]) : htp_mm_q8_0_flat_row_size(src1->ne[0]); + src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(act->ne[0]) : htp_mm_q8_0_flat_row_size(act->ne[0]); } else { - src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(src1->ne[0]) : htp_mm_q8_0_tiled_row_size(src1->ne[0]); + src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(act->ne[0]) : htp_mm_q8_0_tiled_row_size(act->ne[0]); } struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, src1->ne[0], src1_nrows, octx->n_threads, - 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, false, true, false); + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], src1_nrows, octx->n_threads, + 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, false, true); size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; if (octx->ctx->vtcm_size < vtcm_size) { - FARF(ERROR, "matmul-qkv: current VTCM reservation %zu is too small, needed %zu\n", + FARF(ERROR, "matmul-nx: current VTCM reservation %zu is too small, needed %zu\n", octx->ctx->vtcm_size, vtcm_size); return HTP_STATUS_VTCM_TOO_SMALL; } uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; - mmctx->vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); - mmctx->vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src2); - mmctx->vtcm_src3 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src3); + mmctx->vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); - octx->src1_spad.src = NULL; octx->src0_spad.src = NULL; + octx->src1_spad.src = NULL; octx->src2_spad.src = NULL; octx->src3_spad.src = NULL; octx->dst_spad.src = NULL; mmctx->vtcm_src0_stride = is_repacked ? 0 : src0_row_size_padded; - mmctx->vtcm_src2_stride = is_repacked ? 0 : src0_row_size_padded; - mmctx->vtcm_src3_stride = is_repacked ? 0 : src0_row_size_padded; mmctx->vtcm_src1_stride = src1_row_size; mmctx->vtcm_src0_size_per_thread = L.src0_bytes / octx->n_threads; mmctx->vtcm_src1_size_per_thread = L.src1_bytes; - mmctx->vtcm_src2_size_per_thread = L.src2_bytes / octx->n_threads; - mmctx->vtcm_src3_size_per_thread = L.src3_bytes / octx->n_threads; mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads; mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; @@ -3810,165 +3381,25 @@ int op_matmul_qkv(struct htp_ops_context * octx) { if (is_repacked) { if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) { switch (src0->type) { - case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_qkv_2d_repacked_q4_0_flat; break; - case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_qkv_2d_repacked_q4_1_flat; break; - case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_qkv_2d_repacked_q8_0_flat; break; - case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_qkv_2d_repacked_iq4nl_flat; break; - case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_qkv_2d_repacked_mxfp4_flat; break; + case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_nx_2d_repacked_q4_0_flat; break; + case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1_flat; break; + case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_nx_2d_repacked_q8_0_flat; break; + case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_nx_2d_repacked_iq4nl_flat; break; + case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_nx_2d_repacked_mxfp4_flat; break; default: return HTP_STATUS_NO_SUPPORT; } } else { switch (src0->type) { - case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_qkv_2d_repacked_q4_0; break; - case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_qkv_2d_repacked_q4_1; break; - case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_qkv_2d_repacked_q8_0; break; - case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_qkv_2d_repacked_iq4nl; break; - case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_qkv_2d_repacked_mxfp4; break; + case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_nx_2d_repacked_q4_0; break; + case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1; break; + case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_nx_2d_repacked_q8_0; break; + case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_nx_2d_repacked_iq4nl; break; + case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_nx_2d_repacked_mxfp4; break; default: return HTP_STATUS_NO_SUPPORT; } } } else { - matmul_job_func = hvx_mm_qkv_2d; - } - - htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - - worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs); - - return HTP_STATUS_OK; -} - -int op_matmul_ffn(struct htp_ops_context * octx) { - struct htp_thread_trace * tr = &octx->ctx->trace[0]; - htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - - const struct htp_tensor * restrict src0 = octx->src[0]; // Wgate - const struct htp_tensor * restrict src1 = octx->src[1]; // y - const struct htp_tensor * restrict src2 = octx->src[2]; // Wup - const struct htp_tensor * restrict dst_gate = octx->dsts[0]; - const struct htp_tensor * restrict dst_up = octx->dsts[1]; - - bool is_repacked = (src0->type == HTP_TYPE_Q4_0 || src0->type == HTP_TYPE_Q4_1 || - src0->type == HTP_TYPE_Q8_0 || src0->type == HTP_TYPE_IQ4_NL || - src0->type == HTP_TYPE_MXFP4); - - struct htp_mm_context mmctx_struct = {0}; - struct htp_mm_context * mmctx = &mmctx_struct; - mmctx->octx = octx; - - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; - - // Compute src0_nrows_per_thread - mmctx->src0_nrows_per_thread = (src0_nrows + octx->n_threads - 1) / octx->n_threads; - if (is_repacked) { - mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); - } else { - mmctx->src0_nrows_per_thread += (mmctx->src0_nrows_per_thread & 1); // round up to even - } - - const size_t src0_row_size = src0->nb[1]; - const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); - - if (hvx_mm_init_vec_dot(mmctx, src0->type) != 0) { - return HTP_STATUS_NO_SUPPORT; - } - - const uint32_t qk = QK_Q8_0_TILED; - const uint32_t nb = (src1->ne[0] + qk - 1) / qk; - const uint32_t total_nb = src1_nrows * nb; - - worker_callback_t quant_task_func; - uint32_t n_quant_tasks = 1; - if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) { - n_quant_tasks = MIN(src1_nrows, octx->n_threads); - quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat; - } else if (src1_nrows < octx->n_threads) { - n_quant_tasks = MIN(total_nb, octx->n_threads); - quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; - for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { - uint32_t ib_first = (total_nb * (ith + 0)) / n_quant_tasks; - uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; - mmctx->quant_ib_first[ith] = ib_first; - mmctx->quant_ib_last[ith] = ib_last; - mmctx->quant_r[ith] = ib_first / nb; - mmctx->quant_c[ith] = ib_first % nb; - } - } else { - n_quant_tasks = MIN(src1_nrows, octx->n_threads); - quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; - } - - size_t src1_row_size; - if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) { - src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(src1->ne[0]) : htp_mm_q8_0_flat_row_size(src1->ne[0]); - } else { - src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(src1->ne[0]) : htp_mm_q8_0_tiled_row_size(src1->ne[0]); - } - - struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, src1->ne[0], src1_nrows, octx->n_threads, - 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, false, false, true); - - size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; - - if (octx->ctx->vtcm_size < vtcm_size) { - FARF(ERROR, "matmul-ffn: current VTCM reservation %zu is too small, needed %zu\n", octx->ctx->vtcm_size, vtcm_size); - return HTP_STATUS_VTCM_TOO_SMALL; - } - - uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; - mmctx->vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); - mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); - mmctx->vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src2); - mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); - - octx->src1_spad.src = NULL; - octx->src0_spad.src = NULL; - octx->src2_spad.src = NULL; - octx->dst_spad.src = NULL; - - mmctx->vtcm_src0_stride = is_repacked ? 0 : src0_row_size_padded; - mmctx->vtcm_src2_stride = is_repacked ? 0 : src0_row_size_padded; - mmctx->vtcm_src1_stride = src1_row_size; - - mmctx->vtcm_src0_size_per_thread = L.src0_bytes / octx->n_threads; - mmctx->vtcm_src1_size_per_thread = L.src1_bytes; - mmctx->vtcm_src2_size_per_thread = L.src2_bytes / octx->n_threads; - mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads; - - mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; - mmctx->quant_task_func = quant_task_func; - mmctx->n_quant_tasks = n_quant_tasks; - atomic_init(&mmctx->quant_barrier, n_quant_tasks); - - // Run fused matmul - const uint32_t n_matmul_jobs = octx->n_threads; - worker_callback_t matmul_job_func; - if (is_repacked) { - if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) { - switch (src0->type) { - case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_ffn_2d_repacked_q4_0_flat; break; - case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_ffn_2d_repacked_q4_1_flat; break; - case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_ffn_2d_repacked_q8_0_flat; break; - case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_ffn_2d_repacked_iq4nl_flat; break; - case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_ffn_2d_repacked_mxfp4_flat; break; - default: return HTP_STATUS_NO_SUPPORT; - } - } else { - switch (src0->type) { - case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_ffn_2d_repacked_q4_0; break; - case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_ffn_2d_repacked_q4_1; break; - case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_ffn_2d_repacked_q8_0; break; - case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_ffn_2d_repacked_iq4nl; break; - case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_ffn_2d_repacked_mxfp4; break; - default: return HTP_STATUS_NO_SUPPORT; - } - } - } else { - matmul_job_func = hvx_mm_ffn_2d; + matmul_job_func = hvx_mm_nx_2d; } htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h index 6c393664c..dbc8e3590 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.h +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h @@ -88,6 +88,7 @@ struct htp_mm_kernel_params { int32_t vtcm_src2_size; // src2 scratchpad size in VTCM (fused only) int32_t vtcm_src3_size; // src3 scratchpad size in VTCM (fused only) int32_t vtcm_dst_size; // dst scratchpad size in VTCM + int32_t n_weights; // Number of weights for fused NX // Precomputed division values struct fastdiv_values div_ne12_ne1; @@ -463,8 +464,7 @@ static inline void htp_mm_hvx_vtcm_layout_build( size_t src2_row_size, uint32_t n_prefetch, bool is_matmul_id, - bool is_fused_qkv, - bool is_fused_ffn + bool is_fused_nx ) { size_t src0_sz = 0; size_t src1_sz = 0; @@ -476,44 +476,33 @@ static inline void htp_mm_hvx_vtcm_layout_build( wtype == HTP_TYPE_Q8_0 || wtype == HTP_TYPE_IQ4_NL || wtype == HTP_TYPE_MXFP4); - if (is_fused_qkv || is_fused_ffn) { + if (is_fused_nx) { const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); const size_t quant_scratch_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)) * n_threads; - size_t src0_sz_per_thread = 0; - size_t src2_sz_per_thread = 0; - size_t src3_sz_per_thread = 0; + size_t weight_sz_per_thread = 0; if (is_repack) { uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); uint32_t n_k_tiles = hex_round_up(ne10, 32) / 32; uint32_t tile_row_size = n_k_tiles * aligned_tile_size; - src0_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128); - src2_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128); - if (is_fused_qkv) { - src3_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128); - } + weight_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128); } else { - src0_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128); - src2_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128); - if (is_fused_qkv) { - src3_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128); - } + weight_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128); } - size_t flat_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); - size_t tiled_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); + size_t flat_act_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); + size_t tiled_act_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); - if (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) { - src1_sz = hex_round_up(flat_src1_row_size * src1_nrows, 128); - } else { - src1_sz = hex_round_up(tiled_src1_row_size * src1_nrows, 128); - } + size_t act_sz = (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) + ? hex_round_up(flat_act_row_size * src1_nrows, 128) + : hex_round_up(tiled_act_row_size * src1_nrows, 128); - src0_sz = src0_sz_per_thread * n_threads; - src2_sz = src2_sz_per_thread * n_threads; - src3_sz = src3_sz_per_thread * n_threads; + src0_sz = weight_sz_per_thread * n_threads; // shared single-weight prefetch buffer + src1_sz = act_sz; // quantized activation buffer + src2_sz = 0; + src3_sz = 0; dst_sz = quant_scratch_size; } else if (is_matmul_id) { const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128); @@ -616,8 +605,8 @@ static inline void htp_mm_hvx_vtcm_layout_build( } size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src1, src1_sz); VTCM_LAYOUT_ALLOC(off, off_src0, src0_sz); + VTCM_LAYOUT_ALLOC(off, off_src1, src1_sz); VTCM_LAYOUT_ALLOC(off, off_src2, src2_sz); VTCM_LAYOUT_ALLOC(off, off_src3, src3_sz); VTCM_LAYOUT_ALLOC(off, off_dst, dst_sz); diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.c b/ggml/src/ggml-hexagon/htp/set-rows-ops.c index 58c54967d..fa14bf0ef 100644 --- a/ggml/src/ggml-hexagon/htp/set-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.c @@ -8,14 +8,20 @@ #include #include -#include "hex-dma.h" +#include "dma-queue.h" +#include "work-queue.h" #include "hvx-utils.h" +#include "hex-utils.h" +#include "hvx-copy.h" +#include "hvx-quant.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" + #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" +#include "htp/set-rows-ops.h" #define set_rows_preamble \ const uint32_t ne00 = octx->src[0]->ne[0]; \ @@ -47,116 +53,142 @@ \ const uint32_t nr = ne01; -struct htp_set_rows_context { +struct set_rows_context { struct htp_ops_context * octx; - struct fastdiv_values div_ne12; - struct fastdiv_values div_ne11; - uint32_t src0_nrows_per_thread; + const struct htp_set_rows_kernel_params * kparams; + struct htp_set_rows_vtcm_layout vtcm_layout; + uint8_t * vtcm_base; }; -static void set_rows_thread_f32_f32(unsigned int nth, unsigned int ith, void *data) { - struct htp_set_rows_context * srctx = (struct htp_set_rows_context *)data; - struct htp_ops_context * octx = srctx->octx; - - set_rows_preamble; - - uint64_t qt = HAP_perf_get_qtimer_count(); - - // parallelize by rows of src0 - const uint32_t dr = srctx->src0_nrows_per_thread; - const uint32_t ir0 = dr * ith; - if (ir0 >= nr) { - return; - } - const uint32_t ir1 = (ir0 + dr < nr) ? (ir0 + dr) : nr; - - const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32); - - for (uint32_t i03 = 0; i03 < ne03; ++i03) { - for (uint32_t i02 = 0; i02 < ne02; ++i02) { - for (uint32_t i = ir0; i < ir1; ++i) { - const uint32_t i12 = fastmodulo(i03, ne12, &srctx->div_ne12); - const uint32_t i11 = fastmodulo(i02, ne11, &srctx->div_ne11); - const uint32_t i10 = i; - - const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12; - - uint32_t i1 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr; - if (i1 >= ne1) { - // ignore invalid indices - continue; - } - - const uintptr_t src0_ptr = octx->src[0]->data + i*nb01 + i02*nb02 + i03*nb03; - const uintptr_t dst_ptr = octx->dst->data + i1*nb1 + i02*nb2 + i03*nb3; - - // copy row - hvx_copy_f32_uu((uint8_t *)dst_ptr, (const uint8_t *)src0_ptr, ne00); - } - } - } - - qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt); - FARF(HIGH, "set-rows-f32-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, - ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt); +#define SET_ROWS_THREAD_DMA_FN(TYPE_NAME, IDX_TYPE, COMPUTE_EXPR) \ +static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ + struct set_rows_context * srctx = (struct set_rows_context *)data; \ + struct htp_ops_context * octx = srctx->octx; \ + const struct htp_set_rows_kernel_params * kparams = srctx->kparams; \ + set_rows_preamble; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + const uint32_t dr = kparams->tasks_per_thread; \ + const uint32_t ir0 = dr * ith; \ + if (ir0 >= kparams->total_tasks) { \ + return; \ + } \ + const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \ + dma_queue * dma_queue = octx->ctx->dma[ith]; \ + const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \ + uint8_t * vtcm_src0 = srctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \ + uint8_t * vtcm_dst = srctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \ + const uint32_t src0_row_size = ne00 * sizeof(float); \ + const uint32_t dst_row_size = htp_tensor_get_row_size(octx->dst->type, ne00); \ + const uint32_t nrows_per_thread = ir1 - ir0; \ + const uint32_t total_steps = ne03 * ne02 * nrows_per_thread; \ + uint32_t pi_step = 0; \ + uint32_t pi02 = 0; \ + uint32_t pi03 = 0; \ + for (uint32_t step = 0, spad_idx = 0; step < total_steps && spad_idx < 2; ++step, spad_idx++) { \ + uint32_t i = ir0 + pi_step; \ + const uintptr_t src0_ptr = octx->src[0]->data + i*nb01 + pi02*nb02 + pi03*nb03; \ + dma_queue_push(dma_queue, \ + dma_make_ptr((void *)octx->dst->data, \ + vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \ + dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 0); \ + dma_queue_push(dma_queue, \ + dma_make_ptr((void *)(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size), \ + (const void *)src0_ptr), \ + vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \ + pi_step++; \ + if (pi_step == nrows_per_thread) { \ + pi_step = 0; \ + pi02++; \ + if (pi02 == ne02) { \ + pi02 = 0; \ + pi03++; \ + } \ + } \ + } \ + uint32_t ci_step = 0; \ + uint32_t ci02 = 0; \ + uint32_t ci03 = 0; \ + uint32_t ci11_base = 0; \ + uint32_t ci12_base = 0; \ + for (uint32_t step = 0; step < total_steps; ++step) { \ + void * dst_spad = (void *) dma_queue_pop(dma_queue).src; \ + void * src_spad = (void *) dma_queue_pop(dma_queue).dst; \ + uint32_t i = ir0 + ci_step; \ + const uintptr_t src1_addr = octx->src[1]->data + i*nb10 + ci11_base*nb11 + ci12_base*nb12; \ + const IDX_TYPE i1 = *(const IDX_TYPE *)src1_addr; \ + const bool valid_i1 = ((uint64_t)i1 < (uint64_t)ne1); \ + const uint32_t target_i1 = (uint32_t)i1; \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, step); \ + if (valid_i1) { \ + COMPUTE_EXPR; \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, step); \ + if (valid_i1) { \ + const uintptr_t dst_ptr = octx->dst->data + target_i1*nb1 + ci02*nb2 + ci03*nb3; \ + dma_queue_push(dma_queue, \ + dma_make_ptr((void *)dst_ptr, (const void *)dst_spad), \ + dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 1); \ + } else { \ + dma_queue_push(dma_queue, \ + dma_make_ptr((void *)octx->dst->data, (const void *)dst_spad), \ + dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 0); \ + } \ + const uint32_t next_step = step + 2; \ + if (next_step < total_steps) { \ + uint32_t ni = ir0 + pi_step; \ + const uintptr_t psrc0_ptr = octx->src[0]->data + ni*nb01 + pi02*nb02 + pi03*nb03; \ + dma_queue_push(dma_queue, \ + dma_make_ptr((void *)src_spad, (const void *)psrc0_ptr), \ + vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \ + pi_step++; \ + if (pi_step == nrows_per_thread) { \ + pi_step = 0; \ + pi02++; \ + if (pi02 == ne02) { \ + pi02 = 0; \ + pi03++; \ + } \ + } \ + } \ + ci_step++; \ + if (ci_step == nrows_per_thread) { \ + ci_step = 0; \ + ci02++; \ + ci11_base++; \ + if (ci11_base == ne11) { \ + ci11_base = 0; \ + } \ + if (ci02 == ne02) { \ + ci02 = 0; \ + ci03++; \ + ci12_base++; \ + if (ci12_base == ne12) { \ + ci12_base = 0; \ + } \ + } \ + } \ + } \ + dma_queue_flush(dma_queue); \ } -static void set_rows_thread_f16_f32(unsigned int nth, unsigned int ith, void *data) { - struct htp_set_rows_context * srctx = (struct htp_set_rows_context *)data; - struct htp_ops_context * octx = srctx->octx; +SET_ROWS_THREAD_DMA_FN(f32, int32_t, { hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); }) +SET_ROWS_THREAD_DMA_FN(f32, int64_t, { hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); }) - set_rows_preamble; +SET_ROWS_THREAD_DMA_FN(f16, int32_t, { hvx_copy_f16_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); }) +SET_ROWS_THREAD_DMA_FN(f16, int64_t, { hvx_copy_f16_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); }) - uint64_t qt = HAP_perf_get_qtimer_count(); - - // parallelize by rows of src0 - const uint32_t dr = srctx->src0_nrows_per_thread; - const uint32_t ir0 = dr * ith; - if (ir0 >= nr) { - return; - } - const uint32_t ir1 = (ir0 + dr < nr) ? (ir0 + dr) : nr; - - const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32); - - for (uint32_t i03 = 0; i03 < ne03; ++i03) { - for (uint32_t i02 = 0; i02 < ne02; ++i02) { - for (uint32_t i = ir0; i < ir1; ++i) { - const uint32_t i12 = fastmodulo(i03, ne12, &srctx->div_ne12); - const uint32_t i11 = fastmodulo(i02, ne11, &srctx->div_ne11); - const uint32_t i10 = i; - - const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12; - - uint32_t i1 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr; - if (i1 >= ne1) { - // ignore invalid indices - continue; - } - - const uint8_t* src0_ptr = (const uint8_t *) octx->src[0]->data + i*nb01 + i02*nb02 + i03*nb03; - uint8_t* dst_ptr = (uint8_t *) octx->dst->data + i1*nb1 + i02*nb2 + i03*nb3; - - hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00); - } - } - } - - qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt); - FARF(HIGH, "set-rows-f16-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, - ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt); -} +SET_ROWS_THREAD_DMA_FN(q8_0, int32_t, { hvx_quantize_row_q8_0_f32(dst_spad, (const float *)src_spad, ne00); }) +SET_ROWS_THREAD_DMA_FN(q8_0, int64_t, { hvx_quantize_row_q8_0_f32(dst_spad, (const float *)src_spad, ne00); }) int op_set_rows(struct htp_ops_context * octx) { + const struct htp_set_rows_kernel_params * kparams = (const struct htp_set_rows_kernel_params *)octx->kernel_params; set_rows_preamble; - const uint32_t n_threads = MIN(nr, octx->n_threads); - if (octx->src[0]->type != HTP_TYPE_F32) { return HTP_STATUS_NO_SUPPORT; } - if (octx->dst->type != HTP_TYPE_F32 && octx->dst->type != HTP_TYPE_F16) { + if (octx->dst->type != HTP_TYPE_F32 && octx->dst->type != HTP_TYPE_F16 && octx->dst->type != HTP_TYPE_Q8_0) { return HTP_STATUS_NO_SUPPORT; } @@ -164,27 +196,27 @@ int op_set_rows(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { - return HTP_STATUS_OK; - } + // l2fetch the src1 (indices) tensor in the main thread + hex_l2fetch_block((const void *)octx->src[1]->data, octx->src[1]->ne[3] * octx->src[1]->nb[3]); - struct htp_set_rows_context srctx; + struct set_rows_context srctx; srctx.octx = octx; - srctx.div_ne12 = init_fastdiv_values(ne12); - srctx.div_ne11 = init_fastdiv_values(ne11); + srctx.kparams = kparams; - srctx.src0_nrows_per_thread = (nr + n_threads - 1) / n_threads; + htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, kparams->n_threads); + srctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base; - switch(octx->dst->type) { - case HTP_TYPE_F32: - worker_pool_run_func(octx->ctx->worker_pool, set_rows_thread_f32_f32, &srctx, n_threads); - break; - case HTP_TYPE_F16: - worker_pool_run_func(octx->ctx->worker_pool, set_rows_thread_f16_f32, &srctx, n_threads); - break; - default: - return HTP_STATUS_NO_SUPPORT; + work_queue_func_t q_func = NULL; + const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32); + + switch (octx->dst->type) { + case HTP_TYPE_F32: q_func = is_i32 ? set_rows_thread_dma_f32_int32_t : set_rows_thread_dma_f32_int64_t; break; + case HTP_TYPE_F16: q_func = is_i32 ? set_rows_thread_dma_f16_int32_t : set_rows_thread_dma_f16_int64_t; break; + case HTP_TYPE_Q8_0: q_func = is_i32 ? set_rows_thread_dma_q8_0_int32_t : set_rows_thread_dma_q8_0_int64_t; break; + default: return HTP_STATUS_NO_SUPPORT; } + work_queue_run(octx->ctx->work_queue, q_func, &srctx, kparams->n_threads); + return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.h b/ggml/src/ggml-hexagon/htp/set-rows-ops.h new file mode 100644 index 000000000..5e98d2cb5 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.h @@ -0,0 +1,74 @@ +#ifndef HTP_SET_ROWS_OPS_H +#define HTP_SET_ROWS_OPS_H + +#include "hex-fastdiv.h" + +struct htp_set_rows_kernel_params { + int32_t n_threads; + int32_t total_tasks; + int32_t tasks_per_thread; + int32_t vtcm_size; + + // Fastdiv helpers + struct fastdiv_values div_ne11; + struct fastdiv_values div_ne12; + struct fastdiv_values div_tasks_per_thread; + struct fastdiv_values div_ne02; +}; + +struct htp_set_rows_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_dst; + + size_t src0_bytes_per_thread; + size_t dst_bytes_per_thread; + + size_t src0_spad_half_size; + size_t dst_spad_half_size; +}; + +static inline void htp_set_rows_vtcm_layout_build( + struct htp_set_rows_vtcm_layout * vtcm_layout, + int dst_type, + uint32_t ne00, + uint32_t n_threads) { + + size_t src0_row_size = ne00 * 4; + size_t dst_row_size = 0; + switch (dst_type) { + case 0: // HTP_TYPE_F32 + dst_row_size = ne00 * 4; + break; + case 1: // HTP_TYPE_F16 + dst_row_size = ne00 * 2; + break; + case 8: // HTP_TYPE_Q8_0 + dst_row_size = (ne00 / 32) * 34; + break; + default: + dst_row_size = 0; + break; + } + + size_t src0_row_size_aligned = (src0_row_size + 255) & ~255; + size_t dst_row_size_aligned = (dst_row_size + 255) & ~255; + + vtcm_layout->src0_spad_half_size = src0_row_size_aligned; + vtcm_layout->dst_spad_half_size = dst_row_size_aligned; + + vtcm_layout->src0_bytes_per_thread = src0_row_size_aligned * 2; + vtcm_layout->dst_bytes_per_thread = dst_row_size_aligned * 2; + + vtcm_layout->off_src0 = 0; + vtcm_layout->off_dst = vtcm_layout->off_src0 + vtcm_layout->src0_bytes_per_thread * n_threads; + vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads; +} + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_set_rows_kernel_params) <= 128, "htp_set_rows_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_set_rows_kernel_params) <= 128, "htp_set_rows_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_SET_ROWS_OPS_H diff --git a/scripts/snapdragon/adb/run-bench.sh b/scripts/snapdragon/adb/run-bench.sh deleted file mode 100755 index eaae80a77..000000000 --- a/scripts/snapdragon/adb/run-bench.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh -# - -# Basedir on device -basedir=/data/local/tmp/llama.cpp - -branch=. -[ "$B" != "" ] && branch=$B - -adbserial= -[ "$S" != "" ] && adbserial="-s $S" - -adbhost= -[ "$H" != "" ] && adbhost="-H $H" - -model="Llama-3.2-3B-Instruct-Q4_0.gguf" -[ "$M" != "" ] && model="$M" - -device="HTP0" -[ "$D" != "" ] && device="$D" - -verbose= -[ "$V" != "" ] && verbose="GGML_HEXAGON_VERBOSE=$V" cli_opts="$cli_opts -v" - -profile= -[ "$PROF" != "" ] && profile="GGML_HEXAGON_PROFILE=$PROF" cli_opts="$cli_opts -v" - -opmask= -[ "$OPSTAGE" != "" ] && opmask="GGML_HEXAGON_OPSTAGE=$OPSTAGE" - -nhvx= -[ "$NHVX" != "" ] && nhvx="GGML_HEXAGON_NHVX=$NHVX" - -ndev= -[ "$NDEV" != "" ] && ndev="GGML_HEXAGON_NDEV=$NDEV" - -hb= -[ "$HB" != "" ] && hb="GGML_HEXAGON_HOSTBUF=$HB" - -set -x - -adb $adbserial $adbhost shell " \ - cd $basedir; \ - LD_LIBRARY_PATH=$basedir/$branch/lib \ - ADSP_LIBRARY_PATH=$basedir/$branch/lib \ - $ndev $nhvx $opmask $verbose $profile $hb ./$branch/bin/llama-bench --device $device --load-mode none -m $basedir/../gguf/$model \ - --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \ - --ubatch-size 1024 -fa 1 -ngl 99 $cli_opts $@ \ -" diff --git a/scripts/snapdragon/adb/run-cli.sh b/scripts/snapdragon/adb/run-cli.sh deleted file mode 100755 index 27a4a1419..000000000 --- a/scripts/snapdragon/adb/run-cli.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/sh -# - -# Basedir on device -basedir=/data/local/tmp/llama.cpp - -cli_opts= - -branch=. -[ "$B" != "" ] && branch=$B - -adbserial= -[ "$S" != "" ] && adbserial="-s $S" - -adbhost= -[ "$H" != "" ] && adbhost="-H $H" - -model="Llama-3.2-3B-Instruct-Q4_0.gguf" -[ "$M" != "" ] && model="$M" - -device="HTP0" -[ "$D" != "" ] && device="$D" - -verbose= -[ "$V" != "" ] && verbose="GGML_HEXAGON_VERBOSE=$V" cli_opts="$cli_opts -v" - -sched= -[ "$SCHED" != "" ] && sched="GGML_SCHED_DEBUG=2" cli_opts="$cli_opts -v" - -profile= -[ "$PROF" != "" ] && profile="GGML_HEXAGON_PROFILE=$PROF" cli_opts="$cli_opts -v" - -opmask= -[ "$OPSTAGE" != "" ] && opmask="GGML_HEXAGON_OPSTAGE=$OPSTAGE" - -nhvx= -[ "$NHVX" != "" ] && nhvx="GGML_HEXAGON_NHVX=$NHVX" - -hmx= -[ "$HMX" != "" ] && hmx="GGML_HEXAGON_USE_HMX=$HMX" - -ndev= -[ "$NDEV" != "" ] && ndev="GGML_HEXAGON_NDEV=$NDEV" - -hb= -[ "$HB" != "" ] && hb="GGML_HEXAGON_HOSTBUF=$HB" - -opbatch= -[ "$OB" != "" ] && opbatch="GGML_HEXAGON_OPBATCH=$OB" - -opqueue= -[ "$OQ" != "" ] && opqueue="GGML_HEXAGON_OPQUEUE=$OQ" - -opflt= -[ "$OF" != "" ] && opflt="GGML_HEXAGON_OPFILTER=$OF" - -vmem= -[ "$VM" != "" ] && opflt="GGML_HEXAGON_VMEM=$VM" - -mbuf= -[ "$MB" != "" ] && opflt="GGML_HEXAGON_MBUF=$MB" -vmem= -[ "$VM" != "" ] && vmem="GGML_HEXAGON_VMEM=$VM" - -mbuf= -[ "$MB" != "" ] && mbuf="GGML_HEXAGON_MBUF=$MB" -set -x - -adb $adbserial $adbhost shell " \ - cd $basedir; ulimit -c unlimited; \ - LD_LIBRARY_PATH=$basedir/$branch/lib \ - ADSP_LIBRARY_PATH=$basedir/$branch/lib \ - $verbose $sched $opmask $profile $nhvx $hmx $ndev $hb $opbatch $opqueue $opflt $vmem $mbuf \ - ./$branch/bin/llama-cli --load-mode none -m $basedir/../gguf/$model \ - --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \ - --ctx-size 8192 --ubatch-size 1024 -fa on \ - -ngl 99 --device $device $cli_opts $@ \ -" diff --git a/scripts/snapdragon/adb/run-completion.sh b/scripts/snapdragon/adb/run-completion.sh deleted file mode 100755 index 30893ed29..000000000 --- a/scripts/snapdragon/adb/run-completion.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/sh -# - -# Basedir on device -basedir=/data/local/tmp/llama.cpp - -cli_opts= - -branch=. -[ "$B" != "" ] && branch=$B - -adbserial= -[ "$S" != "" ] && adbserial="-s $S" - -adbhost= -[ "$H" != "" ] && adbhost="-H $H" - -model="Llama-3.2-3B-Instruct-Q4_0.gguf" -[ "$M" != "" ] && model="$M" - -device="HTP0" -[ "$D" != "" ] && device="$D" - -verbose= -[ "$V" != "" ] && verbose="GGML_HEXAGON_VERBOSE=$V" cli_opts="$cli_opts -v" - -sched= -[ "$SCHED" != "" ] && sched="GGML_SCHED_DEBUG=2" cli_opts="$cli_opts -v" - -profile= -[ "$PROF" != "" ] && profile="GGML_HEXAGON_PROFILE=$PROF" cli_opts="$cli_opts -v" - -opmask= -[ "$OPSTAGE" != "" ] && opmask="GGML_HEXAGON_OPSTAGE=$OPSTAGE" - -nhvx= -[ "$NHVX" != "" ] && nhvx="GGML_HEXAGON_NHVX=$NHVX" - -hmx= -[ "$HMX" != "" ] && hmx="GGML_HEXAGON_USE_HMX=$HMX" - -ndev= -[ "$NDEV" != "" ] && ndev="GGML_HEXAGON_NDEV=$NDEV" - -hb= -[ "$HB" != "" ] && hb="GGML_HEXAGON_HOSTBUF=$HB" - -opbatch= -[ "$OB" != "" ] && opbatch="GGML_HEXAGON_OPBATCH=$OB" - -opqueue= -[ "$OQ" != "" ] && opqueue="GGML_HEXAGON_OPQUEUE=$OQ" - -oppoll= -[ "$OP" != "" ] && oppoll="GGML_HEXAGON_OPPOLL=$OP" - -opflt= -[ "$OF" != "" ] && opflt="GGML_HEXAGON_OPFILTER=$OF" - -opfuse= -[ "$OC" != "" ] && opfuse="GGML_HEXAGON_OPFUSION=$OC" - -vmem= -[ "$VM" != "" ] && vmem="GGML_HEXAGON_VMEM=$VM" - -mbuf= -[ "$MB" != "" ] && mbuf="GGML_HEXAGON_MBUF=$MB" - -mmsel= -[ "$MM" != "" ] && mmsel="GGML_HEXAGON_MM_SELECT=$MM" - -fasel= -[ "$FA" != "" ] && fasel="GGML_HEXAGON_FA_SELECT=$FA" - -set -x - -adb $adbserial $adbhost shell " \ - cd $basedir; ulimit -c unlimited; \ - LD_LIBRARY_PATH=$basedir/$branch/lib \ - ADSP_LIBRARY_PATH=$basedir/$branch/lib \ - $verbose $sched $opmask $profile $nhvx $hmx $ndev $hb $opbatch $opqueue $oppoll $opflt $opfuse $vmem $mbuf $mmsel $fasel \ - ./$branch/bin/llama-completion --load-mode none -m $basedir/../gguf/$model \ - --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \ - --ctx-size 8192 --ubatch-size 1024 -fa on \ - -ngl 99 --device $device $cli_opts $@ \ -" diff --git a/scripts/snapdragon/adb/run-mtmd.sh b/scripts/snapdragon/adb/run-mtmd.sh deleted file mode 100755 index 65dd6ec59..000000000 --- a/scripts/snapdragon/adb/run-mtmd.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/sh -# - -# Basedir on device -basedir=/data/local/tmp/llama.cpp - -cli_opts= - -branch=. -[ "$B" != "" ] && branch=$B - -adbserial= -[ "$S" != "" ] && adbserial="-s $S" - -adbhost= -[ "$H" != "" ] && adbhost="-H $H" - -model="gemma-3-4b-it-Q4_0.gguf" -[ "$M" != "" ] && model="$M" - -mmproj="mmproj-F16.gguf" -[ "$MMPROJ" != "" ] && mmproj="$MMPROJ" - -image= -[ "$IMG" != "" ] && image="$IMG" - -device="HTP0" -[ "$D" != "" ] && device="$D" - -verbose= -[ "$V" != "" ] && verbose="GGML_HEXAGON_VERBOSE=$V" - -experimental="GGML_HEXAGON_EXPERIMENTAL=1" -[ "$E" != "" ] && experimental="GGML_HEXAGON_EXPERIMENTAL=$E" - -sched= -[ "$SCHED" != "" ] && sched="GGML_SCHED_DEBUG=2" cli_opts="$cli_opts -v" - -profile= -[ "$PROF" != "" ] && profile="GGML_HEXAGON_PROFILE=$PROF" - -opmask= -[ "$OPSTAGE" != "" ] && opmask="GGML_HEXAGON_OPSTAGE=$OPSTAGE" - -nhvx= -[ "$NHVX" != "" ] && nhvx="GGML_HEXAGON_NHVX=$NHVX" - -hmx= -[ "$HMX" != "" ] && hmx="GGML_HEXAGON_USE_HMX=$HMX" - -ndev= -[ "$NDEV" != "" ] && ndev="GGML_HEXAGON_NDEV=$NDEV" - -# MTMD backend device for vision model (defaults to CPU if not set) -mtmd_backend= -[ "$MTMD_DEVICE" != "" ] && mtmd_backend="MTMD_BACKEND_DEVICE=$MTMD_DEVICE" - -set -x - -adb $adbserial $adbhost shell " \ - cd $basedir; ulimit -c unlimited; \ - LD_LIBRARY_PATH=$basedir/$branch/lib \ - ADSP_LIBRARY_PATH=$basedir/$branch/lib \ - $verbose $experimental $sched $opmask $profile $hmx $nhvx $ndev $mtmd_backend \ - ./$branch/bin/llama-mtmd-cli --load-mode none -m $basedir/../gguf/$model \ - --mmproj $basedir/../gguf/$mmproj \ - --image $basedir/../gguf/$image \ - --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \ - --ctx-size 8192 --ubatch-size 1024 -fa on \ - -ngl 99 --device $device -v $cli_opts $@ \ -" diff --git a/scripts/snapdragon/adb/run-tool.sh b/scripts/snapdragon/adb/run-tool.sh deleted file mode 100755 index 8bf7ba3a5..000000000 --- a/scripts/snapdragon/adb/run-tool.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/sh -# - -# Basedir on device -basedir=/data/local/tmp/llama.cpp - -cli_opts= - -branch=. -[ "$B" != "" ] && branch=$B - -adbserial= -[ "$S" != "" ] && adbserial="-s $S" - -adbhost= -[ "$H" != "" ] && adbhost="-H $H" - -device="HTP0" -[ "$D" != "" ] && device="$D" - -verbose= -[ "$V" != "" ] && verbose="GGML_HEXAGON_VERBOSE=$V" - -sched= -[ "$SCHED" != "" ] && sched="GGML_SCHED_DEBUG=2" cli_opts="$cli_opts -v" - -profile= -[ "$PROF" != "" ] && profile="GGML_HEXAGON_PROFILE=$PROF" - -opmask= -[ "$OPSTAGE" != "" ] && opmask="GGML_HEXAGON_OPSTAGE=$OPSTAGE" - -nhvx= -[ "$NHVX" != "" ] && nhvx="GGML_HEXAGON_NHVX=$NHVX" - -hmx= -[ "$HMX" != "" ] && hmx="GGML_HEXAGON_USE_HMX=$HMX" - -ndev= -[ "$NDEV" != "" ] && ndev="GGML_HEXAGON_NDEV=$NDEV" - -hb= -[ "$HB" != "" ] && hb="GGML_HEXAGON_HOSTBUF=$HB" - -opbatch= -[ "$OB" != "" ] && opbatch="GGML_HEXAGON_OPBATCH=$OB" - -opqueue= -[ "$OQ" != "" ] && opqueue="GGML_HEXAGON_OPQUEUE=$OQ" - -oppoll= -[ "$OP" != "" ] && oppoll="GGML_HEXAGON_OPPOLL=$OP" - -opfuse= -[ "$OC" != "" ] && opfuse="GGML_HEXAGON_OPFUSION=$OC" - -mmsel= -[ "$MM" != "" ] && mmsel="GGML_HEXAGON_MM_SELECT=$MM" - -fasel= -[ "$FA" != "" ] && fasel="GGML_HEXAGON_FA_SELECT=$FA" - -set -x - -tool=$1; shift - -adb $adbserial $adbhost shell " \ - cd $basedir; ulimit -c unlimited; \ - LD_LIBRARY_PATH=$basedir/$branch/lib \ - ADSP_LIBRARY_PATH=$basedir/$branch/lib \ - $verbose $sched $opmask $profile $nhvx $hmx $ndev $hb $opbatch $opqueue $oppoll $opfuse $mmsel $fasel ./$branch/bin/$tool $@ \ -" diff --git a/scripts/snapdragon/build.py b/scripts/snapdragon/build.py new file mode 100755 index 000000000..02dcb930c --- /dev/null +++ b/scripts/snapdragon/build.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +# +# Build llama.cpp for Snapdragon (via Docker or natively) and push to device. +# + +import sys +import os +import argparse +import subprocess +import platform +import shutil +import logging + +logger = logging.getLogger("build") + + +def parse_target(target_str): + if not target_str: + return None, None + if target_str.startswith("adb") or target_str.startswith("android"): + parts = target_str.split(":", 1) + serial = parts[1] if len(parts) > 1 else None + return "android", serial + elif target_str.startswith("lnx") or target_str.startswith("linux") or target_str.startswith("ubuntu"): + parts = target_str.split(":", 1) + host = parts[1] if len(parts) > 1 else None + return "linux", host + elif target_str in ("wos", "windows"): + return "windows", None + else: + return None, None + + +def get_uid_gid(): + if platform.system() != "Windows": + return [f"{os.getuid()}:{os.getgid()}"] + return [] + + +def main(): + logging.basicConfig(level=logging.INFO, format='%(message)s') + parser = argparse.ArgumentParser( + description="Build llama.cpp for Snapdragon using cross-compilation docker containers or natively." + ) + parser.add_argument("--target", default="android", help="Compilation target and deployment definition (e.g. android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, windows/wos) (default: android)") + parser.add_argument("--build-dir", help="Build directory name (defaults to build-TARGET[-dbg], e.g. build-android)") + parser.add_argument("--install-dir", help="Install directory name (defaults to pkg-TARGET[-dbg], e.g. pkg-android)") + parser.add_argument("--jobs", "-j", type=int, help="Number of build jobs (defaults to CPU thread count)") + parser.add_argument("--no-docker", action="store_true", help="Build natively on the host instead of in a docker container") + parser.add_argument("--preset", help="Override the CMake preset to use") + parser.add_argument("--debug", action="store_true", help="Build in debug mode (uses -debug presets instead of -release)") + + # Push options + parser.add_argument("--push", action="store_true", help="Push built package to the target device via ADB or SSH/SCP") + parser.add_argument("--target-dir", help="Target directory on the device (default: /data/local/tmp/llama.cpp for Android, ~/llama.cpp for Linux)") + + # Toolchain options + parser.add_argument("--toolchain-version", default="v0.7", help="Docker toolchain image version/tag (default: v0.7)") + parser.add_argument("--toolchain-url", default="ghcr.io/snapdragon-toolchain", help="Docker toolchain registry URL/namespace (default: ghcr.io/snapdragon-toolchain)") + + args = parser.parse_args() + + target_type, target_val = parse_target(args.target) + if not target_type: + logger.error(f"Error: Invalid target format '{args.target}'. Must be android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, or windows/wos.") + sys.exit(1) + + # Determine preset and check if it's debug + preset = args.preset + if preset: + is_debug = args.debug or ("debug" in preset.lower()) + else: + is_debug = args.debug + config_type = "debug" if is_debug else "release" + if args.no_docker: + if target_type == "windows" or platform.system() == "Windows": + preset = f"arm64-windows-snapdragon-{config_type}" + elif target_type == "linux": + preset = f"arm64-linux-snapdragon-{config_type}" + else: + preset = f"arm64-android-snapdragon-{config_type}" + else: + preset = f"arm64-linux-snapdragon-{config_type}" if target_type == "linux" else f"arm64-android-snapdragon-{config_type}" + + target_prefix = args.target.split(":", 1)[0] + suffix = "-dbg" if is_debug else "" + + build_dir = args.build_dir + if not build_dir: + build_dir = f"build-{target_prefix}{suffix}" + + install_dir = args.install_dir + if not install_dir: + install_dir = f"pkg-{target_prefix}{suffix}" + + repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) + + # Ensure CMakeUserPresets.json is in the workspace root, update if docs version is newer + preset_src = os.path.join(repo_root, "docs", "backend", "snapdragon", "CMakeUserPresets.json") + preset_dst = os.path.join(repo_root, "CMakeUserPresets.json") + if os.path.exists(preset_src): + should_copy = False + if not os.path.exists(preset_dst): + should_copy = True + else: + # Check modification times + src_mtime = os.path.getmtime(preset_src) + dst_mtime = os.path.getmtime(preset_dst) + if src_mtime > dst_mtime: + preset_bak = preset_dst + ".bak" + logger.info(f"Docs CMakeUserPresets.json is newer. Backing up existing {preset_dst} to {preset_bak}") + shutil.copy2(preset_dst, preset_bak) + should_copy = True + + if should_copy: + logger.info(f"Copying CMakeUserPresets.json from {preset_src} to {preset_dst}") + shutil.copy2(preset_src, preset_dst) + else: + logger.warning("Warning: CMakeUserPresets.json not found in docs/backend/snapdragon/.") + + jobs = args.jobs if args.jobs else os.cpu_count() or 4 + + if target_type == "windows": + logger.info("Windows target selected. Forcing native compilation...") + args.no_docker = True + if platform.system() != "Windows": + logger.warning("Warning: Windows compilation is intended to run on Windows arm64 hosts.") + + if args.no_docker: + # Native/local host build + logger.info("Running native/local CMake build...") + install_prefix = os.path.join(repo_root, install_dir, "llama.cpp") + + # Configure + configure_cmd = ["cmake", f"--preset={preset}", "-B", build_dir] + logger.info(f"+ {' '.join(configure_cmd)}") + res = subprocess.run(configure_cmd, cwd=repo_root) + if res.returncode != 0: + logger.error("CMake configuration failed.") + sys.exit(res.returncode) + + # Build + build_cmd = ["cmake", "--build", build_dir, "-j", str(jobs)] + logger.info(f"+ {' '.join(build_cmd)}") + res = subprocess.run(build_cmd, cwd=repo_root) + if res.returncode != 0: + logger.error("CMake build failed.") + sys.exit(res.returncode) + + # Install + install_cmd = ["cmake", "--install", build_dir, "--prefix", install_prefix] + logger.info(f"+ {' '.join(install_cmd)}") + res = subprocess.run(install_cmd, cwd=repo_root) + if res.returncode != 0: + logger.error("CMake install failed.") + sys.exit(res.returncode) + else: + # Docker-based build + logger.info("Running Docker-based cross-compilation build...") + image_name = "arm64-linux" if target_type == "linux" else "arm64-android" + image = f"{args.toolchain_url}/{image_name}:{args.toolchain_version}" + + install_prefix_container = f"/workspace/{install_dir}/llama.cpp" + + build_sh_cmd = ( + f"cmake --preset {preset} -B /workspace/{build_dir} && " + f"cmake --build /workspace/{build_dir} -j {jobs} && " + f"cmake --install /workspace/{build_dir} --prefix {install_prefix_container}" + ) + + docker_cmd = [ + "docker", "run", "--rm", + "--volume", f"{repo_root}:/workspace", + "--workdir", "/workspace", + "--platform", "linux/amd64" + ] + uid_gid = get_uid_gid() + if uid_gid: + docker_cmd += ["-u", uid_gid[0]] + + docker_cmd += [image, "bash", "-c", build_sh_cmd] + + logger.info(f"+ {' '.join(docker_cmd)}") + res = subprocess.run(docker_cmd, cwd=repo_root) + if res.returncode != 0: + logger.error("Docker-based build failed.") + sys.exit(res.returncode) + + logger.info("\nBuild and installation completed successfully!") + + # Push/deploy if requested + if args.push: + src_path = os.path.join(repo_root, install_dir, "llama.cpp") + if not os.path.exists(src_path): + logger.error(f"Error: installation directory {src_path} does not exist. Cannot deploy.") + sys.exit(1) + + # Resolve target directory on device + target_dir = args.target_dir + if not target_dir: + target_dir = "/data/local/tmp/llama.cpp" if target_type == "android" else "~/llama.cpp" + target_dir = target_dir.rstrip("/") + + sub_items = [item for item in os.listdir(src_path) if not item.startswith(".")] + + if target_type == "android": + logger.info("\nPushing built artifacts to Android device via ADB...") + adb_cmd = ["adb"] + if target_val: # serial + adb_cmd += ["-s", target_val] + + # Clean stale package files on device + if sub_items: + clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items) + clean_cmd = adb_cmd + ["shell", f"rm -rf {clean_paths}"] + logger.info(f"+ {' '.join(clean_cmd)}") + subprocess.run(clean_cmd) + + # Android destination directory is target_dir + push_cmd = adb_cmd + ["push", os.path.join(src_path, "."), target_dir] + logger.info(f"+ {' '.join(push_cmd)}") + res = subprocess.run(push_cmd) + if res.returncode != 0: + logger.error("ADB push failed.") + sys.exit(res.returncode) + logger.info("ADB push completed successfully!") + + elif target_type == "linux": + ssh_host = target_val + if not ssh_host: + logger.error("Error: SSH host not specified in target (e.g. use linux:user@host, lnx:user@host, or ubuntu:user@host). Cannot deploy.") + sys.exit(1) + logger.info(f"\nDeploying built artifacts to Linux device {ssh_host} via SSH/SCP...") + + # Clean stale package files on remote host + if sub_items: + clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items) + clean_cmd = ["ssh", ssh_host, f"rm -rf {clean_paths}"] + logger.info(f"+ {' '.join(clean_cmd)}") + subprocess.run(clean_cmd) + + # Deploy to target_dir + deploy_cmd = ["scp", "-r", os.path.join(src_path, "."), f"{ssh_host}:{target_dir}"] + logger.info(f"+ {' '.join(deploy_cmd)}") + res = subprocess.run(deploy_cmd) + if res.returncode != 0: + logger.error("SSH/SCP deploy failed.") + sys.exit(res.returncode) + logger.info("SSH/SCP deploy completed successfully!") + + elif target_type == "windows": + logger.info("\nPush for Windows on Snapdragon (windows) target is currently a stub.") + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + logger.info("\nInterrupted by user.") + sys.exit(130) diff --git a/scripts/snapdragon/ggml-hexagon-profile.py b/scripts/snapdragon/ggml-hexagon-profile.py index 97a3acd26..038d92fb5 100755 --- a/scripts/snapdragon/ggml-hexagon-profile.py +++ b/scripts/snapdragon/ggml-hexagon-profile.py @@ -34,6 +34,26 @@ trace_pattern = re.compile( r"trace-evt\s+(?P[A-Z_0-9\-]+):\s+thread\s+(?P\d+)\s+info\s+(?P\d+)\s+(?Pstart|stop)\s+(?P\d+)" ) +device_pattern = re.compile(r"\b(HTP\d+(?::\d+)?)\s+(?:profile-op|trace-evt)\b") + + +def extract_device(line): + m = device_pattern.search(line) + if m: + return m.group(1) + return "HTP0" + + +def device_matches(record_device, target_device): + targets = [t.strip() for t in target_device.split(',')] + for target in targets: + if record_device == target: + return True + if record_device.startswith(target + ":"): + return True + return False + + logger = logging.getLogger("ggml-hexagon-profile") @@ -72,7 +92,7 @@ class CycleUnwrapper: return raw + self.high_part -def parse_log(file_path, pmu_index=None): +def parse_log(file_path, pmu_index=None, limit=None, device_filter=None, op_filter_re=None): try: if file_path != "-": f = open(file_path, 'r', encoding='utf-8', errors='ignore') @@ -85,13 +105,22 @@ def parse_log(file_path, pmu_index=None): all_ops: List[Dict[str, Any]] = [] all_traces: List[Dict[str, Any]] = [] current_op: Optional[Dict[str, Any]] = None + ops_count_per_device = {} + if device_filter is not None: + for target in device_filter.split(','): + ops_count_per_device[target.strip()] = 0 + limit_reached = False - timestamp_pattern = re.compile(r"^(?P\d+)\.(?P\d+)\.(?P\d+)\.(?P\d+)\s+[A-Z]\s+") - unwrapper = None - trace_unwrapper = None + timestamp_pattern = re.compile(r"(?P\d+)\.(?P\d+)\.(?P\d+)\.(?P\d+)\s+[A-Z]\s+") + unwrappers = {} + last_batch_start = {} + trace_unwrappers = {} for line in f: - ts_match = timestamp_pattern.match(line) + if "profile-op" not in line and "trace-evt" not in line: + continue + + ts_match = timestamp_pattern.search(line) abs_usec = 0 if ts_match: abs_usec = ( @@ -100,8 +129,11 @@ def parse_log(file_path, pmu_index=None): + int(ts_match.group('us')) ) - if "|" in line and "profile-op" in line: - parts = [p.strip() for p in line.split("|")] + device = extract_device(line) + + idx = line.find("profile-op") + if idx != -1 and "|" in line[idx:]: + parts = [p.strip() for p in line[idx:].split("|")] prefix = parts[0] prefix_match = re.search(r"profile-op\s+(?P[A-Z_0-9+]+)", prefix) if not prefix_match: @@ -145,7 +177,6 @@ def parse_log(file_path, pmu_index=None): except (ValueError, IndexError): pmu_val = None - evt_val = None evt_val = None if types.startswith("evt-cnt "): try: @@ -158,14 +189,18 @@ def parse_log(file_path, pmu_index=None): if op_name == "OPBATCH": if cycles_start_raw: unwrapped_cycles_start = int(cycles_start_raw) - unwrapper = CycleUnwrapper(unwrapped_cycles_start) - trace_unwrapper = CycleUnwrapper(unwrapped_cycles_start) + unwrappers[device] = CycleUnwrapper(unwrapped_cycles_start) + last_batch_start[device] = unwrapped_cycles_start + for k in list(trace_unwrappers.keys()): + if k[0] == device: + del trace_unwrappers[k] else: - if cycles_start_raw and unwrapper is not None: - unwrapped_cycles_start = unwrapper.unwrap(int(cycles_start_raw)) + if cycles_start_raw: + device_unwrapper = unwrappers.get(device) + if device_unwrapper is not None: + unwrapped_cycles_start = device_unwrapper.unwrap(int(cycles_start_raw)) - idx = line.find("profile-op ") - op_text = line[idx + 11:].strip() if idx != -1 else line.strip() + op_text = re.sub(r"^profile-op\s+", "", line[idx:]).strip() if idx != -1 else line.strip() current_op = { 'name': op_name, @@ -180,24 +215,58 @@ def parse_log(file_path, pmu_index=None): 'pmu_val': pmu_val, 'evt_val': evt_val, 'abs_usec': abs_usec, - 'trace_events': [] + 'trace_events': [], + 'device': device } all_ops.append(current_op) + + # Check if matching early exit criteria + matched = False + matched_target = None + if device_filter is not None: + targets = [t.strip() for t in device_filter.split(',')] + for target in targets: + if device == target or device.startswith(target + ":"): + matched = True + matched_target = target + break + else: + matched = True + matched_target = device + + if op_filter_re is not None and not op_filter_re.search(op_text): + matched = False + + if matched: + if matched_target not in ops_count_per_device: + ops_count_per_device[matched_target] = 0 + ops_count_per_device[matched_target] += 1 + + if limit is not None and len(ops_count_per_device) > 0 and all(count >= limit for count in ops_count_per_device.values()): + limit_reached = True + + if limit_reached and op_name == "OPBATCH": + break continue trace_match = trace_pattern.search(line) if trace_match: + thread = int(trace_match.group('thread')) raw_cyc = int(trace_match.group('cycles')) unwrapped_cyc = None - if trace_unwrapper is not None: - unwrapped_cyc = trace_unwrapper.unwrap(raw_cyc) + th_key = (device, thread) + if th_key not in trace_unwrappers: + batch_start = last_batch_start.get(device) + trace_unwrappers[th_key] = CycleUnwrapper(batch_start) + unwrapped_cyc = trace_unwrappers[th_key].unwrap(raw_cyc) all_traces.append({ - 'thread': int(trace_match.group('thread')), + 'thread': thread, 'event': trace_match.group('event'), 'info': int(trace_match.group('info')), 'cycles': raw_cyc, 'unwrapped_cycles': unwrapped_cyc, - 'state': trace_match.group('state') + 'state': trace_match.group('state'), + 'device': device }) f.close() @@ -207,39 +276,45 @@ def parse_log(file_path, pmu_index=None): op['start_cycles'] = op['unwrapped_cycles_start'] op['end_cycles'] = op['start_cycles'] + op['cycles'] if op['start_cycles'] is not None else None - # Filter ops with valid start_cycles - valid_ops = [op for op in all_ops if op['start_cycles'] is not None and op['end_cycles'] is not None] + # Group ops by device + valid_ops_by_dev = defaultdict(list) + for op in all_ops: + if op['start_cycles'] is not None and op['end_cycles'] is not None: + valid_ops_by_dev[op['device']].append(op) - # Separate OPBATCH ops from other ops - opbatch_ops = [op for op in valid_ops if op['name'] == "OPBATCH"] - other_ops = [op for op in valid_ops if op['name'] != "OPBATCH"] - - # Sort them by start_cycles to enable binary search - opbatch_ops.sort(key=lambda op: op['start_cycles']) - other_ops.sort(key=lambda op: op['start_cycles']) - - opbatch_starts = [op['start_cycles'] for op in opbatch_ops] - other_starts = [op['start_cycles'] for op in other_ops] - - # Map trace events to any operator whose cycles contain them + # Group trace events by device + traces_by_dev = defaultdict(list) for e in all_traces: - cyc = e['unwrapped_cycles'] - if cyc is None: - continue + if e['unwrapped_cycles'] is not None: + traces_by_dev[e['device']].append(e) - # Map to OPBATCH - idx = bisect.bisect_right(opbatch_starts, cyc) - 1 - if idx >= 0: - op = opbatch_ops[idx] - if op['start_cycles'] <= cyc <= op['end_cycles']: - op['trace_events'].append(e) + for device, dev_ops in valid_ops_by_dev.items(): + opbatch_ops = [op for op in dev_ops if op['name'] == "OPBATCH"] + other_ops = [op for op in dev_ops if op['name'] != "OPBATCH"] - # Map to other ops - idx = bisect.bisect_right(other_starts, cyc) - 1 - if idx >= 0: - op = other_ops[idx] - if op['start_cycles'] <= cyc <= op['end_cycles']: - op['trace_events'].append(e) + opbatch_ops.sort(key=lambda op: op['start_cycles']) + other_ops.sort(key=lambda op: op['start_cycles']) + + opbatch_starts = [op['start_cycles'] for op in opbatch_ops] + other_starts = [op['start_cycles'] for op in other_ops] + + dev_traces = traces_by_dev.get(device, []) + for e in dev_traces: + cyc = e['unwrapped_cycles'] + + # Map to OPBATCH + idx = bisect.bisect_right(opbatch_starts, cyc) - 1 + if idx >= 0: + op = opbatch_ops[idx] + if op['start_cycles'] <= cyc <= op['end_cycles']: + op['trace_events'].append(e) + + # Map to other ops + idx = bisect.bisect_right(other_starts, cyc) - 1 + if idx >= 0: + op = other_ops[idx] + if op['start_cycles'] <= cyc <= op['end_cycles']: + op['trace_events'].append(e) return all_ops @@ -563,6 +638,7 @@ def main(): parser.add_argument("--timeline", type=str, nargs='?', const='summary', choices=["summary", "bubbles"], help="Output ASCII art event summary or thread idle bubble analysis (default: summary)") parser.add_argument("--filter", type=str, help="Regex filter matching against the original profile-op line") + parser.add_argument("--device", type=str, help="Device to filter by (e.g. HTP0, HTP0:0) or 'split' to generate separate reports per device") group = parser.add_mutually_exclusive_group() group.add_argument("--head", type=int, help="Limit to first N ops") @@ -586,29 +662,84 @@ def main(): logger.warning(f"Invalid width format '{w}'") final_pmu_name = (args.pmu_name or f"#{args.pmu_index}") if args.pmu_index is not None else None - ops = parse_log(args.logfile, pmu_index=args.pmu_index) + op_filter_re = None if args.filter: try: - filter_re = re.compile(args.filter) + op_filter_re = re.compile(args.filter) except re.error as e: logger.error(f"Invalid regex filter: {e}") sys.exit(1) - ops = [op for op in ops if filter_re.search(op['op_text'])] - if args.head is not None: - ops = ops[:args.head] - elif args.tail is not None: - ops = ops[-args.tail:] + limit = args.head if args.head is not None else None + device_filter = args.device if (args.device and args.device != "split") else None + ops = parse_log(args.logfile, pmu_index=args.pmu_index, limit=limit, device_filter=device_filter, op_filter_re=op_filter_re) - if args.timeline: - for op in ops: - if args.timeline == "summary": - print_ascii_summary(op['name'], op['dims'], op['types'], op['usec'], op['cycles'], op['trace_events']) - elif args.timeline == "bubbles": - print_bubbles_timeline(op) + if args.device and args.device != "split": + ops = [op for op in ops if device_matches(op['device'], args.device)] + + if args.device == "split": + unique_devices = sorted(list(set(op['device'] for op in ops))) + for dev in unique_devices: + dev_ops = [op for op in ops if device_matches(op['device'], dev)] + + if args.filter: + try: + filter_re = re.compile(args.filter) + except re.error as e: + logger.error(f"Invalid regex filter: {e}") + sys.exit(1) + dev_ops = [op for op in dev_ops if filter_re.search(op['op_text'])] + + if args.head is not None: + dev_ops = dev_ops[:args.head] + elif args.tail is not None: + dev_ops = dev_ops[-args.tail:] + + logger.info("\n=========================================") + logger.info(f" Device: {dev}") + logger.info("=========================================") + + if args.timeline: + for op in dev_ops: + if args.timeline == "summary": + print_ascii_summary(op['name'], op['dims'], op['types'], op['usec'], op['cycles'], op['trace_events']) + elif args.timeline == "bubbles": + print_bubbles_timeline(op) + else: + generate_report(dev_ops, args.top, overrides, args.sort, pmu_name=final_pmu_name) else: - generate_report(ops, args.top, overrides, args.sort, pmu_name=final_pmu_name) + if args.filter: + try: + filter_re = re.compile(args.filter) + except re.error as e: + logger.error(f"Invalid regex filter: {e}") + sys.exit(1) + ops = [op for op in ops if filter_re.search(op['op_text'])] + + if args.head is not None or args.tail is not None: + ops_by_dev = defaultdict(list) + for op in ops: + ops_by_dev[op['device']].append(op) + + filtered_ops = [] + for dev in sorted(ops_by_dev.keys()): + dev_ops = ops_by_dev[dev] + if args.head is not None: + dev_ops = dev_ops[:args.head] + elif args.tail is not None: + dev_ops = dev_ops[-args.tail:] + filtered_ops.extend(dev_ops) + ops = filtered_ops + + if args.timeline: + for op in ops: + if args.timeline == "summary": + print_ascii_summary(op['name'], op['dims'], op['types'], op['usec'], op['cycles'], op['trace_events']) + elif args.timeline == "bubbles": + print_bubbles_timeline(op) + else: + generate_report(ops, args.top, overrides, args.sort, pmu_name=final_pmu_name) if __name__ == "__main__": diff --git a/scripts/snapdragon/ggml-hexagon-trace.py b/scripts/snapdragon/ggml-hexagon-trace.py index 4755adfa1..99bf771b8 100755 --- a/scripts/snapdragon/ggml-hexagon-trace.py +++ b/scripts/snapdragon/ggml-hexagon-trace.py @@ -20,6 +20,31 @@ trace_pattern = re.compile( r"trace-evt\s+(?P[A-Z_0-9\-]+):\s+thread\s+(?P\d+)\s+info\s+(?P\d+)\s+(?Pstart|stop)\s+(?P\d+)" ) +device_pattern = re.compile(r"\b(HTP\d+(?::\d+)?)\s+(?:profile-op|trace-evt)\b") + + +def extract_device(line): + m = device_pattern.search(line) + if m: + return m.group(1) + return "HTP0" + + +def device_matches(record_device, target_device): + targets = [t.strip() for t in target_device.split(',')] + for target in targets: + if record_device == target: + return True + if record_device.startswith(target + ":"): + return True + return False + + +def get_split_output_path(base_path, device_name): + safe_device = device_name.replace(':', '_') + root, ext = os.path.splitext(base_path) + return f"{root}-{safe_device}{ext}" + def normalize_event_name(evt_type, info=0): if evt_type == "HVX_COMP": @@ -54,7 +79,79 @@ class CycleUnwrapper: return raw + self.high_part -def parse_log(file_path): +class DeviceTimeMapper: + def __init__(self, dev, ops): + self.dev = dev + self.batches = [] + for op in ops: + if op.get('device') == dev and op.get('name') == 'OPBATCH' and op.get('unwrapped_cycles_start') is not None: + cycles = op.get('cycles', 0) + usec = op.get('usec', 0) + start_cyc = op['unwrapped_cycles_start'] + freq = (cycles / usec) if usec > 0 and cycles > 0 else 1000.0 + if freq <= 0: + freq = 1000.0 + self.batches.append({ + 'start_cycles': start_cyc, + 'cycles': cycles, + 'end_cycles': start_cyc + cycles, + 'usec': usec, + 'dur_ns': usec * 1000, + 'freq_mhz': freq, + }) + + self.batches.sort(key=lambda b: b['start_cycles']) + + for i, b in enumerate(self.batches): + if i == 0: + b['start_time_ns'] = 0 + else: + prev = self.batches[i - 1] + idle_cyc = max(0, b['start_cycles'] - prev['end_cycles']) + idle_ns = int(round((idle_cyc / prev['freq_mhz']) * 1000)) + b['start_time_ns'] = prev['start_time_ns'] + prev['dur_ns'] + idle_ns + + self.batch_starts = [b['start_cycles'] for b in self.batches] + + valid_starts = [op['unwrapped_cycles_start'] for op in ops if op.get('device') == dev and op.get('unwrapped_cycles_start') is not None] + self.min_cyc = min(valid_starts) if valid_starts else 0 + if self.batches: + self.default_freq = self.batches[0]['freq_mhz'] + else: + freqs = [op['cycles'] / op['usec'] for op in ops if op.get('device') == dev and op.get('usec', 0) > 0 and op.get('cycles', 0) > 0] + self.default_freq = statistics.mean(freqs) if freqs else 1000.0 + + def get_batch(self, cyc): + if not self.batches: + return None + idx = bisect.bisect_right(self.batch_starts, cyc) - 1 + if idx >= 0: + return self.batches[idx] + return self.batches[0] + + def get_freq(self, cyc=None): + if cyc is not None: + b = self.get_batch(cyc) + if b is not None: + return b['freq_mhz'] + return self.default_freq + + def cycle_to_ns(self, cyc): + if cyc is None: + return 0 + b = self.get_batch(cyc) + if b is not None: + return b['start_time_ns'] + int(round(((cyc - b['start_cycles']) / b['freq_mhz']) * 1000)) + return int(round(((cyc - self.min_cyc) / self.default_freq) * 1000)) + + def dur_cycles_to_ns(self, cyc_start, cyc_dur): + if cyc_dur is None: + return 0 + freq = self.get_freq(cyc_start) + return int(round((cyc_dur / freq) * 1000)) + + +def parse_log(file_path, limit=None, device_filter=None, op_filter_re=None): try: if file_path != "-": f = open(file_path, 'r', encoding='utf-8', errors='ignore') @@ -67,14 +164,25 @@ def parse_log(file_path): all_ops: List[Dict[str, Any]] = [] all_traces: List[Dict[str, Any]] = [] current_op: Optional[Dict[str, Any]] = None - unwrapper = None - trace_unwrapper = None + ops_count_per_device = {} + if device_filter is not None: + for target in device_filter.split(','): + ops_count_per_device[target.strip()] = 0 + limit_reached = False + unwrappers = {} + last_batch_start = {} + trace_unwrappers = {} line_idx = 0 for line in f: line_idx += 1 - if "|" in line and "profile-op" in line: - parts = [p.strip() for p in line.split("|")] + if "profile-op" not in line and "trace-evt" not in line: + continue + device = extract_device(line) + + idx = line.find("profile-op") + if idx != -1 and "|" in line[idx:]: + parts = [p.strip() for p in line[idx:].split("|")] prefix = parts[0] prefix_match = re.search(r"profile-op\s+(?P[A-Z_0-9+]+)", prefix) if not prefix_match: @@ -115,14 +223,18 @@ def parse_log(file_path): if op_name == "OPBATCH": if cycles_start_raw: unwrapped_cycles_start = int(cycles_start_raw) - unwrapper = CycleUnwrapper(unwrapped_cycles_start) - trace_unwrapper = CycleUnwrapper(unwrapped_cycles_start) + unwrappers[device] = CycleUnwrapper(unwrapped_cycles_start) + last_batch_start[device] = unwrapped_cycles_start + for k in list(trace_unwrappers.keys()): + if k[0] == device: + del trace_unwrappers[k] else: - if cycles_start_raw and unwrapper is not None: - unwrapped_cycles_start = unwrapper.unwrap(int(cycles_start_raw)) + if cycles_start_raw: + device_unwrapper = unwrappers.get(device) + if device_unwrapper is not None: + unwrapped_cycles_start = device_unwrapper.unwrap(int(cycles_start_raw)) - idx = line.find("profile-op ") - op_text = line[idx + 11:].strip() if idx != -1 else line.strip() + op_text = re.sub(r"^profile-op\s+", "", line[idx:]).strip() if idx != -1 else line.strip() evt_str = None if types.startswith("evt-cnt "): @@ -142,24 +254,59 @@ def parse_log(file_path): 'cycles_start': int(cycles_start_raw) if cycles_start_raw else None, 'unwrapped_cycles_start': unwrapped_cycles_start, 'trace_events': [], - 'line_num': line_idx + 'line_num': line_idx, + 'device': device } all_ops.append(current_op) + + # Check if matching early exit criteria + matched = False + matched_target = None + if device_filter is not None: + targets = [t.strip() for t in device_filter.split(',')] + for target in targets: + if device == target or device.startswith(target + ":"): + matched = True + matched_target = target + break + else: + matched = True + matched_target = device + + if op_filter_re is not None and not op_filter_re.search(op_text): + matched = False + + if matched: + if matched_target not in ops_count_per_device: + ops_count_per_device[matched_target] = 0 + ops_count_per_device[matched_target] += 1 + + if limit is not None and len(ops_count_per_device) > 0 and all(count >= limit for count in ops_count_per_device.values()): + limit_reached = True + + if limit_reached and op_name == "OPBATCH": + break continue trace_match = trace_pattern.search(line) if trace_match: + thread = int(trace_match.group('thread')) raw_cyc = int(trace_match.group('cycles')) unwrapped_cyc = None - if trace_unwrapper is not None: - unwrapped_cyc = trace_unwrapper.unwrap(raw_cyc) + th_key = (device, thread) + if th_key not in trace_unwrappers: + batch_start = last_batch_start.get(device) + trace_unwrappers[th_key] = CycleUnwrapper(batch_start) + unwrapped_cyc = trace_unwrappers[th_key].unwrap(raw_cyc) all_traces.append({ - 'thread': int(trace_match.group('thread')), + 'thread': thread, 'event': trace_match.group('event'), 'info': int(trace_match.group('info')), 'cycles': raw_cyc, 'unwrapped_cycles': unwrapped_cyc, - 'state': trace_match.group('state') + 'state': trace_match.group('state'), + 'line_num': line_idx, + 'device': device }) f.close() @@ -274,27 +421,24 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): logger.warning("No operators found after filtering.") return - # Compute average frequency - frequencies = [] - for op in filtered_ops: - if op['usec'] > 0 and op['cycles'] > 0: - frequencies.append(op['cycles'] / op['usec']) - avg_freq_mhz = statistics.mean(frequencies) if frequencies else 1000.0 - if avg_freq_mhz <= 0: - avg_freq_mhz = 1000.0 - # Assign start and end cycles to each operator for op in filtered_ops: op['start_cycles'] = op['unwrapped_cycles_start'] - op['end_cycles'] = op['start_cycles'] + op['cycles'] + op['end_cycles'] = op['start_cycles'] + op['cycles'] if op['start_cycles'] is not None else None - global_min_cyc = min(op['start_cycles'] for op in filtered_ops if op['start_cycles'] is not None) + # Get list of unique devices present in the operations + unique_devices = sorted(list(set(op['device'] for op in filtered_ops))) + device_to_idx = {dev: idx for idx, dev in enumerate(unique_devices)} + time_mappers = {dev: DeviceTimeMapper(dev, filtered_ops) for dev in unique_devices} # Process events completed_events = [] if trace_events: trace_events = sorted(trace_events, key=lambda e: e['unwrapped_cycles']) - one_usec_cycles = max(avg_freq_mhz, 1.0) + + one_usec_cycles = {} + for dev in unique_devices: + one_usec_cycles[dev] = max(time_mappers[dev].get_freq(), 1.0) active_starts = {} for e in trace_events: @@ -303,31 +447,36 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): info = e['info'] state = e['state'] cyc = e['unwrapped_cycles'] + dev = e['device'] - key = (t, evt, info) + key = (dev, t, evt, info) if state == 'start': # Handle missing stop (start followed by another start) if key in active_starts: - prev_start = active_starts[key] + prev_e = active_starts[key] completed_events.append({ 'thread': t, 'event': evt, 'info': info, - 'start_cyc': prev_start, - 'end_cyc': prev_start + one_usec_cycles, + 'start_cyc': prev_e['unwrapped_cycles'], + 'end_cyc': prev_e['unwrapped_cycles'] + one_usec_cycles.get(dev, 1000.0), + 'line_num': prev_e.get('line_num'), 'missing_stop': True, + 'device': dev }) - active_starts[key] = cyc + active_starts[key] = e elif state == 'stop': if key in active_starts: - start_cyc = active_starts[key] + prev_e = active_starts[key] del active_starts[key] completed_events.append({ 'thread': t, 'event': evt, 'info': info, - 'start_cyc': start_cyc, + 'start_cyc': prev_e['unwrapped_cycles'], 'end_cyc': cyc, + 'line_num': prev_e.get('line_num'), + 'device': dev }) else: # Handle missing start (stop without start) @@ -335,31 +484,36 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): 'thread': t, 'event': evt, 'info': info, - 'start_cyc': cyc - one_usec_cycles, + 'start_cyc': cyc - one_usec_cycles.get(dev, 1000.0), 'end_cyc': cyc, + 'line_num': e.get('line_num'), 'missing_start': True, + 'device': dev }) # Clear remaining unmatched starts - for key, start_cyc in active_starts.items(): - t, evt, info = key + for key, prev_e in active_starts.items(): + dev, t, evt, info = key completed_events.append({ 'thread': t, 'event': evt, 'info': info, - 'start_cyc': start_cyc, - 'end_cyc': start_cyc + one_usec_cycles, + 'start_cyc': prev_e['unwrapped_cycles'], + 'end_cyc': prev_e['unwrapped_cycles'] + one_usec_cycles.get(dev, 1000.0), + 'line_num': prev_e.get('line_num'), 'missing_stop': True, + 'device': dev }) completed_events.sort(key=lambda e: e['start_cyc']) - # Convert event times to microseconds and apply clamp rounded to 1ns resolution (3 decimals) + # Convert event times to nanoseconds using per-device / per-batch time mapper for e in completed_events: - start_us = (e['start_cyc'] - global_min_cyc) / avg_freq_mhz - dur_us = (e['end_cyc'] - e['start_cyc']) / avg_freq_mhz - e['ts_ns'] = int(round(start_us * 1000)) - e['dur_ns'] = int(round(max(dur_us, 0.1) * 1000)) + dev = e['device'] + tm = time_mappers[dev] + e['ts_ns'] = tm.cycle_to_ns(e['start_cyc']) + dur_ns = tm.dur_cycles_to_ns(e['start_cyc'], e['end_cyc'] - e['start_cyc']) + e['dur_ns'] = max(dur_ns, 100) # Allocate slots (sub-tracks) to prevent overlaps on same virtual track active_slots = defaultdict(list) @@ -368,14 +522,15 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): evt = e['event'] ts = e['ts_ns'] dur = e['dur_ns'] + dev = e['device'] norm_evt = normalize_event_name(evt, e['info']) if norm_evt == "DMA": - track_key = (t, "DMA") + track_key = (dev, t, "DMA") elif t == 10: - track_key = (t, "HMX") + track_key = (dev, t, "HMX") else: - track_key = (t, "HVX") + track_key = (dev, t, "HVX") slots = active_slots[track_key] allocated_slot = -1 @@ -395,6 +550,7 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): t = e['thread'] evt = e['event'] slot = e['slot'] + dev = e['device'] norm_evt = normalize_event_name(evt, e['info']) if norm_evt == "DMA": @@ -408,56 +564,69 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): evt_id = 2 t_sort = 1 if t == 10 else t + 2 + dev_idx = device_to_idx[dev] + # Unique UUID for each sub-track if t == 10: - uuid = 20 # HMX thread track UUID + uuid = dev_idx * 10000000 + 20 # HMX thread track UUID else: - uuid = int(t_sort * 1000000 + evt_id * 1000 + slot) + uuid = int(dev_idx * 10000000 + t_sort * 1000000 + evt_id * 1000 + slot) e['uuid'] = uuid - used_tracks[uuid] = (t, track_evt, slot) + used_tracks[uuid] = (dev, t, track_evt, slot) with open(output_path, "wb") as f: - # Define Process with EXPLICIT child sorting - proc_desc = make_process_descriptor(1, "HTP NPU") - proc_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(1, process=proc_desc, child_ordering=3)) - write_trace_packet_to_file(f, proc_packet) + for dev in unique_devices: + dev_idx = device_to_idx[dev] + pid = dev_idx + 1 + proc_uuid = dev_idx * 10000000 + 1 - # Define Operators Track (UUID = 2) as a thread track at rank 1, tid 8 - op_thread_desc = make_thread_descriptor(1, 8, "Ops", sort_index=1) - op_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(2, parent_uuid=1, thread=op_thread_desc)) - write_trace_packet_to_file(f, op_packet) + # Define Process with EXPLICIT child sorting + proc_name = dev + proc_desc = make_process_descriptor(pid, proc_name) + proc_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(proc_uuid, process=proc_desc, child_ordering=3)) + write_trace_packet_to_file(f, proc_packet) - # Define HMX Thread Track (UUID = 20) at rank 2, tid 9 - hmx_thread_desc = make_thread_descriptor(1, 9, "HMX", sort_index=2) - hmx_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(20, parent_uuid=1, thread=hmx_thread_desc)) - write_trace_packet_to_file(f, hmx_packet) + # Define Operators Track as a thread track + op_track_uuid = dev_idx * 10000000 + 2 + op_tid = pid * 100 + 8 + op_thread_desc = make_thread_descriptor(pid, op_tid, "Ops", sort_index=1) + op_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(op_track_uuid, parent_uuid=proc_uuid, thread=op_thread_desc)) + write_trace_packet_to_file(f, op_packet) - # Define Thread Tracks (T0, T1, ..., T9) - unique_threads = sorted(list(set(t for (t, _, _) in used_tracks.values() if t != 10))) - for t in unique_threads: - thread_uuid = 10 + t - thread_name = f"T{t}" - # Sort order starts from index 3 (T0 -> 3, T1 -> 4, etc.) - sort_index = 3 + t - tid = 10 + t - thread_desc = make_thread_descriptor(1, tid, thread_name, sort_index=sort_index) - thread_packet = make_trace_packet(0, track_descriptor=make_track_descriptor( - thread_uuid, - parent_uuid=1, - thread=thread_desc, - sibling_order_rank=sort_index, - child_ordering=3 # Explicit child sorting for sub-tracks - )) - write_trace_packet_to_file(f, thread_packet) + # Define HMX Thread Track at rank 2 + hmx_track_uuid = dev_idx * 10000000 + 20 + hmx_tid = pid * 100 + 9 + hmx_thread_desc = make_thread_descriptor(pid, hmx_tid, "HMX", sort_index=2) + hmx_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(hmx_track_uuid, parent_uuid=proc_uuid, thread=hmx_thread_desc)) + write_trace_packet_to_file(f, hmx_packet) + + # Define Thread Tracks (T0, T1, ..., T9) for this device + dev_used_tracks = {uuid: val for uuid, val in used_tracks.items() if val[0] == dev} + unique_threads = sorted(list(set(t for (_, t, _, _) in dev_used_tracks.values() if t != 10))) + for t in unique_threads: + thread_uuid = dev_idx * 10000000 + 10 + t + thread_name = f"T{t}" + sort_index = 3 + t + tid = pid * 100 + 10 + t + thread_desc = make_thread_descriptor(pid, tid, thread_name, sort_index=sort_index) + thread_packet = make_trace_packet(0, track_descriptor=make_track_descriptor( + thread_uuid, + parent_uuid=proc_uuid, + thread=thread_desc, + sibling_order_rank=sort_index, + child_ordering=3 # Explicit child sorting for sub-tracks + )) + write_trace_packet_to_file(f, thread_packet) # Define Track descriptors for sub-tracks parented to thread tracks for uuid in sorted(used_tracks.keys()): - if uuid == 20: + dev, t, evt, slot = used_tracks[uuid] + dev_idx = device_to_idx[dev] + if t == 10: continue - t, evt, slot = used_tracks[uuid] name = f"T{t} {evt}" rank = 0 if evt == "HVX" else 1 - parent_thread_uuid = 10 + t + parent_thread_uuid = dev_idx * 10000000 + 10 + t # Sibling merge behavior: 1 (SIBLING_MERGE_BEHAVIOR_BY_TRACK_NAME) track_desc = make_track_descriptor( uuid=uuid, @@ -470,15 +639,18 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): write_trace_packet_to_file(f, track_packet) # Emit Operators - last_op_end_ns = 0 + last_op_end_ns = defaultdict(int) for op in filtered_ops: - op_start_ns = int(round(((op['start_cycles'] - global_min_cyc) / avg_freq_mhz) * 1000)) - op_dur_ns = int(round((op['cycles'] / avg_freq_mhz) * 1000)) + dev = op['device'] + dev_idx = device_to_idx[dev] + tm = time_mappers[dev] + op_start_ns = tm.cycle_to_ns(op['start_cycles']) + op_dur_ns = tm.dur_cycles_to_ns(op['start_cycles'], op['cycles']) if op['name'] != "OPBATCH": - if op_start_ns < last_op_end_ns: - op_start_ns = last_op_end_ns + if op_start_ns < last_op_end_ns[dev]: + op_start_ns = last_op_end_ns[dev] clamped_dur = max(op_dur_ns, 100) # Clamp to 100ns (0.1us) - last_op_end_ns = op_start_ns + clamped_dur + last_op_end_ns[dev] = op_start_ns + clamped_dur else: clamped_dur = max(op_dur_ns, 100) @@ -495,24 +667,41 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): if 'evt' in op and op['evt']: debug_annots.append(make_debug_annotation("evt", string_val=op['evt'])) + op_track_uuid = dev_idx * 10000000 + 2 + # Slice Begin - evt_begin = make_track_event(1, 2, name=f"{op['name']} ({op['dims']})", category="operator", debug_annotations=debug_annots) + evt_begin = make_track_event(1, op_track_uuid, name=f"{op['name']} ({op['dims']})", category="operator", debug_annotations=debug_annots) packet_begin = make_trace_packet(op_start_ns, track_event=evt_begin) write_trace_packet_to_file(f, packet_begin) # Slice End - evt_end = make_track_event(2, 2) + evt_end = make_track_event(2, op_track_uuid) packet_end = make_trace_packet(op_start_ns + clamped_dur, track_event=evt_end) write_trace_packet_to_file(f, packet_end) # Emit Thread Trace Events for e in completed_events: norm_name = normalize_event_name(e['event'], e['info']) - name = f"DMA {e['info']}" if norm_name == "DMA" else norm_name + if norm_name == "DMA": + name = f"DMA {e['info']}" + elif norm_name == "FENCE": + name = f"FENCE {e['info']}" if e.get('info') is not None and e['info'] != 0 else "FENCE" + else: + name = norm_name + if e.get('missing_start') or e.get('missing_stop'): name += "!" debug_annots = [] + if 'line_num' in e and e['line_num'] is not None: + debug_annots.append(make_debug_annotation("line", int_val=e['line_num'])) + if norm_name == "FENCE" and e.get('info') is not None: + debug_annots.append(make_debug_annotation("seq", int_val=e['info'])) + elif norm_name == "DMA" and e.get('info') is not None: + debug_annots.append(make_debug_annotation("channel", int_val=e['info'])) + elif e.get('info') is not None and e['info'] != 0: + debug_annots.append(make_debug_annotation("info", int_val=e['info'])) + if e.get('missing_start'): debug_annots.append(make_debug_annotation("missing_start", string_val="true")) if e.get('missing_stop'): @@ -536,6 +725,7 @@ def main(): parser.add_argument("logfile", help="Path to hex-log profile file") parser.add_argument("-o", "--output", default="optrace.perfetto-trace", help="Output trace file path (default: optrace.perfetto-trace)") parser.add_argument("--filter", type=str, help="Regex filter matching against the original profile-op line") + parser.add_argument("--device", type=str, help="Device to filter by (e.g. HTP0, HTP0:0) or 'split' to generate separate files per device") group = parser.add_mutually_exclusive_group() group.add_argument("--head", type=int, help="Limit to first N ops") @@ -544,7 +734,21 @@ def main(): args = parser.parse_args() logging.basicConfig(level=logging.INFO, format='%(message)s') - ops, traces = parse_log(args.logfile) + op_filter_re = None + if args.filter: + try: + op_filter_re = re.compile(args.filter) + except re.error as e: + logger.error(f"Invalid regex filter: {e}") + sys.exit(1) + + limit = args.head if args.head is not None else None + device_filter = args.device if (args.device and args.device != "split") else None + ops, traces = parse_log(args.logfile, limit=limit, device_filter=device_filter, op_filter_re=op_filter_re) + + if args.device and args.device != "split": + ops = [op for op in ops if device_matches(op['device'], args.device)] + traces = [t for t in traces if device_matches(t['device'], args.device)] if args.filter: try: @@ -554,35 +758,60 @@ def main(): sys.exit(1) ops = [op for op in ops if filter_re.search(op['op_text'])] - if args.head is not None: - ops = ops[:args.head] - elif args.tail is not None: - ops = ops[-args.tail:] + if args.head is not None or args.tail is not None: + ops_by_dev = defaultdict(list) + for op in ops: + ops_by_dev[op['device']].append(op) + + filtered_ops = [] + for dev in sorted(ops_by_dev.keys()): + dev_ops = ops_by_dev[dev] + if args.head is not None: + dev_ops = dev_ops[:args.head] + elif args.tail is not None: + dev_ops = dev_ops[-args.tail:] + filtered_ops.extend(dev_ops) + ops = filtered_ops if args.filter or args.head is not None or args.tail is not None: - valid_ranges = [] + # Group valid ranges by device + valid_ranges_by_dev = defaultdict(list) for op in ops: start_cyc = op['unwrapped_cycles_start'] end_cyc = start_cyc + op['cycles'] if start_cyc is not None else None if start_cyc is not None and end_cyc is not None: - valid_ranges.append((start_cyc, end_cyc)) + valid_ranges_by_dev[op['device']].append((start_cyc, end_cyc)) - valid_ranges.sort(key=lambda r: r[0]) - range_starts = [r[0] for r in valid_ranges] + for dev in valid_ranges_by_dev: + valid_ranges_by_dev[dev].sort(key=lambda r: r[0]) + + range_starts_by_dev = {dev: [r[0] for r in ranges] for dev, ranges in valid_ranges_by_dev.items()} filtered_traces = [] for e in traces: cyc = e['unwrapped_cycles'] if cyc is None: continue + dev = e['device'] + range_starts = range_starts_by_dev.get(dev) + if not range_starts: + continue idx = bisect.bisect_right(range_starts, cyc) - 1 if idx >= 0: - start, end = valid_ranges[idx] + start, end = valid_ranges_by_dev[dev][idx] if start <= cyc <= end: filtered_traces.append(e) traces = filtered_traces - generate_perfetto_trace(ops, traces, args.output) + if args.device == "split": + unique_devices = sorted(list(set(op['device'] for op in ops))) + for dev in unique_devices: + dev_ops = [op for op in ops if device_matches(op['device'], dev)] + dev_traces = [t for t in traces if device_matches(t['device'], dev)] + out_path = get_split_output_path(args.output, dev) + generate_perfetto_trace(dev_ops, dev_traces, out_path) + else: + generate_perfetto_trace(ops, traces, args.output) if __name__ == "__main__": diff --git a/scripts/snapdragon/run.py b/scripts/snapdragon/run.py new file mode 100755 index 000000000..d639b16b8 --- /dev/null +++ b/scripts/snapdragon/run.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +# +# Run llama.cpp tools on Snapdragon devices (natively, via ADB, or SSH). +# + +import sys +import os +import argparse +import subprocess +import platform +import shlex +import logging + +logger = logging.getLogger("run") + + +def parse_target(target_str): + if not target_str: + return None, None + if target_str.startswith("adb") or target_str.startswith("android"): + parts = target_str.split(":", 1) + serial = parts[1] if len(parts) > 1 else None + return "android", serial + elif target_str.startswith("lnx") or target_str.startswith("linux") or target_str.startswith("ubuntu"): + parts = target_str.split(":", 1) + host = parts[1] if len(parts) > 1 else None + return "linux", host + elif target_str in ("wos", "windows"): + return "windows", None + else: + return None, None + + +def shlex_join(args_list): + if hasattr(shlex, 'join'): + return shlex.join(args_list) + import pipes + return " ".join(pipes.quote(x) for x in args_list) + + +def main(): + logging.basicConfig(level=logging.INFO, format='%(message)s') + # Split arguments at '--' + if '--' in sys.argv: + idx = sys.argv.index('--') + run_args = sys.argv[1:idx] + cmd_args = sys.argv[idx + 1:] + else: + run_args = sys.argv[1:] + cmd_args = [] + + parser = argparse.ArgumentParser( + description="Unified runner for llama.cpp tools on Snapdragon (natively, via ADB, or via SSH)." + ) + parser.add_argument("--target", help="Execution target (e.g. android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, windows/wos) (default: local run)") + parser.add_argument("--target-dir", help="Target directory on the device (default: /data/local/tmp/llama.cpp for Android, ~/llama.cpp for Linux)") + parser.add_argument("--install-dir", help="Install directory name (defaults to pkg-TARGET or pkg-TARGET-dbg prefix based on target)") + parser.add_argument("--debug", action="store_true", help="Use debug build (defaults to pkg-TARGET-dbg folder)") + parser.add_argument("--devices", "--device", "-d", help="Select execution devices (split into NPU and OpenCL GPUs automatically, default: HTP0)") + parser.add_argument("--verbose", help="Verbose level (enables both Hexagon and OpenCL kernel cache debugging)") + parser.add_argument("--profile", help="Profiling flag (enables Hexagon profiling and OpenCL autotuning)") + parser.add_argument("--sched-debug", action="store_true", help="Enable GGML/llama.cpp scheduler debug output (GGML_SCHED_DEBUG=2)") + parser.add_argument("--mtmd-device", help="Specify the backend device ID for Multi-Threaded Multi-Device setup (MTMD_BACKEND_DEVICE)") + + # Hexagon specific parameters + parser.add_argument("--hex-verbose", help="Enable verbose logging (GGML_HEXAGON_VERBOSE)") + parser.add_argument("--hex-profile", help="Enable NPU/Hexagon profiling and performance metrics print (GGML_HEXAGON_PROFILE)") + parser.add_argument("--hex-nhvx", help="Number of HVX units to use (GGML_HEXAGON_NHVX)") + parser.add_argument("--hex-nhmx", help="Number of HMX units to use. 0 disables HMX power-up (GGML_HEXAGON_NHMX)") + parser.add_argument("--hex-hostbuf", help="Enable host buffers (GGML_HEXAGON_HOSTBUF)") + parser.add_argument("--hex-opbatch", help="Maximum number of operations to batch into a single HTP execution (GGML_HEXAGON_OPBATCH)") + parser.add_argument("--hex-opqueue", help="Size of the asynchronous NPU operation queue (GGML_HEXAGON_OPQUEUE)") + parser.add_argument("--hex-oppoll", default="1", help="Enable (1) or Disable (0) polling for NPU opbatch completion (GGML_HEXAGON_OPPOLL) (default: 1)") + parser.add_argument("--hex-opfilter", help="Regex pattern to filter/select which operators are offloaded to NPU (GGML_HEXAGON_OPFILTER)") + parser.add_argument("--hex-opfusion", help="NPU graph node fusion optimization level (0: disabled, 1: enabled) (GGML_HEXAGON_OPFUSION)") + parser.add_argument("--hex-vmem", help="Maximum NPU VMEM size limit in MB to allocate (GGML_HEXAGON_VMEM)") + parser.add_argument("--hex-mbuf", help="Maximum host buffer size limit in MB to allocate (GGML_HEXAGON_MBUF)") + parser.add_argument("--hex-mm-select", help="Select MUL_MAT and MUL_MAT_ID kernel (GGML_HEXAGON_MM_SELECT) 3:HMX,2:HVX-tiled,1:HVX-flat,0:disable") + parser.add_argument("--hex-fa-select", help="Select Flash Attention kernel (GGML_HEXAGON_FA_SELECT) 2:HMX,1:HVX,0:disable") + parser.add_argument("--hex-ar-select", help="Select All-Reduce kernel (GGML_HEXAGON_AR_SELECT) 1:enable,0:disable") + parser.add_argument("--hex-etm", help="Enable Embedded Trace Macrocell hardware tracing / trace logging (GGML_HEXAGON_ETM)") + parser.add_argument("--hex-arch", help="Target Hexagon NPU architecture version override (v73, v75, v79, v81, etc.) (GGML_HEXAGON_ARCH)") + parser.add_argument("--hex-optrace", help="Trace buffer size in number of records (GGML_HEXAGON_OPTRACE)") + + # OpenCL specific parameters + parser.add_argument("--cl-platform", help="Select OpenCL platform name/regex (e.g. Qualified Qualcomm OpenCL platform) (GGML_OPENCL_PLATFORM)") + parser.add_argument("--cl-device", help="Select OpenCL device name/regex (e.g. Adreno GPU) (GGML_OPENCL_DEVICE)") + parser.add_argument("--cl-opfilter", help="Regex pattern to filter/select which operators are offloaded to OpenCL (GGML_OPENCL_OPFILTER)") + parser.add_argument("--cl-disable-fusion", action="store_true", help="Disable OpenCL kernel fusion optimizations (GGML_OPENCL_DISABLE_FUSION)") + parser.add_argument("--cl-cache-dir", help="Directory path to store compiled OpenCL program binaries (GGML_OPENCL_KERNEL_CACHE_DIR)") + parser.add_argument("--cl-cache-debug", help="Enable verbose debugging logs for the kernel caching system (GGML_OPENCL_KERNEL_CACHE_DEBUG)") + parser.add_argument("--cl-fa-tune", action="store_true", help="Enable automatic Flash Attention kernel autotuning (GGML_OPENCL_FA_TUNE)") + parser.add_argument("--cl-adreno-xmem", action="store_true", help="Enforce matmul using texture/image (xmem) memory paths on Adreno GPUs (GGML_OPENCL_ADRENO_XMEM_GEMM)") + parser.add_argument("--cl-adreno-large-buffer", action="store_true", help="Allow allocating larger buffer sizes on Adreno GPUs (GGML_OPENCL_ADRENO_USE_LARGE_BUFFER)") + + args = parser.parse_args(run_args) + + if not cmd_args: + parser.print_help() + logger.error("\nError: No command specified after '--'") + sys.exit(1) + + target_type = None + target_val = None + target_prefix = None + if args.target: + target_type, target_val = parse_target(args.target) + if not target_type: + logger.error(f"Error: Invalid target format '{args.target}'. Must be android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, or windows/wos.") + sys.exit(1) + target_prefix = args.target.split(":", 1)[0] + + # Resolve install directory + install_dir = args.install_dir + if not install_dir: + if target_prefix: + suffix = "-dbg" if args.debug else "" + install_dir = f"pkg-{target_prefix}{suffix}" + else: + # Smart branch folder detection for local run if default is not set + prefixes = ("wos", "windows", "lnx", "linux", "ubuntu", "adb", "android") + suffixes = ("-dbg", "") if args.debug else ("", "-dbg") + found = False + for suffix in suffixes: + for prefix in prefixes: + test_path = f"./pkg-{prefix}{suffix}/llama.cpp" + if os.path.exists(test_path): + install_dir = f"pkg-{prefix}{suffix}" + found = True + break + if found: + break + if not install_dir: + install_dir = "pkg-android" # Fallback default + + # Host side package path + package_path = os.path.join(install_dir, "llama.cpp") + + # Environment variables to map + env_vars = {} + + def set_env(env_name, opt_val): + if opt_val is not None: + env_vars[env_name] = str(opt_val) + elif env_name in os.environ: + env_vars[env_name] = os.environ[env_name] + + # Resolve and filter devices (HTP vs OpenCL) + devices_val = args.devices if args.devices is not None else "HTP0" + if devices_val.isdigit(): + hex_devices = devices_val + cl_device = "" + else: + parts = [p.strip() for p in devices_val.split(",")] + # Any device containing "htp" is Hexagon, rest is OpenCL + hex_parts = [p for p in parts if "htp" in p.lower()] + cl_parts = [p for p in parts if "htp" not in p.lower()] + hex_devices = ",".join(hex_parts) + cl_device = ",".join(cl_parts) + + # Set Hexagon devices + if hex_devices: + env_vars["GGML_HEXAGON_DEVICES"] = hex_devices + elif "GGML_HEXAGON_DEVICES" in os.environ: + env_vars["GGML_HEXAGON_DEVICES"] = os.environ["GGML_HEXAGON_DEVICES"] + + # Set OpenCL device (unless overridden by --cl-device) + final_cl_device = args.cl_device if args.cl_device is not None else cl_device + if final_cl_device: + env_vars["GGML_OPENCL_DEVICE"] = final_cl_device + elif "GGML_OPENCL_DEVICE" in os.environ: + env_vars["GGML_OPENCL_DEVICE"] = os.environ["GGML_OPENCL_DEVICE"] + + # Map shared & backend-specific parameters with correct overrides + + # Verbose logging mapping + hex_verbose_val = args.hex_verbose if args.hex_verbose is not None else args.verbose + set_env("GGML_HEXAGON_VERBOSE", hex_verbose_val) + + cl_cache_debug_val = args.cl_cache_debug if args.cl_cache_debug is not None else args.verbose + set_env("GGML_OPENCL_KERNEL_CACHE_DEBUG", cl_cache_debug_val) + + # Profiling mapping + hex_profile_val = args.hex_profile if args.hex_profile is not None else args.profile + set_env("GGML_HEXAGON_PROFILE", hex_profile_val) + + if args.cl_fa_tune or args.profile is not None: + env_vars["GGML_OPENCL_FA_TUNE"] = "1" + elif "GGML_OPENCL_FA_TUNE" in os.environ: + env_vars["GGML_OPENCL_FA_TUNE"] = os.environ["GGML_OPENCL_FA_TUNE"] + + # Other Hexagon environment variables + set_env("GGML_HEXAGON_NHVX", args.hex_nhvx) + set_env("GGML_HEXAGON_NHMX", args.hex_nhmx) + set_env("GGML_HEXAGON_HOSTBUF", args.hex_hostbuf) + set_env("GGML_HEXAGON_OPBATCH", args.hex_opbatch) + set_env("GGML_HEXAGON_OPQUEUE", args.hex_opqueue) + set_env("GGML_HEXAGON_OPPOLL", args.hex_oppoll) + set_env("GGML_HEXAGON_OPFILTER", args.hex_opfilter) + set_env("GGML_HEXAGON_OPFUSION", args.hex_opfusion) + set_env("GGML_HEXAGON_VMEM", args.hex_vmem) + set_env("GGML_HEXAGON_MBUF", args.hex_mbuf) + set_env("GGML_HEXAGON_MM_SELECT", args.hex_mm_select) + set_env("GGML_HEXAGON_FA_SELECT", args.hex_fa_select) + set_env("GGML_HEXAGON_AR_SELECT", args.hex_ar_select) + set_env("GGML_HEXAGON_ETM", args.hex_etm) + set_env("GGML_HEXAGON_ARCH", args.hex_arch) + set_env("GGML_HEXAGON_OPTRACE", args.hex_optrace) + set_env("MTMD_BACKEND_DEVICE", args.mtmd_device) + + # OpenCL environment variables + set_env("GGML_OPENCL_PLATFORM", args.cl_platform) + set_env("GGML_OPENCL_OPFILTER", args.cl_opfilter) + set_env("GGML_OPENCL_KERNEL_CACHE_DIR", args.cl_cache_dir) + + if args.cl_disable_fusion: + env_vars["GGML_OPENCL_DISABLE_FUSION"] = "1" + elif "GGML_OPENCL_DISABLE_FUSION" in os.environ: + env_vars["GGML_OPENCL_DISABLE_FUSION"] = os.environ["GGML_OPENCL_DISABLE_FUSION"] + + if args.cl_adreno_xmem: + env_vars["GGML_OPENCL_ADRENO_XMEM_GEMM"] = "1" + elif "GGML_OPENCL_ADRENO_XMEM_GEMM" in os.environ: + env_vars["GGML_OPENCL_ADRENO_XMEM_GEMM"] = os.environ["GGML_OPENCL_ADRENO_XMEM_GEMM"] + + if args.cl_adreno_large_buffer: + env_vars["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] = "1" + elif "GGML_OPENCL_ADRENO_USE_LARGE_BUFFER" in os.environ: + env_vars["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] = os.environ["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] + + if args.sched_debug: + env_vars["GGML_SCHED_DEBUG"] = "2" + + # Resolve executable path + executable = cmd_args[0] + known_binaries = ["llama-cli", "llama-bench", "llama-completion", "llama-mtmd-cli", "test-backend-ops"] + if executable in known_binaries: + if target_type in ("android", "linux"): + resolved_exec = f"./bin/{executable}" + else: + if platform.system() == "Windows": + resolved_exec = os.path.normpath(os.path.join(package_path, "bin", f"{executable}.exe")) + else: + resolved_exec = os.path.normpath(os.path.join(package_path, "bin", executable)) + cmd_args[0] = resolved_exec + + # Infer device string to pass to the tool + basename = os.path.basename(executable) + if basename.endswith(".exe"): + basename = basename[:-4] + + device_val = None + if basename == "test-backend-ops": + for i in range(len(cmd_args)): + if cmd_args[i] in ("-p", "--params") and i + 1 < len(cmd_args): + val = cmd_args[i + 1] + new_val = "" + for j, char in enumerate(val): + if char in ('[', ']'): + if j > 0 and val[j - 1] == '\\': + new_val += char + else: + new_val += '\\' + char + else: + new_val += char + cmd_args[i + 1] = new_val + + has_b = any(arg == "-b" for arg in cmd_args) + if not has_b: + if args.devices: + if args.devices.isdigit(): + n = int(args.devices) + device_val = ",".join(f"HTP{i}" for i in range(n)) + else: + device_val = args.devices + elif "D" in os.environ: + device_val = os.environ["D"] + elif "DEVICE" in os.environ: + device_val = os.environ["DEVICE"] + else: + device_val = "HTP0" + if device_val: + cmd_args += ["-b", device_val] + else: + has_device = any(arg.startswith("--device") for arg in cmd_args) + if not has_device: + if args.devices: + if args.devices.isdigit(): + n = int(args.devices) + device_val = ",".join(f"HTP{i}" for i in range(n)) + else: + device_val = args.devices + elif "D" in os.environ: + device_val = os.environ["D"] + elif "DEVICE" in os.environ: + device_val = os.environ["DEVICE"] + else: + device_val = "HTP0" + if device_val: + cmd_args += ["--device", device_val] + + # Automatically add -v to known llama tools if sched-debug, verbose, or profile are set + verbose_trigger = ( + args.sched_debug + or args.verbose is not None + or args.profile is not None + or args.hex_verbose is not None + or args.hex_profile is not None + or args.hex_optrace is not None + ) + if verbose_trigger and basename in ("llama-cli", "llama-completion", "llama-bench", "llama-server", "llama-mtmd-cli"): + if "-v" not in cmd_args and "--verbose" not in cmd_args: + cmd_args.append("-v") + + # Inject defaults for llama-cli, llama-completion, and llama-server if not overridden by the user + if basename in ("llama-cli", "llama-completion", "llama-server"): + if "-ngl" not in cmd_args and "--n-gpu-layers" not in cmd_args: + cmd_args += ["-ngl", "99"] + if "--ubatch-size" not in cmd_args and "-ub" not in cmd_args: + cmd_args += ["--ubatch-size", "1024"] + if "-fa" not in cmd_args and "--flash-attn" not in cmd_args: + cmd_args += ["-fa", "on"] + + if basename in ("llama-cli", "llama-completion", "llama-server", "llama-bench"): + if "-t" not in cmd_args and "--threads" not in cmd_args: + cmd_args += ["-t", "6"] + + # Resolve target directory on device + target_dir = args.target_dir + if not target_dir: + target_dir = "/data/local/tmp/llama.cpp" if target_type == "android" else "~/llama.cpp" + + if target_type == "android": + # Run via ADB + adb_base = ["adb"] + if target_val: # serial + adb_base += ["-s", target_val] + + env_parts = [ + "LD_LIBRARY_PATH=./lib", + "ADSP_LIBRARY_PATH=./lib" + ] + for k, v in env_vars.items(): + env_parts.append(f"{k}={v}") + env_str = " ".join(env_parts) + + cmd_str = shlex_join(cmd_args) + adb_shell_cmd = f"cd {target_dir} && ulimit -c unlimited && {env_str} {cmd_str}" + full_cmd = adb_base + ["shell", adb_shell_cmd] + + logger.info(f"+ {' '.join(full_cmd)}") + res = subprocess.run(full_cmd) + sys.exit(res.returncode) + + elif target_type == "linux": + ssh_host = target_val + if not ssh_host: + logger.error("Error: SSH host not specified in target (e.g. use linux:user@host, lnx:user@host, or ubuntu:user@host). Cannot execute.") + sys.exit(1) + + # Linux remote run via SSH + env_parts = [ + "LD_LIBRARY_PATH=./lib", + "ADSP_LIBRARY_PATH=./lib" + ] + for k, v in env_vars.items(): + env_parts.append(f"{k}={v}") + env_str = " ".join(env_parts) + + cmd_str = shlex_join(cmd_args) + ssh_shell_cmd = f"cd {target_dir} && ulimit -c unlimited && {env_str} {cmd_str}" + full_cmd = ["ssh", ssh_host, ssh_shell_cmd] + + logger.info(f"+ {' '.join(full_cmd)}") + res = subprocess.run(full_cmd) + sys.exit(res.returncode) + + elif target_type == "windows": + logger.info("Windows target execution is currently a stub.") + sys.exit(0) + + else: + # Run locally + local_env = os.environ.copy() + lib_dir = os.path.normpath(os.path.join(package_path, "lib")) + local_env["ADSP_LIBRARY_PATH"] = lib_dir + if platform.system() == "Windows": + local_env["PATH"] = lib_dir + os.path.pathsep + local_env.get("PATH", "") + else: + local_env["LD_LIBRARY_PATH"] = lib_dir + os.path.pathsep + local_env.get("LD_LIBRARY_PATH", "") + + for k, v in env_vars.items(): + local_env[k] = v + + logger.info(f"+ {shlex_join(cmd_args)}") + res = subprocess.run(cmd_args, env=local_env) + sys.exit(res.returncode) + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + logger.info("\nInterrupted by user.") + sys.exit(130) diff --git a/scripts/snapdragon/windows/run-bench.ps1 b/scripts/snapdragon/windows/run-bench.ps1 deleted file mode 100644 index 6eb656e66..000000000 --- a/scripts/snapdragon/windows/run-bench.ps1 +++ /dev/null @@ -1,48 +0,0 @@ - -#!/usr/bin/env pwsh - -# Basedir on device -$basedir=".\pkg-snapdragon" - -$cli_opts=$args - -$model="Llama-3.2-3B-Instruct-Q4_0.gguf" -if ($null -ne $env:M) { - $model=$env:M -} - -$device="HTP0" -if ($null -ne $env:D) { - $device=$env:D -} - -if ($null -ne $env:V) { - $env:GGML_HEXAGON_VERBOSE=$env:V -} - -if ($null -ne $env:PROF) { - $env:GGML_HEXAGON_PROFILE=$env:PROF -} - -if ($null -ne $env:OPSTAGE) { - $env:GGML_HEXAGON_OPSTAGE=$env:OPSTAGE -} - -if ($null -ne $env:NHVX) { - $env:GGML_HEXAGON_NHVX=$env:NHVX -} - -if ($null -ne $env:NDEV) { - $env:GGML_HEXAGON_NDEV=$env:NDEV -} - -if ($null -ne $env:HB) { - $env:GGML_HEXAGON_HOSTBUF=$env:HB -} - -$env:ADSP_LIBRARY_PATH="$basedir\lib" - -& "$basedir\bin\llama-bench.exe" ` - --load-mode none -m $basedir\..\..\gguf\$model ` - --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 ` - --ubatch-size 1024 -ngl 99 --device $device $cli_opts diff --git a/scripts/snapdragon/windows/run-cli.ps1 b/scripts/snapdragon/windows/run-cli.ps1 deleted file mode 100644 index 5da8bff33..000000000 --- a/scripts/snapdragon/windows/run-cli.ps1 +++ /dev/null @@ -1,53 +0,0 @@ - -#!/usr/bin/env pwsh - -# Basedir on device -$basedir=".\pkg-snapdragon" - -$cli_opts=$args - -$model="Llama-3.2-3B-Instruct-Q4_0.gguf" -if ($null -ne $env:M) { - $model=$env:M -} - -$device="HTP0" -if ($null -ne $env:D) { - $device=$env:D -} - -if ($null -ne $env:V) { - $env:GGML_HEXAGON_VERBOSE=$env:V -} - -if ($null -ne $env:SCHED) { - $env:GGML_SCHED_DEBUG=$env:SCHED; $cli_opts="$cli_opts -v" -} - -if ($null -ne $env:PROF) { - $env:GGML_HEXAGON_PROFILE=$env:PROF -} - -if ($null -ne $env:OPSTAGE) { - $env:GGML_HEXAGON_OPSTAGE=$env:OPSTAGE -} - -if ($null -ne $env:NHVX) { - $env:GGML_HEXAGON_NHVX=$env:NHVX -} - -if ($null -ne $env:NDEV) { - $env:GGML_HEXAGON_NDEV=$env:NDEV -} - -if ($null -ne $env:HB) { - $env:GGML_HEXAGON_HOSTBUF=$env:HB -} - -$env:ADSP_LIBRARY_PATH="$basedir\lib" - -& "$basedir\bin\llama-cli.exe" ` - --load-mode none -m $basedir\..\..\gguf\$model ` - --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 ` - --ctx-size 8192 --ubatch-size 1024 -fa on ` - -ngl 99 --device $device $cli_opts diff --git a/scripts/snapdragon/windows/run-completion.ps1 b/scripts/snapdragon/windows/run-completion.ps1 deleted file mode 100644 index 08ef139b7..000000000 --- a/scripts/snapdragon/windows/run-completion.ps1 +++ /dev/null @@ -1,53 +0,0 @@ - -#!/usr/bin/env pwsh - -# Basedir on device -$basedir=".\pkg-snapdragon" - -$cli_opts=$args - -$model="Llama-3.2-3B-Instruct-Q4_0.gguf" -if ($null -ne $env:M) { - $model=$env:M -} - -$device="HTP0" -if ($null -ne $env:D) { - $device=$env:D -} - -if ($null -ne $env:V) { - $env:GGML_HEXAGON_VERBOSE=$env:V -} - -if ($null -ne $env:SCHED) { - $env:GGML_SCHED_DEBUG=$env:SCHED; $cli_opts="$cli_opts -v" -} - -if ($null -ne $env:PROF) { - $env:GGML_HEXAGON_PROFILE=$env:PROF -} - -if ($null -ne $env:OPSTAGE) { - $env:GGML_HEXAGON_OPSTAGE=$env:OPSTAGE -} - -if ($null -ne $env:NHVX) { - $env:GGML_HEXAGON_NHVX=$env:NHVX -} - -if ($null -ne $env:NDEV) { - $env:GGML_HEXAGON_NDEV=$env:NDEV -} - -if ($null -ne $env:HB) { - $env:GGML_HEXAGON_HOSTBUF=$env:HB -} - -$env:ADSP_LIBRARY_PATH="$basedir\lib" - -& "$basedir\bin\llama-completion.exe" ` - --load-mode none -m $basedir\..\..\gguf\$model ` - --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 ` - --ctx-size 8192 --ubatch-size 1024 -fa on ` - -ngl 99 -no-cnv --device $device $cli_opts diff --git a/scripts/snapdragon/windows/run-mtmd.ps1 b/scripts/snapdragon/windows/run-mtmd.ps1 deleted file mode 100644 index 6e270ec90..000000000 --- a/scripts/snapdragon/windows/run-mtmd.ps1 +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env pwsh - -# Basedir on device -$basedir=".\pkg-snapdragon" - -$cli_opts=$args - -$model="gemma-3-4b-it-Q4_0.gguf" -if ($null -ne $env:M) { - $model=$env:M -} - -$mmproj="mmproj-F16.gguf" -if ($null -ne $env:MMPROJ) { - $mmproj=$env:MMPROJ -} - -$image="" -if ($null -ne $env:IMG) { - $image=$env:IMG -} - -$device="HTP0" -if ($null -ne $env:D) { - $device=$env:D -} - -if ($null -ne $env:V) { - $env:GGML_HEXAGON_VERBOSE=$env:V -} - -if ($null -ne $env:SCHED) { - $env:GGML_SCHED_DEBUG=$env:SCHED; $cli_opts="$cli_opts -v" -} - -if ($null -ne $env:PROF) { - $env:GGML_HEXAGON_PROFILE=$env:PROF -} - -if ($null -ne $env:OPSTAGE) { - $env:GGML_HEXAGON_OPSTAGE=$env:OPSTAGE -} - -if ($null -ne $env:NHVX) { - $env:GGML_HEXAGON_NHVX=$env:NHVX -} - -if ($null -ne $env:NDEV) { - $env:GGML_HEXAGON_NDEV=$env:NDEV -} - -if ($null -ne $env:HB) { - $env:GGML_HEXAGON_HOSTBUF=$env:HB -} - -if ($null -ne $env:MTMD_DEVICE) { - $env:MTMD_BACKEND_DEVICE=$env:MTMD_DEVICE -} - -$env:ADSP_LIBRARY_PATH="$basedir\lib" - -& "$basedir\bin\llama-mtmd-cli.exe" ` - --load-mode none -m $basedir\..\..\gguf\$model ` - --mmproj $basedir\..\..\gguf\$mmproj ` - --image $basedir\..\..\gguf\$image ` - --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 ` - --ctx-size 8192 --ubatch-size 1024 -fa on ` - -ngl 99 --device $device -v $cli_opts diff --git a/scripts/snapdragon/windows/run-tool.ps1 b/scripts/snapdragon/windows/run-tool.ps1 deleted file mode 100644 index 15c880f2d..000000000 --- a/scripts/snapdragon/windows/run-tool.ps1 +++ /dev/null @@ -1,56 +0,0 @@ - -#!/usr/bin/env pwsh - -# Basedir on device -$basedir=".\pkg-snapdragon" - -if ($args.Count -eq 0) { - Write-Host "No arguments provided.Expected the tool and argument to run." - exit -1 -} - -$tool=$args[0] -$cli_opts=@() - -if ($args.Count -gt 1) { - $cli_opts=$args[1..($args.Count - 1)] - $remainingArgs = $args[1..($args.Count - 1)] -} - -$device="HTP0" -if ($null -ne $env:D) { - $device=$env:D -} - -if ($null -ne $env:V) { - $env:GGML_HEXAGON_VERBOSE=$env:V -} - -if ($null -ne $env:SCHED) { - $env:GGML_SCHED_DEBUG=$env:SCHED; $cli_opts="$cli_opts -v" -} - -if ($null -ne $env:PROF) { - $env:GGML_HEXAGON_PROFILE=$env:PROF -} - -if ($null -ne $env:OPSTAGE) { - $env:GGML_HEXAGON_OPSTAGE=$env:OPSTAGE -} - -if ($null -ne $env:NHVX) { - $env:GGML_HEXAGON_NHVX=$env:NHVX -} - -if ($null -ne $env:NDEV) { - $env:GGML_HEXAGON_NDEV=$env:NDEV -} - -if ($null -ne $env:HB) { - $env:GGML_HEXAGON_HOSTBUF=$env:HB -} - -$env:ADSP_LIBRARY_PATH="$basedir\lib" - -& "$basedir\bin\$tool" ` - $cli_opts diff --git a/scripts/snapdragon/windows/setup-build.ps1 b/scripts/snapdragon/windows/setup-build.ps1 deleted file mode 100644 index d8ef24d44..000000000 --- a/scripts/snapdragon/windows/setup-build.ps1 +++ /dev/null @@ -1,105 +0,0 @@ -# Requires Run as Administrator is NOT strictly necessary for User-scope env vars, -# but recommended for creating directories in C:\ root if permissions are restricted. - -$ErrorActionPreference = "Stop" - -# --- Configuration --- -$BaseDir = "C:\Qualcomm" - -# SDK 1: Hexagon -$HexagonUrl = "https://github.com/snapdragon-toolchain/hexagon-sdk/releases/download/v6.6.0.0/hexagon-sdk-v6.6.0.0-arm64-wos.tar.xz" -$HexagonParent = Join-Path $BaseDir "Hexagon_SDK" -$HexagonSdkVersion = "6.6.0.0" -$HexagonToolsVersion = "19.0.07" -$HexagonSdkTarget = Join-Path $HexagonParent $HexagonSdkVersion -$HexagonToolsTarget = Join-Path $HexagonSdkTarget "\tools\HEXAGON_Tools\$HexagonToolsVersion" - -# SDK 2: OpenCL -$OpenCLUrl = "https://github.com/snapdragon-toolchain/opencl-sdk/releases/download/v2.3.2/adreno-opencl-sdk-v2.3.2-arm64-wos.tar.xz" -$OpenCLParent = Join-Path $BaseDir "OpenCL_SDK" -$OpenCLVersion = "2.3.2" -$OpenCLTarget = Join-Path $OpenCLParent $OpenCLVersion - -# --- Helper Function --- -function Install-QualcommSDK { - param ( - [string]$Url, - [string]$ParentDir, - [string]$TargetDir, - [string]$Name - ) - - # 1. Create Parent Directory - if (-not (Test-Path -Path $ParentDir)) { - Write-Host "Creating directory: $ParentDir" -ForegroundColor Cyan - New-Item -Path $ParentDir -ItemType Directory -Force | Out-Null - } - - # 2. Check for Specific Version Directory - if (Test-Path -Path $TargetDir) { - Write-Host "$Name ($TargetDir) already exists. Skipping download." -ForegroundColor Green - } - else { - Write-Host "$Name not found. preparing to download..." -ForegroundColor Yellow - - # Create the target directory to extract into - New-Item -Path $TargetDir -ItemType Directory -Force | Out-Null - - # Define temporary archive path - $TempFile = Join-Path $ParentDir "temp_sdk.tar.xz" - - try { - # Download - Write-Host "Downloading from: $Url" - Invoke-WebRequest -Uri $Url -OutFile $TempFile - - # Untar - # Note: We assume Windows includes tar.exe (Win 10 build 17063+) - Write-Host "Extracting archive to $TargetDir..." - - # We use -C to extract contents INTO the target directory created above - tar -xJvf $TempFile -C $TargetDir\.. - - Write-Host "Extraction complete." -ForegroundColor Green - } - catch { - Write-Error "Failed to download or extract $Name. Error: $_" - # Cleanup target dir if failed so script tries again next time - Remove-Item -Path $TargetDir -Recurse -Force -ErrorAction SilentlyContinue - } - finally { - # Cleanup Archive - if (Test-Path $TempFile) { Remove-Item $TempFile -Force } - } - } -} - -# --- Execution --- - -# 1. Ensure Base C:\Qualcomm exists -if (-not (Test-Path $BaseDir)) { - New-Item -Path $BaseDir -ItemType Directory -Force | Out-Null -} - -# 2. Run Install Logic -Install-QualcommSDK -Url $HexagonUrl -ParentDir $HexagonParent -TargetDir $HexagonSdkTarget -Name "Hexagon SDK" -Install-QualcommSDK -Url $OpenCLUrl -ParentDir $OpenCLParent -TargetDir $OpenCLTarget -Name "OpenCL SDK" - -# --- Environment Variables --- - -Write-Host "`nSetting Environment Variables..." -ForegroundColor Cyan - -# Set OPENCL_SDK_ROOT -[System.Environment]::SetEnvironmentVariable('OPENCL_SDK_ROOT', $OpenCLTarget, [System.EnvironmentVariableTarget]::User) -$env:OPENCL_SDK_ROOT = $OpenCLTarget # Set for current session as well -Write-Host "OPENCL_SDK_ROOT set to: $OpenCLTarget" - -# Set HEXAGON_SDK_ROOT -[System.Environment]::SetEnvironmentVariable('HEXAGON_SDK_ROOT', $HexagonSdkTarget, [System.EnvironmentVariableTarget]::User) -$env:HEXAGON_SDK_ROOT = $HexagonSdkTarget # Set for current session as well -Write-Host "HEXAGON_SDK_ROOT set to: $HexagonSdkTarget" - -# Set HEXAGON_SDK_ROOT -[System.Environment]::SetEnvironmentVariable('HEXAGON_TOOLS_ROOT', $HexagonToolsTarget, [System.EnvironmentVariableTarget]::User) -$env:HEXAGON_TOOLS_ROOT = $HexagonToolsTarget # Set for current session as well -Write-Host "HEXAGON_TOOLS_ROOT set to: $HexagonToolsTarget" diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 53e93a144..b1d86a4e2 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -2469,8 +2469,13 @@ struct test_set_rows : public test_case { // See dicussion here: https://github.com/ggml-org/llama.cpp/pull/23760#issuecomment-4566312209 double max_nmse_err(ggml_backend_t backend) override { ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); - if (type_dst == GGML_TYPE_Q8_0 && strcmp(ggml_backend_reg_name(reg), "WebGPU") == 0) { - return std::max(test_case::max_nmse_err(backend), 2e-7); + if (type_dst == GGML_TYPE_Q8_0) { + if (strcmp(ggml_backend_reg_name(reg), "WebGPU") == 0) { + return std::max(test_case::max_nmse_err(backend), 2e-7); + } + if (strcmp(ggml_backend_reg_name(reg), "HTP") == 0) { + return std::max(test_case::max_nmse_err(backend), 5e-6); + } } return test_case::max_nmse_err(backend); }