name: CI (self-hosted CUDA backend) on: workflow_dispatch: # allows manual triggering push: branches: - master paths: [ '.github/workflows/ci-self-hosted-cuda.yml', 'ci/run.sh', '**/CMakeLists.txt', '**/.cmake', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.cuh' ] pull_request: types: [opened, synchronize, reopened] paths: [ '.github/workflows/ci-self-hosted-cuda.yml', 'ci/run.sh', '**/CMakeLists.txt', '**/.cmake', 'ggml/src/*', 'ggml/src/ggml-cpu/**', 'ggml/src/ggml-cuda/**' ] cache-mode: none permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} cancel-in-progress: true env: # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302) HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} GGML_NLOOP: 3 GGML_N_THREADS: 1 LLAMA_ARG_LOG_COLORS: 1 LLAMA_ARG_LOG_PREFIX: 1 LLAMA_ARG_LOG_TIMESTAMPS: 1 jobs: gpu-cuda: runs-on: "hf-jobs-t4-medium:cuda13" steps: - name: Clone id: checkout uses: actions/checkout@v6 - name: Install dependencies run: | sudo apt update sudo apt install -y cmake libssl-dev time unzip wget python3 python3-venv python3-pip - name: ccache uses: ggml-org/ccache-action@v1.2.24 with: restore: false save: false - name: ccache-buckets-restore uses: ./.github/actions/ccache-buckets with: key: self-hosted-gpu-cuda folder: llama.cpp hf_bucket: ggml-org/cache - name: Test id: ggml-ci run: | nvidia-smi GG_BUILD_CUDA=1 CUDACXX=/usr/local/cuda/bin/nvcc bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - name: ccache-buckets-save if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: ./.github/actions/ccache-buckets env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: key: self-hosted-gpu-cuda folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache save: true gpu-rocm: strategy: fail-fast: false matrix: include: # Strix Halo (iGPU, RDNA3.5) - gpu: gfx1151 runner: [self-hosted, Linux, strix-halo, devlab-dispatch] # async-execution correctness workaround, gfx1151 only (see Test step) hip_launch_blocking: "1" # Radeon AI PRO R9700 (dGPU, RDNA4) - gpu: gfx1201 runner: [self-hosted, Linux, radeon-r9700, devlab-dispatch] hip_launch_blocking: "0" # HIP-graph capture fails on gfx1201 (hipMemset -> hipErrorStreamCaptureImplicit) disable_hip_graphs: true name: gpu-rocm (${{ matrix.gpu }}) runs-on: ${{ matrix.runner }} timeout-minutes: 120 env: ROCM_VERSION: "10.0.0" steps: - name: Clone id: checkout uses: actions/checkout@v6 - name: Verify GPU device run: | ls /dev/kfd /dev/dri/renderD* || { echo "ERROR: no amdgpu compute device"; exit 1; } # same TheRock wheels as the ubuntu-24-rocm release build, so the runner does not need a ROCm install - name: Setup TheRock with Wheels run: | python3 -m venv "$RUNNER_TEMP/rocm" "$RUNNER_TEMP/rocm/bin/python" -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel,device-${{ matrix.gpu }}]==${ROCM_VERSION}" ROCM_PATH=$("$RUNNER_TEMP/rocm/bin/rocm-sdk" path --root) { echo "ROCM_PATH=$ROCM_PATH" echo "HIP_PATH=$ROCM_PATH" echo "CMAKE_PREFIX_PATH=$("$RUNNER_TEMP/rocm/bin/rocm-sdk" path --cmake)" echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}" } >> "$GITHUB_ENV" "$RUNNER_TEMP/rocm/bin/rocm-sdk" path --bin >> "$GITHUB_PATH" - name: ccache uses: ggml-org/ccache-action@v1.2.24 with: restore: false save: false # ROCm is installed again on each run, so hash the compiler content, not its mtime - name: Tune ccache for reinstalled ROCm toolchain run: | ccache --set-config=compiler_check=content ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime ccache --set-config=depend_mode=true - name: ccache-buckets-restore uses: ./.github/actions/ccache-buckets with: key: self-hosted-gpu-rocm-${{ matrix.gpu }} folder: llama.cpp hf_bucket: ggml-org/cache - name: Test id: ggml-ci # HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness # issue on integrated RDNA3.5 (gfx1151) where batched inference returns # incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches # restores correctness. Remove once the underlying ROCm/HIP issue is fixed. env: HIP_LAUNCH_BLOCKING: ${{ matrix.hip_launch_blocking }} run: | rocminfo # Exported rather than set in env:, because ggml only checks that the # variable is defined: an empty value would disable graphs on gfx1151 too. if [ "${{ matrix.disable_hip_graphs }}" = "true" ]; then export GGML_CUDA_DISABLE_GRAPHS=1; fi GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=${{ matrix.gpu }} bash ./ci/run.sh "$RUNNER_TEMP/results/llama.cpp" "$RUNNER_TEMP/mnt/llama.cpp" - name: ccache-buckets-save if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: ./.github/actions/ccache-buckets env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: key: self-hosted-gpu-rocm-${{ matrix.gpu }} folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache save: true # TODO: provision AMD GPU machine # amd-rocm: # runs-on: [self-hosted, Linux, AMD] # steps: # - name: Clone # id: checkout # uses: actions/checkout@v6 # - name: Test # id: ggml-ci # run: | # amd-smi static # GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp