From fc0be86182c86ac19ea09b2839b2fa1187ef595c Mon Sep 17 00:00:00 2001 From: Iswarya Alex <47045679+iswaryaalex@users.noreply.github.com> Date: Mon, 9 Nov 2026 04:01:52 -0800 Subject: [PATCH] ci : Run gfx1151 ROCm job on Strix Halo Devlab runners (#31567) * fix(rocm,ci): Update ci workflow to use devlab ci runners for gfx1151 * ci : move gpu-rocm job to Strix Halo devlab runners Assisted-by: Claude * ci : ROCm 10 wheels with per-GPU device packages for gpu-rocm; add gfx1201 The runners' system ROCm 7.2.1 cannot compile the current HIP code for gfx1151 ("Invalid dpp_ctrl value" in mmf-instance-ncols_*.cu). Go back to the TheRock ROCm 10 wheels with the device- extra, so rocBLAS has the target's kernels, and keep ci/run.sh's results and models under $RUNNER_TEMP instead of the runners' RAM-backed home. Add a gfx1201 (Radeon AI PRO R9700) row with HIP graphs disabled. Assisted-by: Claude --------- Co-authored-by: David Hekimian --- .github/workflows/ci-self-hosted-cuda.yml | 80 ++++++++++++++++++++++- 1 file changed, 77 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-self-hosted-cuda.yml b/.github/workflows/ci-self-hosted-cuda.yml index 89605f4dd..d433b9526 100644 --- a/.github/workflows/ci-self-hosted-cuda.yml +++ b/.github/workflows/ci-self-hosted-cuda.yml @@ -93,13 +93,72 @@ jobs: save: true gpu-rocm: - runs-on: [self-hosted, Linux, AMD] + strategy: + fail-fast: false + matrix: + include: + # Strix Halo (iGPU, RDNA3.5) + - gpu: gfx1151 + runner: [self-hosted, Linux, strix-halo, devlab-dispatch] + # async-execution correctness workaround, gfx1151 only (see Test step) + hip_launch_blocking: "1" + # Radeon AI PRO R9700 (dGPU, RDNA4) + - gpu: gfx1201 + runner: [self-hosted, Linux, radeon-r9700, devlab-dispatch] + hip_launch_blocking: "0" + # HIP-graph capture fails on gfx1201 (hipMemset -> hipErrorStreamCaptureImplicit) + disable_hip_graphs: true + + name: gpu-rocm (${{ matrix.gpu }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + + env: + ROCM_VERSION: "10.0.0" steps: - name: Clone id: checkout uses: actions/checkout@v6 + - name: Verify GPU device + run: | + ls /dev/kfd /dev/dri/renderD* || { echo "ERROR: no amdgpu compute device"; exit 1; } + + # same TheRock wheels as the ubuntu-24-rocm release build, so the runner does not need a ROCm install + - name: Setup TheRock with Wheels + run: | + python3 -m venv "$RUNNER_TEMP/rocm" + "$RUNNER_TEMP/rocm/bin/python" -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel,device-${{ matrix.gpu }}]==${ROCM_VERSION}" + ROCM_PATH=$("$RUNNER_TEMP/rocm/bin/rocm-sdk" path --root) + { + echo "ROCM_PATH=$ROCM_PATH" + echo "HIP_PATH=$ROCM_PATH" + echo "CMAKE_PREFIX_PATH=$("$RUNNER_TEMP/rocm/bin/rocm-sdk" path --cmake)" + echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}" + } >> "$GITHUB_ENV" + "$RUNNER_TEMP/rocm/bin/rocm-sdk" path --bin >> "$GITHUB_PATH" + + - name: ccache + uses: ggml-org/ccache-action@v1.2.24 + with: + restore: false + save: false + + # ROCm is installed again on each run, so hash the compiler content, not its mtime + - name: Tune ccache for reinstalled ROCm toolchain + run: | + ccache --set-config=compiler_check=content + ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime + ccache --set-config=depend_mode=true + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + with: + key: self-hosted-gpu-rocm-${{ matrix.gpu }} + folder: llama.cpp + hf_bucket: ggml-org/cache + - name: Test id: ggml-ci # HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness @@ -107,10 +166,25 @@ jobs: # incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches # restores correctness. Remove once the underlying ROCm/HIP issue is fixed. env: - HIP_LAUNCH_BLOCKING: "1" + HIP_LAUNCH_BLOCKING: ${{ matrix.hip_launch_blocking }} run: | rocminfo - GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + # Exported rather than set in env:, because ggml only checks that the + # variable is defined: an empty value would disable graphs on gfx1151 too. + if [ "${{ matrix.disable_hip_graphs }}" = "true" ]; then export GGML_CUDA_DISABLE_GRAPHS=1; fi + GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=${{ matrix.gpu }} bash ./ci/run.sh "$RUNNER_TEMP/results/llama.cpp" "$RUNNER_TEMP/mnt/llama.cpp" + + - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: self-hosted-gpu-rocm-${{ matrix.gpu }} + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ggml-org/cache + save: true # TODO: provision AMD GPU machine # amd-rocm: