* fix(rocm,ci): Update ci workflow to use devlab ci runners for gfx1151
* ci : move gpu-rocm job to Strix Halo devlab runners
Assisted-by: Claude
* ci : ROCm 10 wheels with per-GPU device packages for gpu-rocm; add gfx1201
The runners' system ROCm 7.2.1 cannot compile the current HIP code for
gfx1151 ("Invalid dpp_ctrl value" in mmf-instance-ncols_*.cu). Go back
to the TheRock ROCm 10 wheels with the device-<gfx> extra, so rocBLAS
has the target's kernels, and keep ci/run.sh's results and models under
$RUNNER_TEMP instead of the runners' RAM-backed home. Add a gfx1201
(Radeon AI PRO R9700) row with HIP graphs disabled.
Assisted-by: Claude
---------
Co-authored-by: David Hekimian <dhekimian@users.noreply.github.com>
203 lines
6.4 KiB
YAML
203 lines
6.4 KiB
YAML
name: CI (self-hosted CUDA backend)
|
|
|
|
on:
|
|
workflow_dispatch: # allows manual triggering
|
|
push:
|
|
branches:
|
|
- master
|
|
paths: [
|
|
'.github/workflows/ci-self-hosted-cuda.yml',
|
|
'ci/run.sh',
|
|
'**/CMakeLists.txt',
|
|
'**/.cmake',
|
|
'**/*.h',
|
|
'**/*.hpp',
|
|
'**/*.c',
|
|
'**/*.cpp',
|
|
'**/*.cu',
|
|
'**/*.cuh'
|
|
]
|
|
|
|
pull_request:
|
|
types: [opened, synchronize, reopened]
|
|
paths: [
|
|
'.github/workflows/ci-self-hosted-cuda.yml',
|
|
'ci/run.sh',
|
|
'**/CMakeLists.txt',
|
|
'**/.cmake',
|
|
'ggml/src/*',
|
|
'ggml/src/ggml-cpu/**',
|
|
'ggml/src/ggml-cuda/**'
|
|
]
|
|
|
|
cache-mode: none
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
# note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
|
|
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
|
GGML_NLOOP: 3
|
|
GGML_N_THREADS: 1
|
|
LLAMA_ARG_LOG_COLORS: 1
|
|
LLAMA_ARG_LOG_PREFIX: 1
|
|
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
|
|
|
jobs:
|
|
gpu-cuda:
|
|
runs-on: "hf-jobs-t4-medium:cuda13"
|
|
|
|
steps:
|
|
- name: Clone
|
|
id: checkout
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
sudo apt update
|
|
sudo apt install -y cmake libssl-dev time unzip wget python3 python3-venv python3-pip
|
|
|
|
- name: ccache
|
|
uses: ggml-org/ccache-action@v1.2.24
|
|
with:
|
|
restore: false
|
|
save: false
|
|
|
|
- name: ccache-buckets-restore
|
|
uses: ./.github/actions/ccache-buckets
|
|
with:
|
|
key: self-hosted-gpu-cuda
|
|
folder: llama.cpp
|
|
hf_bucket: ggml-org/cache
|
|
|
|
- name: Test
|
|
id: ggml-ci
|
|
run: |
|
|
nvidia-smi
|
|
GG_BUILD_CUDA=1 CUDACXX=/usr/local/cuda/bin/nvcc bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
|
|
|
- name: ccache-buckets-save
|
|
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
|
uses: ./.github/actions/ccache-buckets
|
|
env:
|
|
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
|
with:
|
|
key: self-hosted-gpu-cuda
|
|
folder: llama.cpp
|
|
evict-old-files: 1d
|
|
hf_bucket: ggml-org/cache
|
|
save: true
|
|
|
|
gpu-rocm:
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
# Strix Halo (iGPU, RDNA3.5)
|
|
- gpu: gfx1151
|
|
runner: [self-hosted, Linux, strix-halo, devlab-dispatch]
|
|
# async-execution correctness workaround, gfx1151 only (see Test step)
|
|
hip_launch_blocking: "1"
|
|
# Radeon AI PRO R9700 (dGPU, RDNA4)
|
|
- gpu: gfx1201
|
|
runner: [self-hosted, Linux, radeon-r9700, devlab-dispatch]
|
|
hip_launch_blocking: "0"
|
|
# HIP-graph capture fails on gfx1201 (hipMemset -> hipErrorStreamCaptureImplicit)
|
|
disable_hip_graphs: true
|
|
|
|
name: gpu-rocm (${{ matrix.gpu }})
|
|
runs-on: ${{ matrix.runner }}
|
|
timeout-minutes: 120
|
|
|
|
env:
|
|
ROCM_VERSION: "10.0.0"
|
|
|
|
steps:
|
|
- name: Clone
|
|
id: checkout
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Verify GPU device
|
|
run: |
|
|
ls /dev/kfd /dev/dri/renderD* || { echo "ERROR: no amdgpu compute device"; exit 1; }
|
|
|
|
# same TheRock wheels as the ubuntu-24-rocm release build, so the runner does not need a ROCm install
|
|
- name: Setup TheRock with Wheels
|
|
run: |
|
|
python3 -m venv "$RUNNER_TEMP/rocm"
|
|
"$RUNNER_TEMP/rocm/bin/python" -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel,device-${{ matrix.gpu }}]==${ROCM_VERSION}"
|
|
ROCM_PATH=$("$RUNNER_TEMP/rocm/bin/rocm-sdk" path --root)
|
|
{
|
|
echo "ROCM_PATH=$ROCM_PATH"
|
|
echo "HIP_PATH=$ROCM_PATH"
|
|
echo "CMAKE_PREFIX_PATH=$("$RUNNER_TEMP/rocm/bin/rocm-sdk" path --cmake)"
|
|
echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}"
|
|
} >> "$GITHUB_ENV"
|
|
"$RUNNER_TEMP/rocm/bin/rocm-sdk" path --bin >> "$GITHUB_PATH"
|
|
|
|
- name: ccache
|
|
uses: ggml-org/ccache-action@v1.2.24
|
|
with:
|
|
restore: false
|
|
save: false
|
|
|
|
# ROCm is installed again on each run, so hash the compiler content, not its mtime
|
|
- name: Tune ccache for reinstalled ROCm toolchain
|
|
run: |
|
|
ccache --set-config=compiler_check=content
|
|
ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime
|
|
ccache --set-config=depend_mode=true
|
|
|
|
- name: ccache-buckets-restore
|
|
uses: ./.github/actions/ccache-buckets
|
|
with:
|
|
key: self-hosted-gpu-rocm-${{ matrix.gpu }}
|
|
folder: llama.cpp
|
|
hf_bucket: ggml-org/cache
|
|
|
|
- name: Test
|
|
id: ggml-ci
|
|
# HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness
|
|
# issue on integrated RDNA3.5 (gfx1151) where batched inference returns
|
|
# incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches
|
|
# restores correctness. Remove once the underlying ROCm/HIP issue is fixed.
|
|
env:
|
|
HIP_LAUNCH_BLOCKING: ${{ matrix.hip_launch_blocking }}
|
|
run: |
|
|
rocminfo
|
|
# Exported rather than set in env:, because ggml only checks that the
|
|
# variable is defined: an empty value would disable graphs on gfx1151 too.
|
|
if [ "${{ matrix.disable_hip_graphs }}" = "true" ]; then export GGML_CUDA_DISABLE_GRAPHS=1; fi
|
|
GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=${{ matrix.gpu }} bash ./ci/run.sh "$RUNNER_TEMP/results/llama.cpp" "$RUNNER_TEMP/mnt/llama.cpp"
|
|
|
|
- name: ccache-buckets-save
|
|
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
|
uses: ./.github/actions/ccache-buckets
|
|
env:
|
|
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
|
with:
|
|
key: self-hosted-gpu-rocm-${{ matrix.gpu }}
|
|
folder: llama.cpp
|
|
evict-old-files: 1d
|
|
hf_bucket: ggml-org/cache
|
|
save: true
|
|
|
|
# TODO: provision AMD GPU machine
|
|
# amd-rocm:
|
|
# runs-on: [self-hosted, Linux, AMD]
|
|
|
|
# steps:
|
|
# - name: Clone
|
|
# id: checkout
|
|
# uses: actions/checkout@v6
|
|
|
|
# - name: Test
|
|
# id: ggml-ci
|
|
# run: |
|
|
# amd-smi static
|
|
# GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|