From 790b5713ca94e30f6c604daf79716f26111d20e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Tue, 25 Aug 2026 16:28:25 +0200 Subject: [PATCH] ci : store ccache on HF buckets (test with cuda-ubuntu for now) (#27699) * add ccache-buckets action * use ccache-buckets * only save on master * install python3-venv for hip * add jq and python3 for cuda * only delete caches older than 5 minutes --- .github/actions/ccache-buckets/action.yml | 91 +++++++++++++++++++++++ .github/workflows/build-cuda-ubuntu.yml | 84 ++++++++++++++------- 2 files changed, 148 insertions(+), 27 deletions(-) create mode 100644 .github/actions/ccache-buckets/action.yml diff --git a/.github/actions/ccache-buckets/action.yml b/.github/actions/ccache-buckets/action.yml new file mode 100644 index 000000000..8eb65d275 --- /dev/null +++ b/.github/actions/ccache-buckets/action.yml @@ -0,0 +1,91 @@ +name: "ccache-buckets" +description: "Save/restore latest GitHub Actions ccache matching a key prefix to/from HF buckets" +inputs: + key: + description: "Cache key prefix to match and load" + required: true + folder: + description: "Bucket folder containing ccache files" + required: true + evict-old-files: + description: "Corresponds to the ccache --evict-older-than AGE option, where AGE is the number of seconds or days followed by the 's' or 'd' suffix respectively." + default: '' + save: + description: "Save ccache" + required: false + default: false + type: boolean + hf_bucket: + description: 'Hugging Face buckets path' + required: true + +runs: + using: "composite" + steps: + - name: Install Hugging Face Hub CLI + shell: bash + run: | + python3 -m venv .venv-hf + .venv-hf/bin/pip install -U huggingface_hub==1.28.0 + + - name: Restore ccache from buckets + if: ${{ inputs.save != 'true' }} + shell: bash + run: | + set +e -uo pipefail + source .venv-hf/bin/activate + CCACHE_DIR=$(ccache -k cache_dir) + if [[ -d "$CCACHE_DIR" ]]; then + CACHE_PATH=$(hf buckets list "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}" --json | jq -r '[.[] | select(.type == "file") | select(.path | startswith("${{ inputs.folder }}/${{ inputs.key }}") and endswith(".tar.gz"))] | sort_by(.path) | last | .path // ""') + if [[ -n "$CACHE_PATH" ]]; then + echo "Restoring ccache from '$CACHE_PATH'." + hf buckets cp "hf://buckets/${{ inputs.hf_bucket }}/$CACHE_PATH" ccache_bucket.tar.gz + mkdir -p ccache_bucket + if tar -xzf ccache_bucket.tar.gz -C ccache_bucket; then + rm -rf "$CCACHE_DIR" + mv ccache_bucket "$CCACHE_DIR" + ccache -z + fi + rm ccache_bucket.tar.gz + else + echo "No ccache found." + fi + else + echo "'$CCACHE_DIR' not found." + fi + + - name: Save ccache to buckets + if: ${{ inputs.save == 'true' }} + shell: bash + run: | + set +e -uo pipefail + source .venv-hf/bin/activate + CCACHE_DIR=$(ccache -k cache_dir) + if [[ -d "$CCACHE_DIR" ]]; then + ccache -s + if [[ -n "${{ inputs.evict-old-files }}" ]]; then + ccache --evict-older-than "${{ inputs.evict-old-files }}" + fi + DATESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + CACHEFILE="${{ inputs.key }}-$DATESTAMP.tar.gz" + if tar -czf ccache_bucket.tar.gz -C "$CCACHE_DIR" .; then + hf buckets cp ccache_bucket.tar.gz "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}/$CACHEFILE" + fi + rm ccache_bucket.tar.gz + else + echo "'$CCACHE_DIR' not found." + fi + + - name: Remove old ccache files from buckets + if: ${{ inputs.save == 'true' }} + shell: bash + run: | + set +e -uo pipefail + source .venv-hf/bin/activate + CACHE_FILES=$(hf buckets list "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}" --json | jq -r '[.[] | select(.type == "file") | select((.uploaded_at | .[:19]+"Z" | fromdateiso8601) < (now - 5 * 60)) | select(.path | startswith("${{ inputs.folder }}/${{ inputs.key }}") and endswith(".tar.gz"))] | sort_by(.path)[:-1] | .[] | [.path // ""] | @tsv') + if [[ -n "$CACHE_FILES" ]]; then + echo "Removing old ccache files..." + while IFS=$'\t' read -r CACHE_PATH; do + hf buckets rm "hf://buckets/${{ inputs.hf_bucket }}/$CACHE_PATH" -y + done <<< "$CACHE_FILES" + fi diff --git a/.github/workflows/build-cuda-ubuntu.yml b/.github/workflows/build-cuda-ubuntu.yml index ed5164756..215299670 100644 --- a/.github/workflows/build-cuda-ubuntu.yml +++ b/.github/workflows/build-cuda-ubuntu.yml @@ -50,14 +50,22 @@ jobs: DEBIAN_FRONTEND: noninteractive run: | apt update - apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev + apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev jq python3 python3-venv python3-pip - name: ccache uses: ggml-org/ccache-action@v1.2.21 with: key: cuda-ubuntu-24.04-cuda - evict-old-files: 1d - save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + save: false + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: cuda-ubuntu-24.04-cuda + folder: llama.cpp + hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }} - name: Build with CMake # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project @@ -72,15 +80,17 @@ jobs: -DGGML_CUDA_CUB_3DOT2=ON cmake --build build - - name: ccache-clear - uses: ./.github/actions/ccache-clear + - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: ./.github/actions/ccache-buckets env: - GH_TOKEN: ${{ github.token }} + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: key: cuda-ubuntu-24.04-cuda - older: 5m - min: 1 - dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }} + save: true hip: runs-on: ubuntu-22.04 @@ -95,14 +105,22 @@ jobs: id: depends run: | sudo apt-get update - sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev + sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev jq python3-venv - name: ccache uses: ggml-org/ccache-action@v1.2.21 with: key: cuda-ubuntu-22.04-hip - evict-old-files: 1d - save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + save: false + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: cuda-ubuntu-22.04-hip + folder: llama.cpp + hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }} - name: Build with native CMake HIP support id: cmake_build @@ -113,15 +131,17 @@ jobs: -DGGML_HIP=ON cmake --build build --config Release -j $(nproc) - - name: ccache-clear - uses: ./.github/actions/ccache-clear + - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: ./.github/actions/ccache-buckets env: - GH_TOKEN: ${{ github.token }} + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: key: cuda-ubuntu-22.04-hip - older: 5m - min: 1 - dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }} + save: true musa: runs-on: ubuntu-22.04 @@ -136,14 +156,22 @@ jobs: id: depends run: | apt-get update - apt-get install -y build-essential git cmake libssl-dev + apt-get install -y build-essential git cmake libssl-dev jq - name: ccache uses: ggml-org/ccache-action@v1.2.21 with: key: cuda-ubuntu-22.04-musa - evict-old-files: 1d - save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + save: false + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: cuda-ubuntu-22.04-musa + folder: llama.cpp + hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }} - name: Build with native CMake MUSA support id: cmake_build @@ -152,12 +180,14 @@ jobs: -DGGML_MUSA=ON time cmake --build build --config Release -j $(nproc) - - name: ccache-clear - uses: ./.github/actions/ccache-clear + - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: ./.github/actions/ccache-buckets env: - GH_TOKEN: ${{ github.token }} + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: key: cuda-ubuntu-22.04-musa - older: 5m - min: 1 - dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }} + save: true