ci : store ccache on HF buckets (test with cuda-ubuntu for now) (#27699)

* add ccache-buckets action

* use ccache-buckets

* only save on master

* install python3-venv for hip

* add jq and python3 for cuda

* only delete caches older than 5 minutes

.github/actions/ccache-buckets/action.yml

@@ -0,0 +1,91 @@
+name: "ccache-buckets"
+description: "Save/restore latest GitHub Actions ccache matching a key prefix to/from HF buckets"
+inputs:
+  key:
+    description: "Cache key prefix to match and load"
+    required: true
+  folder:
+    description: "Bucket folder containing ccache files"
+    required: true
+  evict-old-files:
+    description: "Corresponds to the ccache --evict-older-than AGE option, where AGE is the number of seconds or days followed by the 's' or 'd' suffix respectively."
+    default: ''
+  save:
+    description: "Save ccache"
+    required: false
+    default: false
+    type: boolean
+  hf_bucket:
+    description: 'Hugging Face buckets path'
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Hugging Face Hub CLI
+      shell: bash
+      run: |
+        python3 -m venv .venv-hf
+        .venv-hf/bin/pip install -U huggingface_hub==1.28.0
+
+    - name: Restore ccache from buckets
+      if: ${{ inputs.save != 'true' }}
+      shell: bash
+      run: |
+        set +e -uo pipefail
+        source .venv-hf/bin/activate
+        CCACHE_DIR=$(ccache -k cache_dir)
+        if [[ -d "$CCACHE_DIR" ]]; then
+            CACHE_PATH=$(hf buckets list "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}" --json | jq -r '[.[] | select(.type == "file") | select(.path | startswith("${{ inputs.folder }}/${{ inputs.key }}") and endswith(".tar.gz"))] | sort_by(.path) | last | .path // ""')
+            if [[ -n "$CACHE_PATH" ]]; then
+                echo "Restoring ccache from '$CACHE_PATH'."
+                hf buckets cp "hf://buckets/${{ inputs.hf_bucket }}/$CACHE_PATH" ccache_bucket.tar.gz
+                mkdir -p ccache_bucket
+                if tar -xzf ccache_bucket.tar.gz -C ccache_bucket; then
+                    rm -rf "$CCACHE_DIR"
+                    mv ccache_bucket "$CCACHE_DIR"
+                    ccache -z
+                fi
+                rm ccache_bucket.tar.gz
+            else
+                echo "No ccache found."
+            fi
+        else
+            echo "'$CCACHE_DIR' not found."
+        fi
+
+    - name: Save ccache to buckets
+      if: ${{ inputs.save == 'true' }}
+      shell: bash
+      run: |
+        set +e -uo pipefail
+        source .venv-hf/bin/activate
+        CCACHE_DIR=$(ccache -k cache_dir)
+        if [[ -d "$CCACHE_DIR" ]]; then
+            ccache -s
+            if [[ -n "${{ inputs.evict-old-files }}" ]]; then
+                ccache --evict-older-than "${{ inputs.evict-old-files }}"
+            fi
+            DATESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+            CACHEFILE="${{ inputs.key }}-$DATESTAMP.tar.gz"
+            if tar -czf ccache_bucket.tar.gz -C "$CCACHE_DIR" .; then
+                hf buckets cp ccache_bucket.tar.gz "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}/$CACHEFILE"
+            fi
+            rm ccache_bucket.tar.gz
+        else
+            echo "'$CCACHE_DIR' not found."
+        fi
+
+    - name: Remove old ccache files from buckets
+      if: ${{ inputs.save == 'true' }}
+      shell: bash
+      run: |
+        set +e -uo pipefail
+        source .venv-hf/bin/activate
+        CACHE_FILES=$(hf buckets list "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}" --json | jq -r '[.[] | select(.type == "file") | select((.uploaded_at | .[:19]+"Z" | fromdateiso8601) < (now - 5 * 60)) | select(.path | startswith("${{ inputs.folder }}/${{ inputs.key }}") and endswith(".tar.gz"))] | sort_by(.path)[:-1] | .[] | [.path // ""] | @tsv')
+        if [[ -n "$CACHE_FILES" ]]; then
+            echo "Removing old ccache files..."
+            while IFS=$'\t' read -r CACHE_PATH; do
+                hf buckets rm "hf://buckets/${{ inputs.hf_bucket }}/$CACHE_PATH" -y
+            done <<< "$CACHE_FILES"
+        fi

.github/workflows/build-cuda-ubuntu.yml

@@ -50,14 +50,22 @@ jobs:
           DEBIAN_FRONTEND: noninteractive
         run: |
           apt update
-          apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev
+          apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev jq python3 python3-venv python3-pip
 
       - name: ccache
         uses: ggml-org/ccache-action@v1.2.21
         with:
           key: cuda-ubuntu-24.04-cuda
-          evict-old-files: 1d
+          save: false
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+
+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+        with:
+          key: cuda-ubuntu-24.04-cuda
+          folder: llama.cpp
+          hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }}
 
       - name: Build with CMake
         # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
@@ -72,15 +80,17 @@ jobs:
             -DGGML_CUDA_CUB_3DOT2=ON
           cmake --build build
 
-      - name: ccache-clear
+      - name: ccache-buckets-save
-        uses: ./.github/actions/ccache-clear
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
         env:
-          GH_TOKEN: ${{ github.token }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
         with:
           key: cuda-ubuntu-24.04-cuda
-          older: 5m
+          folder: llama.cpp
-          min: 1
+          evict-old-files: 1d
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
+          hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }}
+          save: true
 
   hip:
     runs-on: ubuntu-22.04
@@ -95,14 +105,22 @@ jobs:
         id: depends
         run: |
           sudo apt-get update
-          sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev
+          sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev jq python3-venv
 
       - name: ccache
         uses: ggml-org/ccache-action@v1.2.21
         with:
           key: cuda-ubuntu-22.04-hip
-          evict-old-files: 1d
+          save: false
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+
+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+        with:
+          key: cuda-ubuntu-22.04-hip
+          folder: llama.cpp
+          hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }}
 
       - name: Build with native CMake HIP support
         id: cmake_build
@@ -113,15 +131,17 @@ jobs:
             -DGGML_HIP=ON
           cmake --build build --config Release -j $(nproc)
 
-      - name: ccache-clear
+      - name: ccache-buckets-save
-        uses: ./.github/actions/ccache-clear
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
         env:
-          GH_TOKEN: ${{ github.token }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
         with:
           key: cuda-ubuntu-22.04-hip
-          older: 5m
+          folder: llama.cpp
-          min: 1
+          evict-old-files: 1d
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
+          hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }}
+          save: true
 
   musa:
     runs-on: ubuntu-22.04
@@ -136,14 +156,22 @@ jobs:
         id: depends
         run: |
           apt-get update
-          apt-get install -y build-essential git cmake libssl-dev
+          apt-get install -y build-essential git cmake libssl-dev jq
 
       - name: ccache
         uses: ggml-org/ccache-action@v1.2.21
         with:
           key: cuda-ubuntu-22.04-musa
-          evict-old-files: 1d
+          save: false
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+
+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+        with:
+          key: cuda-ubuntu-22.04-musa
+          folder: llama.cpp
+          hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }}
 
       - name: Build with native CMake MUSA support
         id: cmake_build
@@ -152,12 +180,14 @@ jobs:
             -DGGML_MUSA=ON
           time cmake --build build --config Release -j $(nproc)
 
-      - name: ccache-clear
+      - name: ccache-buckets-save
-        uses: ./.github/actions/ccache-clear
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
         env:
-          GH_TOKEN: ${{ github.token }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
         with:
           key: cuda-ubuntu-22.04-musa
-          older: 5m
+          folder: llama.cpp
-          min: 1
+          evict-old-files: 1d
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
+          hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }}
+          save: true