ci : make cache bucket public (#27728)
* check for hf token * make bucket public
This commit is contained in:
@@ -58,34 +58,38 @@ runs:
|
||||
if: ${{ inputs.save == 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set +e -uo pipefail
|
||||
source .venv-hf/bin/activate
|
||||
CCACHE_DIR=$(ccache -k cache_dir)
|
||||
if [[ -d "$CCACHE_DIR" ]]; then
|
||||
ccache -s
|
||||
if [[ -n "${{ inputs.evict-old-files }}" ]]; then
|
||||
ccache --evict-older-than "${{ inputs.evict-old-files }}"
|
||||
if [[ -n "$HF_TOKEN" ]]; then
|
||||
set +e -uo pipefail
|
||||
source .venv-hf/bin/activate
|
||||
CCACHE_DIR=$(ccache -k cache_dir)
|
||||
if [[ -d "$CCACHE_DIR" ]]; then
|
||||
ccache -s
|
||||
if [[ -n "${{ inputs.evict-old-files }}" ]]; then
|
||||
ccache --evict-older-than "${{ inputs.evict-old-files }}"
|
||||
fi
|
||||
DATESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
|
||||
CACHEFILE="${{ inputs.key }}-$DATESTAMP.tar.gz"
|
||||
if tar -czf ccache_bucket.tar.gz -C "$CCACHE_DIR" .; then
|
||||
hf buckets cp ccache_bucket.tar.gz "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}/$CACHEFILE"
|
||||
fi
|
||||
rm ccache_bucket.tar.gz
|
||||
else
|
||||
echo "'$CCACHE_DIR' not found."
|
||||
fi
|
||||
DATESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
|
||||
CACHEFILE="${{ inputs.key }}-$DATESTAMP.tar.gz"
|
||||
if tar -czf ccache_bucket.tar.gz -C "$CCACHE_DIR" .; then
|
||||
hf buckets cp ccache_bucket.tar.gz "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}/$CACHEFILE"
|
||||
fi
|
||||
rm ccache_bucket.tar.gz
|
||||
else
|
||||
echo "'$CCACHE_DIR' not found."
|
||||
fi
|
||||
|
||||
- name: Remove old ccache files from buckets
|
||||
if: ${{ inputs.save == 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set +e -uo pipefail
|
||||
source .venv-hf/bin/activate
|
||||
CACHE_FILES=$(hf buckets list "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}" --json | jq -r '[.[] | select(.type == "file") | select((.uploaded_at | .[:19]+"Z" | fromdateiso8601) < (now - 5 * 60)) | select(.path | startswith("${{ inputs.folder }}/${{ inputs.key }}") and endswith(".tar.gz"))] | sort_by(.path)[:-1] | .[] | [.path // ""] | @tsv')
|
||||
if [[ -n "$CACHE_FILES" ]]; then
|
||||
echo "Removing old ccache files..."
|
||||
while IFS=$'\t' read -r CACHE_PATH; do
|
||||
hf buckets rm "hf://buckets/${{ inputs.hf_bucket }}/$CACHE_PATH" -y
|
||||
done <<< "$CACHE_FILES"
|
||||
if [[ -n "$HF_TOKEN" ]]; then
|
||||
set +e -uo pipefail
|
||||
source .venv-hf/bin/activate
|
||||
CACHE_FILES=$(hf buckets list "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}" --json | jq -r '[.[] | select(.type == "file") | select((.uploaded_at | .[:19]+"Z" | fromdateiso8601) < (now - 5 * 60)) | select(.path | startswith("${{ inputs.folder }}/${{ inputs.key }}") and endswith(".tar.gz"))] | sort_by(.path)[:-1] | .[] | [.path // ""] | @tsv')
|
||||
if [[ -n "$CACHE_FILES" ]]; then
|
||||
echo "Removing old ccache files..."
|
||||
while IFS=$'\t' read -r CACHE_PATH; do
|
||||
hf buckets rm "hf://buckets/${{ inputs.hf_bucket }}/$CACHE_PATH" -y
|
||||
done <<< "$CACHE_FILES"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -65,7 +65,7 @@ jobs:
|
||||
with:
|
||||
key: cuda-ubuntu-24.04-cuda
|
||||
folder: llama.cpp
|
||||
hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }}
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build with CMake
|
||||
# TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
|
||||
@@ -89,7 +89,7 @@ jobs:
|
||||
key: cuda-ubuntu-24.04-cuda
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }}
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
hip:
|
||||
@@ -120,7 +120,7 @@ jobs:
|
||||
with:
|
||||
key: cuda-ubuntu-22.04-hip
|
||||
folder: llama.cpp
|
||||
hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }}
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build with native CMake HIP support
|
||||
id: cmake_build
|
||||
@@ -140,7 +140,7 @@ jobs:
|
||||
key: cuda-ubuntu-22.04-hip
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }}
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
musa:
|
||||
@@ -171,7 +171,7 @@ jobs:
|
||||
with:
|
||||
key: cuda-ubuntu-22.04-musa
|
||||
folder: llama.cpp
|
||||
hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }}
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build with native CMake MUSA support
|
||||
id: cmake_build
|
||||
@@ -189,5 +189,5 @@ jobs:
|
||||
key: cuda-ubuntu-22.04-musa
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }}
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
Reference in New Issue
Block a user