* server : functionality test to make sure every time when we add new spec dec method the existing ones are not broken. * server: use compact models for speculative decoding CI * ci: avoid GHA cache for speculative models * ci: run speculative tests on DGX Spark * server: address speculative test review * server: use consolidated speculative models * server: refine speculative test calibration * docs: record speculative Spark validation * ci : run the server-speculative steps with bash * server: recalibrate speculative test floors on GB10 --------- Co-authored-by: Pascal <admin@serveurperso.com>
66 lines
1.7 KiB
YAML
66 lines
1.7 KiB
YAML
name: Server (speculative decoding)
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
sha:
|
|
description: 'Commit SHA1 to build'
|
|
required: false
|
|
type: string
|
|
|
|
cache-mode: none
|
|
permissions:
|
|
contents: read
|
|
|
|
env:
|
|
LLAMA_ARG_LOG_COLORS: 1
|
|
LLAMA_ARG_LOG_PREFIX: 1
|
|
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
server-speculative:
|
|
runs-on: [self-hosted, Linux, gb10, perf]
|
|
container: "nvidia/cuda:13.4.3-devel-ubuntu26.04"
|
|
timeout-minutes: 60
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
steps:
|
|
- name: Clone
|
|
uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
apt update
|
|
apt install -y cmake libssl-dev python3 python3-venv python3-pip
|
|
|
|
- name: Build
|
|
run: |
|
|
cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
|
|
cmake --build build --config Release -j $(nproc) --target llama-server
|
|
|
|
- name: Python setup
|
|
run: |
|
|
cd tools/server/tests
|
|
python3 -m venv venv
|
|
source venv/bin/activate
|
|
pip install -r requirements.txt
|
|
|
|
- name: Test speculative decoding
|
|
env:
|
|
SPEC_METHODS: compact
|
|
SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || format('{0}/spec-models', runner.temp) }}
|
|
N_GPU_LAYERS: 99
|
|
run: |
|
|
cd tools/server/tests
|
|
source venv/bin/activate
|
|
PYTEST_WORKERS=1 ./tests.sh -s -v unit/test_spec_methods.py
|