Files
llama.cpp/scripts/fork-validate.sh
T
LumpiastyandClaude Opus 5.5 80e662a2a1 scripts: fork-validate.sh - A/B ordinary handoff latency against --ref
--arbiter only reported the candidate's handoff latency, while the fork's
process rule (two production deploys lost) is to A/B the ordinary handoff,
not only the pathological cases. After the arbiter cases, a-handoff-latency
starts fresh A/B server pairs of each build on the scratch arena and runs
blocks ref, cand, cand, ref of 16 alternating handoffs each. It FAILs if the
candidate median exceeds the reference median by more than max(25%, 200 ms)
and reports both medians, maxima and per-block medians.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 00:39:13 +02:00

783 lines
45 KiB
Bash

#!/usr/bin/env bash
# fork-validate.sh - validate a candidate build of the RX 580 llama.cpp fork against a reference build.
#
# Runs ON THE POD (Vulkan / RADV, RX 580). Prints PASS/FAIL/SKIP per check plus a summary; exits
# nonzero if any check FAILs. Only touches its own scratch dir under /tmp, its own scratch ports and
# processes it started itself (killed by explicit PID). Build dirs are only read.
#
# fork-validate.sh --build /root/arms/X/build --ref /root/llama.cpp/build [checks] [options]
#
# Checks (no check flag = --ops --concat --arbiter):
# --ops test-backend-ops on Vulkan0 for CONCAT, FLASH_ATTN_EXT, MUL_MAT, MUL_MAT_ID, GET_ROWS
# -> the fork's kernels and upstream's agree with the CPU reference; asserts the case count
# is > 0 because "-b Vulkan" (no index) silently runs nothing and still prints OK.
# --concat Vulkan concat tiled-transpose fast path (ggml_vk_concat, gate src_transposed_2d)
# -> it fires at prefill and NOT at decode on a real hybrid (delta-net) model; it is the
# fork's biggest prefill win, and taking it at decode once cost -17% decode.
# Needs a build carrying the GGML_VK_CONCAT_FASTPATH knob (=0 off, =log report);
# either --build has it or pass --concat-build DIR (candidate + the knob patch).
# --arbiter on-demand VRAM sharing (LLAMA_SLEEP_VRAM_ONLY ...) with two real servers on a scratch
# arena -> handoffs never strand, release/restore is bit-exact, multimodal survives it,
# and every shutdown path (mid-release, mid-wait, mid-generation, cold, warm, kill -9 of
# a waiter) exits cleanly and hands the GPU token on. These are the past production
# incidents (stranded requests, wedged token, exit 134 on cold teardown).
# Then a-handoff-latency: ordinary handoff latency of --build vs --ref in the same
# session (fresh server pairs, blocks ref,cand,cand,ref x 16 handoffs) -> FAIL if the
# candidate median exceeds the reference median by more than max(25%, 200 ms). Two
# production deploys were lost to arbiter changes that only showed up here.
# --perf llama-bench, candidate vs reference, alternating ref,cand,cand,ref, on the 35B MoE
# -> no prefill regression > 5% at depth 0/16k/32k (FAIL), decode tg256@d4096
# reported; peak VRAM per arm (FAIL if > 95% of the card). Compares MEDIANS of every
# repetition sample pooled over a build's two arms (4 per prefill cell, 6 for decode),
# never means: CPU contention on the node produces single catastrophic samples.
# Needs llama-swap STOPPED (run it under /root/bench/moe/swapdown.sh); refuses to
# run if any other llama process is up. Samples are kept in <scratch>/perf.tsv.
# --moecache MODEL
# one arbiter server with --moe-cache-mib N on a MoE model, forced cold/warm twice
# -> greedy output identical before/after (the expert-cache SLRU is re-initialised on
# release; a stale SLRU maps experts to empty banks and produces garbage).
#
# Options:
# --concat-build DIR build with the GGML_VK_CONCAT_FASTPATH knob for --concat (default: --build)
# --small MODEL small hybrid model for --concat/--arbiter (default: Qwen3.5-0.8B UD-Q4_K_XL)
# --mmproj FILE its multimodal projector (default: mmproj-F16 next to it)
# --perf-model MODEL model for --perf (default: Qwen3.6-35B-A3B UD-Q4_K_XL)
# --moecache-args S extra server args for --moecache (default: "-ngl 99 --n-cpu-moe 30 -c 4096")
# --moecache-mib N cache size for --moecache (default 2048; 256 never hits on a 26B and proves nothing)
# --port-base N scratch ports N..N+2 (default 8091)
# --keep keep the scratch dir even when everything passed (it is kept on any FAIL)
# --perf-report TSV only re-aggregate a saved <scratch>/perf.tsv of an earlier --perf run and
# apply the same gates (no GPU, no --build/--ref needed); runs nothing else
#
# Requirements: bash, coreutils, awk, sed, grep, curl, flock(1); gdb optional (makes the
# SIGTERM-during-release case deterministic). No python/jq.
set -u
# ---------------------------------------------------------------- defaults
HF=/root/.cache/huggingface/hub
SMALL_DIR=$HF/models--unsloth--Qwen3.5-0.8B-GGUF/snapshots/6ab461498e2023f6e3c1baea90a8f0fe38ab64d0
SMALL=$SMALL_DIR/Qwen3.5-0.8B-UD-Q4_K_XL.gguf
MMPROJ=
PERF_MODEL=$HF/models--unsloth--Qwen3.6-35B-A3B-GGUF/snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
MOECACHE_MODEL=
MOECACHE_ARGS="-ngl 99 --n-cpu-moe 30 -c 4096"
MOECACHE_MIB=2048
BUILD= ; REF= ; CONCAT_BUILD=
PORT_BASE=8091
KEEP=0
DO_OPS=0 DO_CONCAT=0 DO_ARB=0 DO_PERF=0 DO_MOEC=0
PERF_REPORT=
usage() { awk 'NR > 1 && /^#/ {sub(/^# ?/, ""); print; next} NR > 1 {exit}' "$0"; exit 2; }
while [ $# -gt 0 ]; do
case "$1" in
--build) BUILD=$2; shift 2 ;;
--ref) REF=$2; shift 2 ;;
--ops) DO_OPS=1; shift ;;
--concat) DO_CONCAT=1; shift ;;
--arbiter) DO_ARB=1; shift ;;
--perf) DO_PERF=1; shift ;;
--moecache) DO_MOEC=1; MOECACHE_MODEL=$2; shift 2 ;;
--moecache-args) MOECACHE_ARGS=$2; shift 2 ;;
--moecache-mib) MOECACHE_MIB=$2; shift 2 ;;
--concat-build) CONCAT_BUILD=$2; shift 2 ;;
--small) SMALL=$2; shift 2 ;;
--mmproj) MMPROJ=$2; shift 2 ;;
--perf-model) PERF_MODEL=$2; shift 2 ;;
--port-base) PORT_BASE=$2; shift 2 ;;
--keep) KEEP=1; shift ;;
--perf-report) PERF_REPORT=$(readlink -f "$2"); shift 2 ;;
-h|--help) usage ;;
*) echo "unknown argument: $1" >&2; usage ;;
esac
done
[ -n "$PERF_REPORT" ] || { [ -n "$BUILD" ] && [ -n "$REF" ]; } || usage
if [ $((DO_OPS + DO_CONCAT + DO_ARB + DO_PERF + DO_MOEC)) -eq 0 ]; then
DO_OPS=1 DO_CONCAT=1 DO_ARB=1
fi
[ -n "$MMPROJ" ] || MMPROJ=$(dirname "$SMALL")/mmproj-F16.gguf
PA=$PORT_BASE PB=$((PORT_BASE + 1)) PC=$((PORT_BASE + 2))
# ---------------------------------------------------------------- reporting
RESULTS=()
NFAIL=0
ts() { date +%H:%M:%S; }
log() { echo "[$(ts)] $*"; }
pass() { RESULTS+=("PASS $1: $2"); echo "[$(ts)] PASS $1: $2"; }
fail() { RESULTS+=("FAIL $1: $2"); echo "[$(ts)] FAIL $1: $2"; NFAIL=$((NFAIL + 1)); }
skip() { RESULTS+=("SKIP $1: $2"); echo "[$(ts)] SKIP $1: $2"; }
# check NAME CONDITION-RESULT(0/1) MESSAGE
check() { if [ "$2" -eq 0 ]; then pass "$1" "$3"; else fail "$1" "$3"; fi; }
now_ms() { date +%s%3N; }
# ---------------------------------------------------------------- scratch + cleanup
S=$(mktemp -d /tmp/fork-validate.XXXXXX)
ARENA=$S/arena
mkdir -p "$ARENA"
cd "$S" || exit 2 # anything written to a relative path lands in the scratch dir
PIDS=() # "pid:starttime" of every background process we start (starttime guards PID reuse)
pid_alive() { # alive and not a zombie
[ -r "/proc/$1/stat" ] || return 1
local st; st=$(sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | awk '{print $1}') || return 1
[ -n "$st" ] && [ "$st" != Z ]
}
starttime() { sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | awk '{print $20}'; }
track() { PIDS+=("$1:$(starttime "$1")"); }
ours() { # "pid:starttime" -> pid if that exact process is still alive
local p=${1%%:*} t=${1#*:}
pid_alive "$p" && [ "$(starttime "$p")" = "$t" ] && echo "$p"
}
cleanup() {
local e p
for e in "${PIDS[@]}"; do p=$(ours "$e") && kill -TERM "$p" 2>/dev/null; done
local i=0
while [ $i -lt 50 ]; do
local any=0
for e in "${PIDS[@]}"; do ours "$e" > /dev/null && any=1; done
[ $any -eq 0 ] && break
sleep 0.1; i=$((i + 1))
done
for e in "${PIDS[@]}"; do p=$(ours "$e") && kill -KILL "$p" 2>/dev/null; done
wait 2>/dev/null
}
finish() {
local rc=$?
trap - EXIT INT TERM HUP
cleanup
echo
echo "==================== fork-validate summary ===================="
echo "candidate: $BUILD ($(git -C "$BUILD/.." rev-parse --short HEAD 2>/dev/null || echo '?'))"
echo "reference: $REF ($(git -C "$REF/.." rev-parse --short HEAD 2>/dev/null || echo '?'))"
local r; for r in "${RESULTS[@]}"; do echo "$r"; done
local np ns
np=$(printf '%s\n' "${RESULTS[@]}" | grep -c '^PASS')
ns=$(printf '%s\n' "${RESULTS[@]}" | grep -c '^SKIP')
echo "total: $np PASS, $NFAIL FAIL, $ns SKIP"
if [ $NFAIL -gt 0 ] || [ $KEEP -eq 1 ] || [ $rc -ne 0 ]; then
echo "scratch kept: $S"
else
rm -rf "$S"
fi
[ $NFAIL -gt 0 ] && exit 1
exit $rc
}
trap finish EXIT
trap 'echo "interrupted"; NFAIL=$((NFAIL + 1)); exit 130' INT TERM HUP
# ---------------------------------------------------------------- environment sanity
for v in $(env | sed -n 's/^\(GGML_[A-Za-z0-9_]*\|LLAMA_\(SLEEP\|VRAM\|MMPROJ\|ARG\|CACHE\|LOG\)[A-Za-z0-9_]*\)=.*/\1/p'); do
log "unsetting inherited $v"; unset "$v"
done
[ -n "$PERF_REPORT" ] || for d in "$BUILD" "$REF"; do
[ -x "$d/bin/llama-server" ] || { echo "no llama-server in $d/bin" >&2; exit 2; }
done
case "$ARENA" in /dev/shm/llama-vram*) echo "refusing the production arena" >&2; exit 2 ;; esac
VRAM_USED=$(ls /sys/class/drm/card*/device/mem_info_vram_used 2>/dev/null | head -1)
VRAM_TOTAL=$(ls /sys/class/drm/card*/device/mem_info_vram_total 2>/dev/null | head -1)
vram_mb() { echo $(( $(cat "$VRAM_USED") / 1048576 )); }
port_busy() { # LISTEN socket on the port (v4 or v6)
local hex; hex=$(printf '%04X' "$1")
awk -v h=":$hex" '$4=="0A" && substr($2, length($2)-4)==h {f=1} END {exit !f}' /proc/net/tcp /proc/net/tcp6 2>/dev/null
}
[ -n "$PERF_REPORT" ] || for p in $PA $PB $PC; do
port_busy "$p" && { echo "scratch port $p is in use" >&2; exit 2; }
done
log "scratch dir $S, ports $PA/$PB/$PC, VRAM used $(vram_mb) MiB"
# ---------------------------------------------------------------- helpers: servers
# start_server NAME BUILD PORT ENV-ASSIGNMENTS... -- SERVER-ARGS... (sets SPID, log at $S/NAME.log)
start_server() {
local name=$1 build=$2 port=$3; shift 3
local envs=()
while [ $# -gt 0 ] && [ "$1" != -- ]; do envs+=("$1"); shift; done
shift
env LD_LIBRARY_PATH="$build/bin" "${envs[@]}" "$build/bin/llama-server" \
--host 127.0.0.1 --port "$port" "$@" >> "$S/$name.log" 2>&1 &
SPID=$!
track "$SPID"
echo "=== start pid $SPID $(date +%T.%N)" >> "$S/$name.log.events"
}
# wait_health PORT PID TIMEOUT_S
wait_health() {
local t0; t0=$(now_ms)
while [ $(( $(now_ms) - t0 )) -lt $(( $3 * 1000 )) ]; do
pid_alive "$2" || return 1
[ "$(curl -s -m 2 -o /dev/null -w '%{http_code}' "http://127.0.0.1:$1/health")" = 200 ] && return 0
sleep 0.2
done
return 1
}
# wait_exit PID TIMEOUT_MS -> prints elapsed ms, returns 1 on timeout. Then reap with `wait PID`.
wait_exit() {
local t0; t0=$(now_ms)
while pid_alive "$1"; do
[ $(( $(now_ms) - t0 )) -ge "$2" ] && { echo $(( $(now_ms) - t0 )); return 1; }
sleep 0.02
done
echo $(( $(now_ms) - t0 ))
}
# stop_server PID -> SIGTERM, wait up to 20 s, sets RC and EXIT_MS
stop_server() {
kill -TERM "$1" 2>/dev/null
EXIT_MS=$(wait_exit "$1" 20000) || { kill -KILL "$1" 2>/dev/null; }
wait "$1" 2>/dev/null; RC=$?
}
# ---------------------------------------------------------------- helpers: locks (/proc/locks)
lock_id() { # maj:min:inode as /proc/locks prints it (hex dev, decimal inode)
local d; d=$(( 16#$(stat -c %D "$1") ))
printf '%02x:%02x:%s' $(( (d >> 8) & 0xfff )) $(( (d & 0xff) | ((d >> 12) & 0xfff00) )) "$(stat -c %i "$1")"
}
lock_pids() { # FILE WRITE|READ -> PIDs holding a flock on it (granted, not blocked)
[ -e "$1" ] || return 0
awk -v id="$(lock_id "$1")" -v m="$2" '$2=="FLOCK" && $4==m && $6==id {print $5}' /proc/locks
}
token_holder() { lock_pids "$ARENA/token.lock" WRITE | head -1; }
holder_num() { local h; h=$(token_holder); echo "${h:-0}"; } # for arithmetic: 0 = nobody
# wait_holder PID TIMEOUT_MS : until PID holds token.lock ("" = until nobody does)
wait_holder() {
local t0; t0=$(now_ms)
while [ "$(token_holder)" != "$1" ]; do
[ $(( $(now_ms) - t0 )) -ge "$2" ] && return 1
sleep 0.01
done
echo $(( $(now_ms) - t0 ))
}
wait_want() { # PID TIMEOUT_MS : until PID holds want.lock shared (registered waiter)
local t0; t0=$(now_ms)
until lock_pids "$ARENA/want.lock" READ | grep -qx "$1"; do
[ $(( $(now_ms) - t0 )) -ge "$2" ] && return 1
sleep 0.01
done
}
# ---------------------------------------------------------------- helpers: requests
json_str() { sed -n 's/.*"'"$1"'":"\(\([^"\\]\|\\.\)*\)".*/\1/p' "$2" | head -1; }
json_num() { grep -o "\"$1\":[0-9]*" "$2" | head -1 | cut -d: -f2; }
# complete PORT OUTFILE JSON-BODY [TIMEOUT_S] -> prints HTTP code ("000" on timeout/refused)
complete() {
curl -sS -m "${4:-60}" -o "$2" -w '%{http_code}' -H 'Content-Type: application/json' \
-d "$3" "http://127.0.0.1:$1/completion" 2>>"$S/curl.err" || true
}
PROMPT='The history of the printing press began in'
GREEDY="\"temperature\":0,\"top_k\":1,\"seed\":42"
greedy_body() { # N_PREDICT CACHE_PROMPT [PROMPT]
echo "{\"prompt\":\"${3:-$PROMPT}\",\"n_predict\":$1,$GREEDY,\"cache_prompt\":$2}"
}
LONG_BODY="{\"prompt\":\"Write a very long story about a lighthouse keeper.\",\"n_predict\":4000,\"ignore_eos\":true,$GREEDY,\"stream\":true}"
# stream_bg PORT OUTFILE BODY -> starts a streaming request, sets CPID
stream_bg() {
curl -sS -N -m 300 -H 'Content-Type: application/json' -d "$3" \
"http://127.0.0.1:$1/completion" > "$2" 2>&1 &
CPID=$!; track "$CPID"
}
wait_tokens() { # FILE N TIMEOUT_MS : until N SSE data lines arrived
local t0; t0=$(now_ms)
while [ "$(grep -c '^data: ' "$1" 2>/dev/null)" -lt "$2" ]; do
[ $(( $(now_ms) - t0 )) -ge "$3" ] && return 1
sleep 0.05
done
}
scan_logs() { # NAME-PATTERN... -> prints offending lines
grep -h -n -E 'GGML_ASSERT|Assertion|terminate called|Segmentation fault|core dumped|ErrorDeviceLost|ErrorOutOfDeviceMemory|DEVICE_LOST|vk::.*Error|GGML_ABORT|fatal error' "$@" 2>/dev/null | head -20
}
# ================================================================ 1. ops
run_ops() {
local op out passed total
for op in CONCAT FLASH_ATTN_EXT MUL_MAT MUL_MAT_ID GET_ROWS; do
out=$S/ops-$op.log
log "ops: test-backend-ops -b Vulkan0 -o $op"
( cd "$S" && LD_LIBRARY_PATH="$BUILD/bin" timeout 1800 "$BUILD/bin/test-backend-ops" -b Vulkan0 -o "$op" ) \
> "$out" 2>&1
local rc=$?
# strip ANSI colours; "N/M tests passed" is printed once per tested backend
read -r passed total < <(sed 's/\x1b\[[0-9;]*m//g' "$out" | sed -n 's/^ *\([0-9]*\)\/\([0-9]*\) tests passed.*/\1 \2/p' | head -1)
passed=${passed:-0}; total=${total:-0}
if [ $rc -eq 0 ] && [ "$total" -gt 0 ] && [ "$passed" -eq "$total" ] \
&& sed 's/\x1b\[[0-9;]*m//g' "$out" | grep -q 'Backend Vulkan0: OK'; then
pass "ops/$op" "$passed/$total cases on Vulkan0"
else
fail "ops/$op" "rc=$rc, $passed/$total passed (0 cases = wrong backend name); see $out"
fi
done
}
# ================================================================ 2. concat fast path
# Method: llama-bench on the small hybrid model (qwen35: delta-net layers concat the conv state with
# the transposed qkv in build_conv_state). (1) with the knob =log, count fast-path dispatches in a
# prefill-only run (must be > 0, all with >= 32 tokens) and a decode-only run (must be 0, while the
# perf logger proves CONCAT nodes did execute); (2) GGML_VK_PERF_LOGGER CONCAT time at prefill: the
# unmodified candidate must match knob-on and be >= 2x faster than knob-off, which ties the
# candidate binary itself (no knob) to the fast path.
concat_us() { # LOG -> "count avg_us" over all perf-logger CONCAT lines
awk '/^CONCAT: / {n += $2; t += $(NF-1)} END {if (n) printf "%d %.2f\n", n, t/n; else print "0 0"}' "$1"
}
run_concat() {
local tb=$CONCAT_BUILD
[ -n "$tb" ] || tb=$BUILD
if ! grep -qa GGML_VK_CONCAT_FASTPATH "$tb/bin/libggml-vulkan.so" 2>/dev/null; then
fail concat "no GGML_VK_CONCAT_FASTPATH knob in $tb (build the candidate + knob patch, pass --concat-build)"
return
fi
[ -f "$SMALL" ] || { fail concat "small model missing: $SMALL"; return; }
local common=(-m "$SMALL" -ngl 99 -fa 1 -lm none -t 6)
local pp=(-p 512 -n 0 -b 512 -ub 512 -r 3)
local tg=(-p 0 -n 32 -r 1)
lb() { # BUILD OUT ENV... -- ARGS
local b=$1 out=$2; shift 2
local envs=(); while [ "$1" != -- ]; do envs+=("$1"); shift; done; shift
env LD_LIBRARY_PATH="$b/bin" GGML_VK_PERF_LOGGER=1 "${envs[@]}" "$b/bin/llama-bench" "${common[@]}" "$@" \
> "$out" 2>&1
}
log "concat: llama-bench prefill/decode arms (perf logger on)"
lb "$tb" "$S/cc-pp-log.log" GGML_VK_CONCAT_FASTPATH=log -- "${pp[@]}"
lb "$tb" "$S/cc-tg-log.log" GGML_VK_CONCAT_FASTPATH=log -- "${tg[@]}"
lb "$tb" "$S/cc-pp-off.log" GGML_VK_CONCAT_FASTPATH=0 -- "${pp[@]}"
lb "$BUILD" "$S/cc-pp-cand.log" -- "${pp[@]}"
lb "$tb" "$S/cc-pp-off2.log" GGML_VK_CONCAT_FASTPATH=0 -- "${pp[@]}"
lb "$BUILD" "$S/cc-pp-cand2.log" -- "${pp[@]}"
cat "$S/cc-pp-off2.log" >> "$S/cc-pp-off.log"; cat "$S/cc-pp-cand2.log" >> "$S/cc-pp-cand.log"
local n_pp n_pp_small n_tg n_tg_concat
n_pp=$(grep -c 'transpose fast path' "$S/cc-pp-log.log")
n_pp_small=$(sed -n 's/.*transpose fast path, transposed ne0=\([0-9]*\).*/\1/p' "$S/cc-pp-log.log" | awk '$1 < 32' | wc -l)
n_tg=$(grep -c 'transpose fast path' "$S/cc-tg-log.log")
read -r n_tg_concat _ < <(concat_us "$S/cc-tg-log.log")
local ne0s; ne0s=$(sed -n 's/.*transposed ne0=\([0-9]*\).*/\1/p' "$S/cc-pp-log.log" | sort -n | uniq -c | awk '{printf "%sx%s ", $1, $2}')
check concat/prefill-fires $(( n_pp > 0 && n_pp_small == 0 ? 0 : 1 )) \
"$n_pp fast-path dispatches in the prefill run (transposed ne0: ${ne0s:-none})"
check concat/decode-skips $(( n_tg == 0 && n_tg_concat > 0 ? 0 : 1 )) \
"$n_tg fast-path dispatches over $n_tg_concat executed CONCAT nodes in the decode run"
local c_on c_off c_cand on off cand
read -r c_on on < <(concat_us "$S/cc-pp-log.log")
read -r c_off off < <(concat_us "$S/cc-pp-off.log")
read -r c_cand cand < <(concat_us "$S/cc-pp-cand.log")
local ok
ok=$(awk -v on="$on" -v off="$off" -v c="$cand" 'BEGIN {print (c > 0 && on > 0 && off/c >= 2 && c/on < 1.5 && on/c < 1.5) ? 0 : 1}')
check concat/candidate-timing "$ok" \
"prefill CONCAT avg: candidate ${cand} us, knob-on ${on} us, knob-off ${off} us (off/cand $(awk -v a="$off" -v b="$cand" 'BEGIN {printf "%.1f", (b > 0) ? a / b : 0}')x)"
local t_on t_off t_cand
t_on=$(grep -E '\| +pp512 ' "$S/cc-pp-log.log" | awk -F'|' '{print $(NF-1)}' | awk '{print $1}' | head -1)
t_off=$(grep -E '\| +pp512 ' "$S/cc-pp-off.log" | awk -F'|' '{print $(NF-1)}' | awk '{print $1}' | tr '\n' ' ')
t_cand=$(grep -E '\| +pp512 ' "$S/cc-pp-cand.log" | awk -F'|' '{print $(NF-1)}' | awk '{print $1}' | tr '\n' ' ')
log "concat: pp512 t/s (perf logger on, so absolute speed is not representative): on $t_on | off $t_off | candidate $t_cand"
}
# ================================================================ 3. arbiter
ARB_ENV=(LLAMA_SLEEP_VRAM_ONLY=1 "LLAMA_VRAM_ARENA=$ARENA" LLAMA_SLEEP_EVICT_KV=1 LLAMA_MMPROJ_ONDEMAND=1)
ARB_ARGS=(-m "$SMALL" -c 8192 -ngl 99 -fa 1 -lm none --parallel 1 --no-warmup --reasoning off)
A_PID= ; B_PID=
start_a() { start_server A "$BUILD" "$PA" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --mmproj "$MMPROJ"; A_PID=$SPID; }
start_b() { start_server B "$BUILD" "$PB" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --no-mmproj; B_PID=$SPID; }
sleeps() { grep -c 'server entering sleeping state' "$1"; }
make_red_ppm() { # 128x128 solid red, binary PPM (stb_image reads it; no image tools needed)
local f=$S/red.ppm i
printf '\377\000\000' > "$S/px"
for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14; do cat "$S/px" "$S/px" > "$S/px2"; mv "$S/px2" "$S/px"; done
{ printf 'P6\n128 128\n255\n'; cat "$S/px"; } > "$f"
base64 -w0 "$f"
}
run_arbiter() {
[ -f "$SMALL" ] && [ -f "$MMPROJ" ] || { fail arbiter "model/mmproj missing: $SMALL $MMPROJ"; return; }
local free_mb=$(( ( $(cat "$VRAM_TOTAL") - $(cat "$VRAM_USED") ) / 1048576 ))
if [ $free_mb -lt 1536 ]; then
fail arbiter "only $free_mb MiB VRAM free (< 1536): run inside the swapdown window"; return
fi
log "arbiter: $free_mb MiB VRAM free; starting A (mmproj) on :$PA"
start_a
wait_health "$PA" "$A_PID" 180 || { fail arbiter/start "A did not become healthy, see $S/A.log"; return; }
# no-handoff reference: A has held the token since its load and never released
local code ref_a
code=$(complete "$PA" "$S/ref-a.json" "$(greedy_body 48 false)")
ref_a=$(json_str content "$S/ref-a.json")
[ "$code" = 200 ] && [ -n "$ref_a" ] || { fail arbiter/start "reference request on A: HTTP $code"; return; }
[ "$(token_holder)" = "$A_PID" ] || { fail arbiter/start "A serves but does not hold token.lock"; return; }
log "arbiter: starting B (text-only) on :$PB"
start_b
wait_health "$PB" "$B_PID" 180 || { fail arbiter/start "B did not become healthy, see $S/B.log"; return; }
pass arbiter/start "A=$A_PID B=$B_PID up, reference output ${#ref_a} chars"
# ---- (a)+(b) alternating handoffs
local i port pid name bad=0 stranded=0 diff_a=0 diff_b=0 ref_b= lat=() holder_bad=0
for i in $(seq 0 30); do
if [ $((i % 2)) -eq 0 ]; then port=$PB pid=$B_PID name=B; else port=$PA pid=$A_PID name=A; fi
local t0 t1 out=$S/h$i.json
t0=$(now_ms)
code=$(complete "$port" "$out" "$(greedy_body 48 false)" 60)
t1=$(now_ms)
[ $i -gt 0 ] && lat+=($((t1 - t0)))
if [ "$code" = 000 ]; then stranded=$((stranded + 1)); continue; fi
[ "$code" != 200 ] && { bad=$((bad + 1)); continue; }
local c; c=$(json_str content "$out")
if [ $name = A ]; then [ "$c" = "$ref_a" ] || diff_a=$((diff_a + 1))
else if [ -z "$ref_b" ]; then ref_b=$c; elif [ "$c" != "$ref_b" ]; then diff_b=$((diff_b + 1)); fi; fi
[ "$(token_holder)" = "$pid" ] || holder_bad=$((holder_bad + 1))
done
local med max
med=$(printf '%s\n' "${lat[@]}" | sort -n | awk '{a[NR]=$1} END {print a[int((NR+1)/2)]}')
max=$(printf '%s\n' "${lat[@]}" | sort -n | tail -1)
check arbiter/a-handoffs $(( stranded == 0 && bad == 0 && holder_bad == 0 ? 0 : 1 )) \
"30 handoffs: $stranded stranded, $bad non-200, $holder_bad served-without-token; request latency median ${med} ms, max ${max} ms"
check arbiter/b-greedy-identical $(( diff_a == 0 && diff_b == 0 ? 0 : 1 )) \
"A: 15/15 == no-handoff reference ($diff_a differ); B: $diff_b of 15 differ from its first; A==B: $([ "$ref_a" = "$ref_b" ] && echo yes || echo no)"
# ---- (b') KV / prompt cache survives an evict + restore (LLAMA_SLEEP_EVICT_KV=1)
local p1 p2 c1 c2
code=$(complete "$PA" "$S/kv1.json" "$(greedy_body 32 true 'Explain in detail how a refrigerator moves heat from inside to outside')")
complete "$PB" "$S/kv-b.json" "$(greedy_body 8 false)" > /dev/null
[ "$(token_holder)" = "$B_PID" ] || log "warning: B does not hold the token before the KV re-check"
complete "$PA" "$S/kv2.json" "$(greedy_body 32 true 'Explain in detail how a refrigerator moves heat from inside to outside')" > /dev/null
p1=$(json_num prompt_n "$S/kv1.json"); p2=$(json_num prompt_n "$S/kv2.json")
c1=$(json_str content "$S/kv1.json"); c2=$(json_str content "$S/kv2.json")
check arbiter/b-kv-survives $(( ${p2:-99} < ${p1:-0} && ${#c1} > 0 ? 0 : 1 )) \
"prompt_n $p1 -> $p2 after evict/restore (cache reused), output $([ "$c1" = "$c2" ] && echo identical || echo DIFFERENT)"
[ "$c1" = "$c2" ] || fail arbiter/b-kv-output "cached-prompt output changed across evict/restore"
# ---- (c) image after release/restore
complete "$PB" "$S/c-b.json" "$(greedy_body 8 false)" > /dev/null # A goes cold
local cold_ok=1; [ "$(token_holder)" = "$B_PID" ] && cold_ok=0
local img; img=$(make_red_ppm)
printf '{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/x-portable-pixmap;base64,%s"}},{"type":"text","text":"What is the main color of this image? Answer with one word."}]}],"max_tokens":16,%s}' \
"$img" "$GREEDY" > "$S/c-body.json"
code=$(curl -sS -m 120 -o "$S/c-a.json" -w '%{http_code}' -H 'Content-Type: application/json' \
-d @"$S/c-body.json" "http://127.0.0.1:$PA/v1/chat/completions" 2>>"$S/curl.err")
local ans; ans=$(json_str content "$S/c-a.json")
check arbiter/c-image-after-restore $(( cold_ok == 0 && ${#code} > 0 && code == 200 ? 0 : 1 )) \
"A cold before: $([ $cold_ok -eq 0 ] && echo yes || echo NO), HTTP $code, answer: '$ans'"
echo "$ans" | grep -qi red || fail arbiter/c-image-answer "expected 'red', got '$ans'"
# text right after the image path (encoder out, weights back)
code=$(complete "$PA" "$S/c-a2.json" "$(greedy_body 48 false)")
[ "$code" = 200 ] && [ "$(json_str content "$S/c-a2.json")" = "$ref_a" ] \
|| fail arbiter/c-text-after-image "HTTP $code, output differs from reference after the image request"
# ---- (g) kill -9 a waiter that holds want.lock -> the holder stops yielding
stream_bg "$PA" "$S/g-a.sse" "$LONG_BODY"; local gcl=$CPID
if ! wait_tokens "$S/g-a.sse" 3 30000; then fail arbiter/g-kill9-waiter "A did not start streaming"; else
local s_before; s_before=$(sleeps "$S/A.log")
complete "$PB" "$S/g-b.json" "$(greedy_body 8 false)" 60 > /dev/null &
local gbc=$!; track "$gbc"
if ! wait_want "$B_PID" 10000; then
fail arbiter/g-kill9-waiter "B never registered on want.lock (setup)"
else
kill -KILL "$B_PID"; wait "$B_PID" 2>/dev/null
kill "$gcl" 2>/dev/null; wait "$gcl" 2>/dev/null # end A's generation (client abort)
sleep 2
local wl; wl=$(lock_pids "$ARENA/want.lock" READ | wc -l)
check arbiter/g-kill9-waiter $(( $(holder_num) == A_PID && $(sleeps "$S/A.log") == s_before && wl == 0 ? 0 : 1 )) \
"after kill -9 of waiting B: want.lock readers $wl, token holder $(token_holder) (A=$A_PID), A sleeps +$(( $(sleeps "$S/A.log") - s_before ))"
fi
wait "$gbc" 2>/dev/null
fi
log "arbiter: restarting B"
start_b
wait_health "$PB" "$B_PID" 180 || { fail arbiter/restart "B did not come back"; return; }
# ---- (e) SIGTERM while A waits for the token -> exits fast, never restores
stream_bg "$PB" "$S/e-b.sse" "$LONG_BODY"; local ecl=$CPID
wait_tokens "$S/e-b.sse" 3 30000 || { fail arbiter/e-term-waiting "B did not start streaming"; }
complete "$PA" "$S/e-a.json" "$(greedy_body 8 false)" 30 > /dev/null &
local eac=$!; track "$eac"
if wait_want "$A_PID" 10000; then
sleep 0.3
local t0 ms took_token=0
t0=$(now_ms); kill -TERM "$A_PID"
while pid_alive "$A_PID"; do
[ "$(token_holder)" = "$A_PID" ] && took_token=1
[ $(( $(now_ms) - t0 )) -gt 10000 ] && break
sleep 0.01
done
ms=$(( $(now_ms) - t0 )); wait "$A_PID"; RC=$?
check arbiter/e-term-waiting $(( RC == 0 && ms <= 1500 && took_token == 0 && $(holder_num) == B_PID ? 0 : 1 )) \
"exit $RC in ${ms} ms, A took the token: $([ $took_token = 1 ] && echo YES || echo no), holder now $(token_holder) (B=$B_PID), log: $(grep -c 'no longer waiting for the GPU token' "$S/A.log") abort line(s)"
else
fail arbiter/e-term-waiting "A never registered on want.lock (setup)"
stop_server "$A_PID"
fi
kill "$ecl" 2>/dev/null; wait "$ecl" "$eac" 2>/dev/null
log "arbiter: restarting A"
start_a
wait_health "$PA" "$A_PID" 180 || { fail arbiter/restart "A did not come back"; return; }
# ---- (d) SIGTERM during A's VRAM release -> exit 0 within ~2 s, B gets the token
# A is warm (fresh load, first release also captures the host shadow: the longest release)
local how sig_ms
if command -v gdb > /dev/null; then
how=gdb
gdb -p "$A_PID" -batch -nx -ex 'set pagination off' -ex 'handle all nostop noprint pass' \
-ex 'break llama_context_release_device' -ex 'continue' \
-ex "shell date +%s%3N > $S/d.sig" -ex 'delete' -ex 'signal SIGTERM' > "$S/d-gdb.log" 2>&1 &
local gpid=$!; track "$gpid"
local k=0; until grep -q '^Continuing' "$S/d-gdb.log" || [ $k -gt 300 ]; do sleep 0.1; k=$((k + 1)); done
sleep 0.5
else
how=log-poll
fi
local s0; s0=$(sleeps "$S/A.log")
complete "$PB" "$S/d-b.json" "$(greedy_body 8 false)" 60 > "$S/d-b.code" &
local dbc=$!; track "$dbc"
if [ $how = log-poll ]; then
local k=0; until [ "$(sleeps "$S/A.log")" -gt "$s0" ] || [ $k -gt 2000 ]; do sleep 0.005; k=$((k + 1)); done
now_ms > "$S/d.sig"; kill -TERM "$A_PID"
else
local k=0; until [ -s "$S/d.sig" ] || [ $k -gt 600 ]; do sleep 0.05; k=$((k + 1)); done
fi
if [ ! -s "$S/d.sig" ]; then
fail arbiter/d-term-in-release "release breakpoint never hit ($how), see $S/d-gdb.log"
kill "$gpid" 2>/dev/null; stop_server "$A_PID"
else
while pid_alive "$A_PID" && [ $(( $(now_ms) - $(cat "$S/d.sig") )) -lt 20000 ]; do sleep 0.01; done
sig_ms=$(( $(now_ms) - $(cat "$S/d.sig") ))
wait "$A_PID"; RC=$?
wait "$dbc"
local dcode; dcode=$(cat "$S/d-b.code")
sleep 0.2
check arbiter/d-term-in-release $(( RC == 0 && sig_ms <= 2500 && dcode == 200 && $(holder_num) == B_PID ? 0 : 1 )) \
"SIGTERM inside llama_context_release_device ($how): A exit $RC in ${sig_ms} ms, B HTTP $dcode, holder $(token_holder) (B=$B_PID)"
fi
log "arbiter: restarting A"
start_a
wait_health "$PA" "$A_PID" 180 || { fail arbiter/restart "A did not come back"; return; }
# ---- (f) SIGTERM mid-generation (streaming)
stream_bg "$PA" "$S/f-a.sse" "$LONG_BODY"; local fcl=$CPID
if ! wait_tokens "$S/f-a.sse" 10 30000; then fail arbiter/f-term-streaming "A did not start streaming"; else
local t0 free_ms=-1 cl_ms=-1 exit_ms=-1
t0=$(now_ms); kill -TERM "$A_PID"
while [ $(( $(now_ms) - t0 )) -lt 10000 ]; do
[ $free_ms -lt 0 ] && [ -z "$(token_holder)" ] && free_ms=$(( $(now_ms) - t0 ))
[ $cl_ms -lt 0 ] && ! pid_alive "$fcl" && cl_ms=$(( $(now_ms) - t0 ))
[ $exit_ms -lt 0 ] && ! pid_alive "$A_PID" && exit_ms=$(( $(now_ms) - t0 ))
[ $free_ms -ge 0 ] && [ $cl_ms -ge 0 ] && [ $exit_ms -ge 0 ] && break
sleep 0.01
done
wait "$A_PID"; RC=$?; wait "$fcl" 2>/dev/null
local errev; errev=$(grep -c '"error"' "$S/f-a.sse")
check arbiter/f-term-streaming $(( RC == 0 && errev > 0 && cl_ms >= 0 && free_ms >= 0 && free_ms <= 2500 && exit_ms >= 0 ? 0 : 1 )) \
"A exit $RC in ${exit_ms} ms, token free in ${free_ms} ms, client closed in ${cl_ms} ms with $errev error event(s) after $(grep -c '^data: ' "$S/f-a.sse") data lines"
fi
log "arbiter: restarting A"
start_a
wait_health "$PA" "$A_PID" 180 || { fail arbiter/restart "A did not come back"; return; }
# ---- (h) clean shutdown warm and cold
[ "$(token_holder)" = "$A_PID" ] || log "warning: A is not warm before the warm-shutdown case"
stop_server "$A_PID"
check arbiter/h-shutdown-warm $(( RC == 0 ? 0 : 1 )) "warm A: exit $RC in ${EXIT_MS} ms"
start_a
wait_health "$PA" "$A_PID" 180 || { fail arbiter/restart "A did not come back"; return; }
complete "$PB" "$S/h-b.json" "$(greedy_body 8 false)" > /dev/null # A goes cold
local acold=no; [ "$(token_holder)" = "$B_PID" ] && acold=yes
stop_server "$A_PID"
check arbiter/h-shutdown-cold $([ $RC -eq 0 ] && [ $acold = yes ] && echo 0 || echo 1) \
"cold A (B holds token: $acold): exit $RC in ${EXIT_MS} ms"
stop_server "$B_PID"
check arbiter/h-shutdown-warm-b $(( RC == 0 ? 0 : 1 )) "warm B: exit $RC in ${EXIT_MS} ms"
local bad_lines; bad_lines=$(scan_logs "$S/A.log" "$S/B.log")
check arbiter/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) \
"$([ -z "$bad_lines" ] && echo 'no asserts / Vulkan errors in A/B logs' || echo "$bad_lines" | head -3 | tr '\n' ' ')"
run_handoff_ab
}
# handoff_block BUILD TAG N : a fresh pair of BUILD on the scratch arena (A with mmproj, B text-only,
# the same config as case a), one warm-up request on A, then N+1 alternating requests starting with
# B. Request 0 is not a handoff (B holds the token after its load); the latency of requests 1..N is
# appended to $S/lat-TAG.txt. HB_BAD counts setup failures, non-200s and answers without the token.
handoff_block() {
local build=$1 tag=$2 n=$3 a b i port pid code t0 t1
HB_BAD=0
: > "$S/lat-$tag.txt"
start_server "L$tag-A" "$build" "$PA" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --mmproj "$MMPROJ"; a=$SPID
if ! wait_health "$PA" "$a" 180; then HB_BAD=1; stop_server "$a"; return 1; fi
complete "$PA" "$S/L$tag-warm.json" "$(greedy_body 8 false)" > /dev/null
start_server "L$tag-B" "$build" "$PB" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --no-mmproj; b=$SPID
if ! wait_health "$PB" "$b" 180; then HB_BAD=1; stop_server "$b"; stop_server "$a"; return 1; fi
for i in $(seq 0 "$n"); do
if [ $((i % 2)) -eq 0 ]; then port=$PB pid=$b; else port=$PA pid=$a; fi
t0=$(now_ms)
code=$(complete "$port" "$S/L$tag-$i.json" "$(greedy_body 48 false)" 60)
t1=$(now_ms)
if [ "$code" != 200 ] || [ "$(token_holder)" != "$pid" ]; then HB_BAD=$((HB_BAD + 1)); continue; fi
[ "$i" -gt 0 ] && echo $((t1 - t0)) >> "$S/lat-$tag.txt"
done
stop_server "$b"; stop_server "$a"
}
# ---- (a') ordinary handoff latency, candidate vs reference, in one session
# Two production deploys were lost to arbiter changes that passed the pathological cases but made
# every ordinary handoff slower. Blocks run ref, cand, cand, ref (a fixed order once gave the later
# slot about 9%); each build's median is over all its handoffs. FAIL if the candidate median exceeds
# the reference median by more than max(25%, 200 ms): small absolute numbers are noisy.
run_handoff_ab() {
local n=16 blk k=0 bad_r=0 bad_c=0 tag
log "arbiter: handoff latency A/B, blocks ref,cand,cand,ref x $n handoffs"
: > "$S/lat-ref.txt"; : > "$S/lat-cand.txt"
for blk in ref cand cand ref; do
k=$((k + 1)); tag=$k-$blk
local b; [ $blk = ref ] && b=$REF || b=$BUILD
handoff_block "$b" "$tag" "$n"
if [ $blk = ref ]; then bad_r=$((bad_r + HB_BAD)); else bad_c=$((bad_c + HB_BAD)); fi
cat "$S/lat-$tag.txt" >> "$S/lat-$blk.txt"
log "arbiter: block $tag median $(median < "$S/lat-$tag.txt") ms over $(wc -l < "$S/lat-$tag.txt") handoffs, $HB_BAD bad"
done
local mr mc nr nc maxr maxc limit
mr=$(median < "$S/lat-ref.txt"); mc=$(median < "$S/lat-cand.txt")
nr=$(wc -l < "$S/lat-ref.txt"); nc=$(wc -l < "$S/lat-cand.txt")
maxr=$(sort -n "$S/lat-ref.txt" | tail -1); maxc=$(sort -n "$S/lat-cand.txt" | tail -1)
if [ -z "$mr" ] || [ -z "$mc" ] || [ $bad_r -gt 0 ] || [ $bad_c -gt 0 ]; then
fail arbiter/a-handoff-latency "blocks incomplete: reference $nr handoffs ($bad_r bad), candidate $nc ($bad_c bad); see $S/L*-*.log"
return
fi
limit=$(awk -v r="$mr" 'BEGIN {d = r * 0.25; if (d < 200) d = 200; printf "%.0f", r + d}')
check arbiter/a-handoff-latency "$(awk -v c="$mc" -v l="$limit" 'BEGIN {print (c > l) ? 1 : 0}')" \
"median handoff latency candidate ${mc} ms (n $nc, max $maxc) vs reference ${mr} ms (n $nr, max $maxr); limit ${limit} ms = ref + max(25%, 200 ms)"
local bad_lines; bad_lines=$(scan_logs "$S"/L*-A.log "$S"/L*-B.log)
[ -z "$bad_lines" ] || fail arbiter/a-handoff-latency-logs "$(echo "$bad_lines" | head -3 | tr '\n' ' ')"
}
# ================================================================ 4. perf
vram_sampler() { # OUTFILE : max of mem_info_vram_used every 0.25 s, written continuously
local m=0 v
while :; do v=$(cat "$VRAM_USED"); [ "$v" -gt "$m" ] && { m=$v; echo "$m" > "$1"; }; sleep 0.25; done
}
# median of the numbers on stdin (one per line); empty input -> empty output
median() { sort -g | awk '{a[NR] = $1} END {if (NR) printf "%.2f", (NR % 2) ? a[(NR + 1) / 2] : (a[NR / 2] + a[NR / 2 + 1]) / 2}'; }
bench_samples() { # JSONL (llama-bench -o jsonl) -> "test<TAB>t/s", one row per repetition sample
sed -n 's/.*"n_prompt": *\([0-9]*\),.*"n_gen": *\([0-9]*\),.*"n_depth": *\([0-9]*\),.*"samples_ts": *\[\([^]]*\)\].*/\1 \2 \3 \4/p' "$1" |
awk '{t = ($1 > 0) ? "pp" $1 : "tg" $2; if ($3 > 0) t = t " @ d" $3
for (i = 4; i <= NF; i++) {v = $i; gsub(/,/, "", v); if (v != "") print t "\t" v}}'
}
run_perf() {
if supervisorctl -c /root/supervisord.conf status llama-swap 2>/dev/null | grep -q RUNNING; then
fail perf "llama-swap is RUNNING; run under /root/bench/moe/swapdown.sh"; return
fi
local others; others=$(ps -eo pid=,comm= | awk '$2 ~ /^llama-(server|bench|cli)$/' | wc -l)
[ "$others" -eq 0 ] || { fail perf "$others other llama processes are running"; return; }
[ -f "$PERF_MODEL" ] || { fail perf "model missing: $PERF_MODEL"; return; }
[ -x "$REF/bin/llama-bench" ] && [ -x "$BUILD/bin/llama-bench" ] || { fail perf "llama-bench missing"; return; }
local common=(-m "$PERF_MODEL" -t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none -o jsonl -oe md)
local arm k=0 base; base=$(vram_mb)
log "perf: idle VRAM $base MiB; arms ref,cand,cand,ref"
for arm in ref cand cand ref; do
k=$((k + 1))
local b; [ $arm = ref ] && b=$REF || b=$BUILD
local tag=$k-$arm
vram_sampler "$S/perf-$tag.vram" & local vs=$!; track "$vs"
# stdout: one JSON line per test with every repetition sample; stderr: the markdown table + logs
log "perf: arm $tag pp sweep"
LD_LIBRARY_PATH="$b/bin" "$b/bin/llama-bench" "${common[@]}" -p 2048 -n 0 -d 0,16384,32768 -r 2 \
> "$S/perf-$tag-pp.jsonl" 2> "$S/perf-$tag-pp.log"
log "perf: arm $tag tg"
LD_LIBRARY_PATH="$b/bin" "$b/bin/llama-bench" "${common[@]}" -p 0 -n 256 -d 4096 -r 3 \
> "$S/perf-$tag-tg.jsonl" 2> "$S/perf-$tag-tg.log"
kill "$vs" 2>/dev/null; wait "$vs" 2>/dev/null
{ bench_samples "$S/perf-$tag-pp.jsonl"; bench_samples "$S/perf-$tag-tg.jsonl"; } | sed "s/^/$arm\t/" >> "$S/perf.tsv"
echo "$arm $(cat "$S/perf-$tag.vram" 2>/dev/null || echo 0)" >> "$S/perf-vram.txt"
done
perf_report "$S/perf.tsv"
local total; total=$(cat "$VRAM_TOTAL")
local vr vc
vr=$(awk '$1=="ref" && $2>m {m=$2} END {print m+0}' "$S/perf-vram.txt")
vc=$(awk '$1=="cand" && $2>m {m=$2} END {print m+0}' "$S/perf-vram.txt")
check perf/peak-vram $(( vc * 100 <= total * 95 ? 0 : 1 )) \
"peak VRAM reference $((vr / 1048576)) MiB, candidate $((vc / 1048576)) MiB of $((total / 1048576)) MiB (limit 95%)"
}
# perf_report TSV : "arm<TAB>test<TAB>t/s" rows, one per llama-bench repetition sample (arm = ref|cand).
# Gates on the MEDIAN of all samples of a build (pooled over its arms): a single catastrophic
# sample (CPU contention from the other pods on the node) moves a mean, not a median.
perf_report() {
local tsv=$1
echo
echo "perf: median t/s over every repetition sample of both arms (order ref,cand,cand,ref)"
printf '%-18s %24s %24s %8s\n' test "reference med (min-max, n)" "candidate med (min-max, n)" delta
local tests t r c d rs cs
tests=$(awk -F'\t' '{print $2}' "$tsv" | awk '!s[$0]++')
local IFS_=$IFS; IFS=$'\n'
for t in $tests; do
r=$(awk -F'\t' -v t="$t" '$1=="ref" && $2==t {print $3}' "$tsv" | median)
c=$(awk -F'\t' -v t="$t" '$1=="cand" && $2==t {print $3}' "$tsv" | median)
rs=$(awk -F'\t' -v t="$t" '$1=="ref" && $2==t {n++; if (n == 1 || $3 < lo) lo = $3; if (n == 1 || $3 > hi) hi = $3} END {if (n) printf "%.1f-%.1f, %d", lo, hi, n}' "$tsv")
cs=$(awk -F'\t' -v t="$t" '$1=="cand" && $2==t {n++; if (n == 1 || $3 < lo) lo = $3; if (n == 1 || $3 > hi) hi = $3} END {if (n) printf "%.1f-%.1f, %d", lo, hi, n}' "$tsv")
d=$(awk -v r="$r" -v c="$c" 'BEGIN {if (r > 0) printf "%+.1f%%", (c/r - 1)*100}')
printf '%-18s %24s %24s %8s\n' "$t" "$r ($rs)" "$c ($cs)" "$d"
case "$t" in pp*)
if [ -z "$r" ] || [ -z "$c" ] || [ "$(awk -v r="$r" -v c="$c" 'BEGIN {print (c < 0.95*r)}')" = 1 ]; then
fail "perf/$t" "candidate median $c vs reference median $r t/s ($d, limit -5%)"
else
pass "perf/$t" "candidate median $c vs reference median $r t/s ($d)"
fi ;;
tg*) RESULTS+=("INFO perf/$t: candidate median $c vs reference median $r t/s ($d; decode is CPU-bound and noisy here, not gated)") ;;
esac
done
IFS=$IFS_
local n_tests; n_tests=$(echo "$tests" | grep -c '^pp')
[ "$n_tests" -ge 3 ] || fail perf/rows "expected 3 pp rows, got $n_tests (see $S/perf-*-pp.log)"
}
# ================================================================ 5. MoE cache release/restore smoke
run_moecache() {
[ -f "$MOECACHE_MODEL" ] || { fail moecache "model missing: $MOECACHE_MODEL"; return; }
local free_mb=$(( ( $(cat "$VRAM_TOTAL") - $(cat "$VRAM_USED") ) / 1048576 ))
log "moecache: $free_mb MiB VRAM free; server on :$PC with --moe-cache-mib $MOECACHE_MIB"
local arena=$S/arena-moec; mkdir -p "$arena"
# -lv 4: the cache size and hit/miss stats are only logged at that verbosity
# shellcheck disable=SC2086
start_server M "$BUILD" "$PC" LLAMA_SLEEP_VRAM_ONLY=1 "LLAMA_VRAM_ARENA=$arena" LLAMA_SLEEP_EVICT_KV=1 -- \
-m "$MOECACHE_MODEL" $MOECACHE_ARGS -fa 1 -lm none --parallel 1 --no-warmup --reasoning off -lv 4 \
--moe-cache-mib "$MOECACHE_MIB"
local mp=$SPID
wait_health "$PC" "$mp" 600 || { fail moecache "server did not start, see $S/M.log"; return; }
# chat endpoint: an instruct model fed a raw prompt can loop, which says nothing about the cache
local body="{\"messages\":[{\"role\":\"user\",\"content\":\"What are the three primary colors of light? Answer in one short sentence.\"}],\"max_tokens\":64,$GREEDY}"
local outs=() i code w
for i in 0 1 2; do
code=$(curl -sS -m 300 -o "$S/m$i.json" -w '%{http_code}' -H 'Content-Type: application/json' \
-d "$body" "http://127.0.0.1:$PC/v1/chat/completions" 2>>"$S/curl.err")
outs+=("$(json_str content "$S/m$i.json")")
[ "$code" = 200 ] || { fail moecache "request $i: HTTP $code"; stop_server "$mp"; return; }
[ $i -eq 2 ] && break
# force a release: a fake waiter holds want.lock shared until the server drops the token
( exec 9<"$arena/want.lock"; flock -s 9; exec sleep 120 ) & w=$!; track "$w"
local k=0; until [ -z "$(awk -v id="$(lock_id "$arena/token.lock")" '$2=="FLOCK" && $4=="WRITE" && $6==id' /proc/locks)" ] || [ $k -gt 600 ]; do sleep 0.05; k=$((k + 1)); done
log "moecache: cycle $((i + 1)) cold, VRAM $(vram_mb) MiB"
kill "$w"; wait "$w" 2>/dev/null
done
stop_server "$mp"
local size hits rel res
size=$(sed -n 's/.*MoE cache size = *\([0-9.]*\) MiB.*/\1/p' "$S/M.log" | head -1)
hits=$(sed -n 's/.*llama_moe_cache: .* hits = \([0-9]*\),.*/\1/p' "$S/M.log" | awk '{s += $1} END {print s+0}')
rel=$(grep -c 'released .* MiB of MoE cache' "$S/M.log"); res=$(grep -c 'restored .* MiB of MoE cache' "$S/M.log")
check moecache/active $(( hits > 0 ? 0 : 1 )) \
"cache ${size:-?} MiB, $hits hits (0 hits = cache too small to ever reuse an expert, so a stale SLRU would go unnoticed), released ${rel}x restored ${res}x"
local same=1; [ "${outs[0]}" = "${outs[1]}" ] && [ "${outs[1]}" = "${outs[2]}" ] && [ -n "${outs[0]}" ] && same=0
check moecache/identical $same "3 greedy answers across 2 cold/warm cycles $([ $same = 0 ] && echo identical || echo DIFFER): '$(echo "${outs[0]}" | cut -c1-80)'"
local coh=1; echo "${outs[2]}" | grep -qi red && echo "${outs[2]}" | grep -qi green && echo "${outs[2]}" | grep -qi blue && coh=0
check moecache/coherent $coh "answer after the 2nd restore names red, green and blue"
check moecache/shutdown $(( RC == 0 ? 0 : 1 )) "exit $RC"
local bad_lines; bad_lines=$(scan_logs "$S/M.log")
check moecache/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) "${bad_lines:-clean}"
}
# ================================================================ main
if [ -n "$PERF_REPORT" ]; then
[ -r "$PERF_REPORT" ] || { echo "cannot read $PERF_REPORT" >&2; exit 2; }
perf_report "$PERF_REPORT"; exit 0
fi
[ $DO_OPS -eq 1 ] && run_ops
[ $DO_CONCAT -eq 1 ] && run_concat
[ $DO_ARB -eq 1 ] && run_arbiter
[ $DO_PERF -eq 1 ] && run_perf
[ $DO_MOEC -eq 1 ] && run_moecache
exit 0