scripts: fork-validate.sh - gate --perf on medians, not means
--perf averaged the two per-arm llama-bench means of each build, so one catastrophic sample (CPU contention from the ~60 other pods on the node) moved the result, against the fork's own rule of medians, never means. llama-bench now writes -o jsonl (every repetition sample) next to the markdown table, and the gates use the median of all samples of a build pooled over its two arms (4 per prefill cell, 6 for decode); the table shows min-max and n. --perf-report TSV re-aggregates a saved <scratch>/perf.tsv without a GPU. Checked offline on the pod: the 2026-09 A/B samples in results.txt give pp2048 reference 250.24 (old mean) vs 247.76 (new median), candidate 280.15 vs 276.93; a synthetic 2026-10-24-like set with one 180 t/s candidate sample read -9.3% (FAIL) the old way and -0.2% (PASS) with medians. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
62d442e7bf
commit
2512290f94
1 file changed
+62
-35
+62
-35
@@ -22,9 +22,12 @@
|
||||
# a waiter) exits cleanly and hands the GPU token on. These are the past production
|
||||
# incidents (stranded requests, wedged token, exit 134 on cold teardown).
|
||||
# --perf llama-bench, candidate vs reference, alternating ref,cand,cand,ref, on the 35B MoE
|
||||
# -> no prefill regression > 5% at depth 0/16k/32k (FAIL), decode reported; peak VRAM
|
||||
# per arm (FAIL if > 95% of the card). Needs llama-swap STOPPED (run it under
|
||||
# /root/bench/moe/swapdown.sh); refuses to run if any other llama process is up.
|
||||
# -> no prefill regression > 5% at depth 0/16k/32k (FAIL), decode tg256@d4096
|
||||
# reported; peak VRAM per arm (FAIL if > 95% of the card). Compares MEDIANS of every
|
||||
# repetition sample pooled over a build's two arms (4 per prefill cell, 6 for decode),
|
||||
# never means: CPU contention on the node produces single catastrophic samples.
|
||||
# Needs llama-swap STOPPED (run it under /root/bench/moe/swapdown.sh); refuses to
|
||||
# run if any other llama process is up. Samples are kept in <scratch>/perf.tsv.
|
||||
# --moecache MODEL
|
||||
# one arbiter server with --moe-cache-mib N on a MoE model, forced cold/warm twice
|
||||
# -> greedy output identical before/after (the expert-cache SLRU is re-initialised on
|
||||
@@ -39,6 +42,8 @@
|
||||
# --moecache-mib N cache size for --moecache (default 2048; 256 never hits on a 26B and proves nothing)
|
||||
# --port-base N scratch ports N..N+2 (default 8091)
|
||||
# --keep keep the scratch dir even when everything passed (it is kept on any FAIL)
|
||||
# --perf-report TSV only re-aggregate a saved <scratch>/perf.tsv of an earlier --perf run and
|
||||
# apply the same gates (no GPU, no --build/--ref needed); runs nothing else
|
||||
#
|
||||
# Requirements: bash, coreutils, awk, sed, grep, curl, flock(1); gdb optional (makes the
|
||||
# SIGTERM-during-release case deterministic). No python/jq.
|
||||
@@ -58,6 +63,7 @@ BUILD= ; REF= ; CONCAT_BUILD=
|
||||
PORT_BASE=8091
|
||||
KEEP=0
|
||||
DO_OPS=0 DO_CONCAT=0 DO_ARB=0 DO_PERF=0 DO_MOEC=0
|
||||
PERF_REPORT=
|
||||
|
||||
usage() { awk 'NR > 1 && /^#/ {sub(/^# ?/, ""); print; next} NR > 1 {exit}' "$0"; exit 2; }
|
||||
|
||||
@@ -78,11 +84,12 @@ while [ $# -gt 0 ]; do
|
||||
--perf-model) PERF_MODEL=$2; shift 2 ;;
|
||||
--port-base) PORT_BASE=$2; shift 2 ;;
|
||||
--keep) KEEP=1; shift ;;
|
||||
--perf-report) PERF_REPORT=$(readlink -f "$2"); shift 2 ;;
|
||||
-h|--help) usage ;;
|
||||
*) echo "unknown argument: $1" >&2; usage ;;
|
||||
esac
|
||||
done
|
||||
[ -n "$BUILD" ] && [ -n "$REF" ] || usage
|
||||
[ -n "$PERF_REPORT" ] || { [ -n "$BUILD" ] && [ -n "$REF" ]; } || usage
|
||||
if [ $((DO_OPS + DO_CONCAT + DO_ARB + DO_PERF + DO_MOEC)) -eq 0 ]; then
|
||||
DO_OPS=1 DO_CONCAT=1 DO_ARB=1
|
||||
fi
|
||||
@@ -161,7 +168,7 @@ trap 'echo "interrupted"; NFAIL=$((NFAIL + 1)); exit 130' INT TERM HUP
|
||||
for v in $(env | sed -n 's/^\(GGML_[A-Za-z0-9_]*\|LLAMA_\(SLEEP\|VRAM\|MMPROJ\|ARG\|CACHE\|LOG\)[A-Za-z0-9_]*\)=.*/\1/p'); do
|
||||
log "unsetting inherited $v"; unset "$v"
|
||||
done
|
||||
for d in "$BUILD" "$REF"; do
|
||||
[ -n "$PERF_REPORT" ] || for d in "$BUILD" "$REF"; do
|
||||
[ -x "$d/bin/llama-server" ] || { echo "no llama-server in $d/bin" >&2; exit 2; }
|
||||
done
|
||||
case "$ARENA" in /dev/shm/llama-vram*) echo "refusing the production arena" >&2; exit 2 ;; esac
|
||||
@@ -172,7 +179,7 @@ port_busy() { # LISTEN socket on the port (v4 or v6)
|
||||
local hex; hex=$(printf '%04X' "$1")
|
||||
awk -v h=":$hex" '$4=="0A" && substr($2, length($2)-4)==h {f=1} END {exit !f}' /proc/net/tcp /proc/net/tcp6 2>/dev/null
|
||||
}
|
||||
for p in $PA $PB $PC; do
|
||||
[ -n "$PERF_REPORT" ] || for p in $PA $PB $PC; do
|
||||
port_busy "$p" && { echo "scratch port $p is in use" >&2; exit 2; }
|
||||
done
|
||||
log "scratch dir $S, ports $PA/$PB/$PC, VRAM used $(vram_mb) MiB"
|
||||
@@ -582,8 +589,12 @@ vram_sampler() { # OUTFILE : max of mem_info_vram_used every 0.25 s, written con
|
||||
local m=0 v
|
||||
while :; do v=$(cat "$VRAM_USED"); [ "$v" -gt "$m" ] && { m=$v; echo "$m" > "$1"; }; sleep 0.25; done
|
||||
}
|
||||
bench_rows() { # LOG -> "test<TAB>t/s" rows from llama-bench markdown
|
||||
awk -F'|' '/\| *(pp|tg)[0-9]+/ {t=$(NF-2); v=$(NF-1); gsub(/^ +| +$/, "", t); split(v, a, " "); print t "\t" a[1]}' "$1"
|
||||
# median of the numbers on stdin (one per line); empty input -> empty output
|
||||
median() { sort -g | awk '{a[NR] = $1} END {if (NR) printf "%.2f", (NR % 2) ? a[(NR + 1) / 2] : (a[NR / 2] + a[NR / 2 + 1]) / 2}'; }
|
||||
bench_samples() { # JSONL (llama-bench -o jsonl) -> "test<TAB>t/s", one row per repetition sample
|
||||
sed -n 's/.*"n_prompt": *\([0-9]*\),.*"n_gen": *\([0-9]*\),.*"n_depth": *\([0-9]*\),.*"samples_ts": *\[\([^]]*\)\].*/\1 \2 \3 \4/p' "$1" |
|
||||
awk '{t = ($1 > 0) ? "pp" $1 : "tg" $2; if ($3 > 0) t = t " @ d" $3
|
||||
for (i = 4; i <= NF; i++) {v = $i; gsub(/,/, "", v); if (v != "") print t "\t" v}}'
|
||||
}
|
||||
run_perf() {
|
||||
if supervisorctl -c /root/supervisord.conf status llama-swap 2>/dev/null | grep -q RUNNING; then
|
||||
@@ -593,7 +604,7 @@ run_perf() {
|
||||
[ "$others" -eq 0 ] || { fail perf "$others other llama processes are running"; return; }
|
||||
[ -f "$PERF_MODEL" ] || { fail perf "model missing: $PERF_MODEL"; return; }
|
||||
[ -x "$REF/bin/llama-bench" ] && [ -x "$BUILD/bin/llama-bench" ] || { fail perf "llama-bench missing"; return; }
|
||||
local common=(-m "$PERF_MODEL" -t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none)
|
||||
local common=(-m "$PERF_MODEL" -t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none -o jsonl -oe md)
|
||||
local arm k=0 base; base=$(vram_mb)
|
||||
log "perf: idle VRAM $base MiB; arms ref,cand,cand,ref"
|
||||
for arm in ref cand cand ref; do
|
||||
@@ -601,37 +612,18 @@ run_perf() {
|
||||
local b; [ $arm = ref ] && b=$REF || b=$BUILD
|
||||
local tag=$k-$arm
|
||||
vram_sampler "$S/perf-$tag.vram" & local vs=$!; track "$vs"
|
||||
# stdout: one JSON line per test with every repetition sample; stderr: the markdown table + logs
|
||||
log "perf: arm $tag pp sweep"
|
||||
LD_LIBRARY_PATH="$b/bin" "$b/bin/llama-bench" "${common[@]}" -p 2048 -n 0 -d 0,16384,32768 -r 2 > "$S/perf-$tag-pp.log" 2>&1
|
||||
LD_LIBRARY_PATH="$b/bin" "$b/bin/llama-bench" "${common[@]}" -p 2048 -n 0 -d 0,16384,32768 -r 2 \
|
||||
> "$S/perf-$tag-pp.jsonl" 2> "$S/perf-$tag-pp.log"
|
||||
log "perf: arm $tag tg"
|
||||
LD_LIBRARY_PATH="$b/bin" "$b/bin/llama-bench" "${common[@]}" -p 0 -n 256 -d 4096 -r 3 > "$S/perf-$tag-tg.log" 2>&1
|
||||
LD_LIBRARY_PATH="$b/bin" "$b/bin/llama-bench" "${common[@]}" -p 0 -n 256 -d 4096 -r 3 \
|
||||
> "$S/perf-$tag-tg.jsonl" 2> "$S/perf-$tag-tg.log"
|
||||
kill "$vs" 2>/dev/null; wait "$vs" 2>/dev/null
|
||||
{ bench_rows "$S/perf-$tag-pp.log"; bench_rows "$S/perf-$tag-tg.log"; } | sed "s/^/$arm\t/" >> "$S/perf.tsv"
|
||||
{ bench_samples "$S/perf-$tag-pp.jsonl"; bench_samples "$S/perf-$tag-tg.jsonl"; } | sed "s/^/$arm\t/" >> "$S/perf.tsv"
|
||||
echo "$arm $(cat "$S/perf-$tag.vram" 2>/dev/null || echo 0)" >> "$S/perf-vram.txt"
|
||||
done
|
||||
echo
|
||||
echo "perf: mean t/s over 2 arms each (order ref,cand,cand,ref)"
|
||||
printf '%-18s %10s %10s %8s\n' test reference candidate delta
|
||||
local tests t r c d bad=0
|
||||
tests=$(awk -F'\t' '{print $2}' "$S/perf.tsv" | awk '!s[$0]++')
|
||||
local IFS_=$IFS; IFS=$'\n'
|
||||
for t in $tests; do
|
||||
r=$(awk -F'\t' -v t="$t" '$1=="ref" && $2==t {s+=$3; n++} END {if (n) printf "%.2f", s/n}' "$S/perf.tsv")
|
||||
c=$(awk -F'\t' -v t="$t" '$1=="cand" && $2==t {s+=$3; n++} END {if (n) printf "%.2f", s/n}' "$S/perf.tsv")
|
||||
d=$(awk -v r="$r" -v c="$c" 'BEGIN {if (r > 0) printf "%+.1f%%", (c/r - 1)*100}')
|
||||
printf '%-18s %10s %10s %8s\n' "$t" "$r" "$c" "$d"
|
||||
case "$t" in pp*)
|
||||
if [ -z "$r" ] || [ -z "$c" ] || [ "$(awk -v r="$r" -v c="$c" 'BEGIN {print (c < 0.95*r)}')" = 1 ]; then
|
||||
bad=$((bad + 1)); fail "perf/$t" "candidate $c vs reference $r t/s ($d, limit -5%)"
|
||||
else
|
||||
pass "perf/$t" "candidate $c vs reference $r t/s ($d)"
|
||||
fi ;;
|
||||
tg*) RESULTS+=("INFO perf/$t: candidate $c vs reference $r t/s ($d; decode is CPU-bound and noisy here, not gated)") ;;
|
||||
esac
|
||||
done
|
||||
IFS=$IFS_
|
||||
local n_tests; n_tests=$(echo "$tests" | grep -c '^pp')
|
||||
[ "$n_tests" -ge 3 ] || fail perf/rows "expected 3 pp rows, got $n_tests (see $S/perf-*-pp.log)"
|
||||
perf_report "$S/perf.tsv"
|
||||
local total; total=$(cat "$VRAM_TOTAL")
|
||||
local vr vc
|
||||
vr=$(awk '$1=="ref" && $2>m {m=$2} END {print m+0}' "$S/perf-vram.txt")
|
||||
@@ -639,6 +631,37 @@ run_perf() {
|
||||
check perf/peak-vram $(( vc * 100 <= total * 95 ? 0 : 1 )) \
|
||||
"peak VRAM reference $((vr / 1048576)) MiB, candidate $((vc / 1048576)) MiB of $((total / 1048576)) MiB (limit 95%)"
|
||||
}
|
||||
# perf_report TSV : "arm<TAB>test<TAB>t/s" rows, one per llama-bench repetition sample (arm = ref|cand).
|
||||
# Gates on the MEDIAN of all samples of a build (pooled over its arms): a single catastrophic
|
||||
# sample (CPU contention from the other pods on the node) moves a mean, not a median.
|
||||
perf_report() {
|
||||
local tsv=$1
|
||||
echo
|
||||
echo "perf: median t/s over every repetition sample of both arms (order ref,cand,cand,ref)"
|
||||
printf '%-18s %24s %24s %8s\n' test "reference med (min-max, n)" "candidate med (min-max, n)" delta
|
||||
local tests t r c d rs cs
|
||||
tests=$(awk -F'\t' '{print $2}' "$tsv" | awk '!s[$0]++')
|
||||
local IFS_=$IFS; IFS=$'\n'
|
||||
for t in $tests; do
|
||||
r=$(awk -F'\t' -v t="$t" '$1=="ref" && $2==t {print $3}' "$tsv" | median)
|
||||
c=$(awk -F'\t' -v t="$t" '$1=="cand" && $2==t {print $3}' "$tsv" | median)
|
||||
rs=$(awk -F'\t' -v t="$t" '$1=="ref" && $2==t {n++; if (n == 1 || $3 < lo) lo = $3; if (n == 1 || $3 > hi) hi = $3} END {if (n) printf "%.1f-%.1f, %d", lo, hi, n}' "$tsv")
|
||||
cs=$(awk -F'\t' -v t="$t" '$1=="cand" && $2==t {n++; if (n == 1 || $3 < lo) lo = $3; if (n == 1 || $3 > hi) hi = $3} END {if (n) printf "%.1f-%.1f, %d", lo, hi, n}' "$tsv")
|
||||
d=$(awk -v r="$r" -v c="$c" 'BEGIN {if (r > 0) printf "%+.1f%%", (c/r - 1)*100}')
|
||||
printf '%-18s %24s %24s %8s\n' "$t" "$r ($rs)" "$c ($cs)" "$d"
|
||||
case "$t" in pp*)
|
||||
if [ -z "$r" ] || [ -z "$c" ] || [ "$(awk -v r="$r" -v c="$c" 'BEGIN {print (c < 0.95*r)}')" = 1 ]; then
|
||||
fail "perf/$t" "candidate median $c vs reference median $r t/s ($d, limit -5%)"
|
||||
else
|
||||
pass "perf/$t" "candidate median $c vs reference median $r t/s ($d)"
|
||||
fi ;;
|
||||
tg*) RESULTS+=("INFO perf/$t: candidate median $c vs reference median $r t/s ($d; decode is CPU-bound and noisy here, not gated)") ;;
|
||||
esac
|
||||
done
|
||||
IFS=$IFS_
|
||||
local n_tests; n_tests=$(echo "$tests" | grep -c '^pp')
|
||||
[ "$n_tests" -ge 3 ] || fail perf/rows "expected 3 pp rows, got $n_tests (see $S/perf-*-pp.log)"
|
||||
}
|
||||
|
||||
# ================================================================ 5. MoE cache release/restore smoke
|
||||
run_moecache() {
|
||||
@@ -685,6 +708,10 @@ run_moecache() {
|
||||
}
|
||||
|
||||
# ================================================================ main
|
||||
if [ -n "$PERF_REPORT" ]; then
|
||||
[ -r "$PERF_REPORT" ] || { echo "cannot read $PERF_REPORT" >&2; exit 2; }
|
||||
perf_report "$PERF_REPORT"; exit 0
|
||||
fi
|
||||
[ $DO_OPS -eq 1 ] && run_ops
|
||||
[ $DO_CONCAT -eq 1 ] && run_concat
|
||||
[ $DO_ARB -eq 1 ] && run_arbiter
|
||||
|
||||
Reference in new issue
Block a user