scripts: fork-validate.sh - A/B ordinary handoff latency against --ref

--arbiter only reported the candidate's handoff latency, while the fork's
process rule (two production deploys lost) is to A/B the ordinary handoff,
not only the pathological cases. After the arbiter cases, a-handoff-latency
starts fresh A/B server pairs of each build on the scratch arena and runs
blocks ref, cand, cand, ref of 16 alternating handoffs each. It FAILs if the
candidate median exceeds the reference median by more than max(25%, 200 ms)
and reports both medians, maxima and per-block medians.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
LumpiastyandClaude Opus 5.5 committed 2026-10-25 00:39:13 +02:00
1 parent 2512290f94
commit 80e662a2a1
1 file changed
+62
+62
View File
@@ -21,6 +21,10 @@
# and every shutdown path (mid-release, mid-wait, mid-generation, cold, warm, kill -9 of
# a waiter) exits cleanly and hands the GPU token on. These are the past production
# incidents (stranded requests, wedged token, exit 134 on cold teardown).
# Then a-handoff-latency: ordinary handoff latency of --build vs --ref in the same
# session (fresh server pairs, blocks ref,cand,cand,ref x 16 handoffs) -> FAIL if the
# candidate median exceeds the reference median by more than max(25%, 200 ms). Two
# production deploys were lost to arbiter changes that only showed up here.
# --perf llama-bench, candidate vs reference, alternating ref,cand,cand,ref, on the 35B MoE
# -> no prefill regression > 5% at depth 0/16k/32k (FAIL), decode tg256@d4096
# reported; peak VRAM per arm (FAIL if > 95% of the card). Compares MEDIANS of every
@@ -582,6 +586,64 @@ run_arbiter() {
local bad_lines; bad_lines=$(scan_logs "$S/A.log" "$S/B.log")
check arbiter/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) \
"$([ -z "$bad_lines" ] && echo 'no asserts / Vulkan errors in A/B logs' || echo "$bad_lines" | head -3 | tr '\n' ' ')"
run_handoff_ab
}
# handoff_block BUILD TAG N : a fresh pair of BUILD on the scratch arena (A with mmproj, B text-only,
# the same config as case a), one warm-up request on A, then N+1 alternating requests starting with
# B. Request 0 is not a handoff (B holds the token after its load); the latency of requests 1..N is
# appended to $S/lat-TAG.txt. HB_BAD counts setup failures, non-200s and answers without the token.
handoff_block() {
local build=$1 tag=$2 n=$3 a b i port pid code t0 t1
HB_BAD=0
: > "$S/lat-$tag.txt"
start_server "L$tag-A" "$build" "$PA" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --mmproj "$MMPROJ"; a=$SPID
if ! wait_health "$PA" "$a" 180; then HB_BAD=1; stop_server "$a"; return 1; fi
complete "$PA" "$S/L$tag-warm.json" "$(greedy_body 8 false)" > /dev/null
start_server "L$tag-B" "$build" "$PB" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --no-mmproj; b=$SPID
if ! wait_health "$PB" "$b" 180; then HB_BAD=1; stop_server "$b"; stop_server "$a"; return 1; fi
for i in $(seq 0 "$n"); do
if [ $((i % 2)) -eq 0 ]; then port=$PB pid=$b; else port=$PA pid=$a; fi
t0=$(now_ms)
code=$(complete "$port" "$S/L$tag-$i.json" "$(greedy_body 48 false)" 60)
t1=$(now_ms)
if [ "$code" != 200 ] || [ "$(token_holder)" != "$pid" ]; then HB_BAD=$((HB_BAD + 1)); continue; fi
[ "$i" -gt 0 ] && echo $((t1 - t0)) >> "$S/lat-$tag.txt"
done
stop_server "$b"; stop_server "$a"
}
# ---- (a') ordinary handoff latency, candidate vs reference, in one session
# Two production deploys were lost to arbiter changes that passed the pathological cases but made
# every ordinary handoff slower. Blocks run ref, cand, cand, ref (a fixed order once gave the later
# slot about 9%); each build's median is over all its handoffs. FAIL if the candidate median exceeds
# the reference median by more than max(25%, 200 ms): small absolute numbers are noisy.
run_handoff_ab() {
local n=16 blk k=0 bad_r=0 bad_c=0 tag
log "arbiter: handoff latency A/B, blocks ref,cand,cand,ref x $n handoffs"
: > "$S/lat-ref.txt"; : > "$S/lat-cand.txt"
for blk in ref cand cand ref; do
k=$((k + 1)); tag=$k-$blk
local b; [ $blk = ref ] && b=$REF || b=$BUILD
handoff_block "$b" "$tag" "$n"
if [ $blk = ref ]; then bad_r=$((bad_r + HB_BAD)); else bad_c=$((bad_c + HB_BAD)); fi
cat "$S/lat-$tag.txt" >> "$S/lat-$blk.txt"
log "arbiter: block $tag median $(median < "$S/lat-$tag.txt") ms over $(wc -l < "$S/lat-$tag.txt") handoffs, $HB_BAD bad"
done
local mr mc nr nc maxr maxc limit
mr=$(median < "$S/lat-ref.txt"); mc=$(median < "$S/lat-cand.txt")
nr=$(wc -l < "$S/lat-ref.txt"); nc=$(wc -l < "$S/lat-cand.txt")
maxr=$(sort -n "$S/lat-ref.txt" | tail -1); maxc=$(sort -n "$S/lat-cand.txt" | tail -1)
if [ -z "$mr" ] || [ -z "$mc" ] || [ $bad_r -gt 0 ] || [ $bad_c -gt 0 ]; then
fail arbiter/a-handoff-latency "blocks incomplete: reference $nr handoffs ($bad_r bad), candidate $nc ($bad_c bad); see $S/L*-*.log"
return
fi
limit=$(awk -v r="$mr" 'BEGIN {d = r * 0.25; if (d < 200) d = 200; printf "%.0f", r + d}')
check arbiter/a-handoff-latency "$(awk -v c="$mc" -v l="$limit" 'BEGIN {print (c > l) ? 1 : 0}')" \
"median handoff latency candidate ${mc} ms (n $nc, max $maxc) vs reference ${mr} ms (n $nr, max $maxr); limit ${limit} ms = ref + max(25%, 200 ms)"
local bad_lines; bad_lines=$(scan_logs "$S"/L*-A.log "$S"/L*-B.log)
[ -z "$bad_lines" ] || fail arbiter/a-handoff-latency-logs "$(echo "$bad_lines" | head -3 | tr '\n' ' ')"
}
# ================================================================ 4. perf