scripts: fork-validate.sh - A/B ordinary handoff latency against --ref
--arbiter only reported the candidate's handoff latency, while the fork's process rule (two production deploys lost) is to A/B the ordinary handoff, not only the pathological cases. After the arbiter cases, a-handoff-latency starts fresh A/B server pairs of each build on the scratch arena and runs blocks ref, cand, cand, ref of 16 alternating handoffs each. It FAILs if the candidate median exceeds the reference median by more than max(25%, 200 ms) and reports both medians, maxima and per-block medians. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2512290f94
commit
80e662a2a1
1 file changed
+62
@@ -21,6 +21,10 @@
|
||||
# and every shutdown path (mid-release, mid-wait, mid-generation, cold, warm, kill -9 of
|
||||
# a waiter) exits cleanly and hands the GPU token on. These are the past production
|
||||
# incidents (stranded requests, wedged token, exit 134 on cold teardown).
|
||||
# Then a-handoff-latency: ordinary handoff latency of --build vs --ref in the same
|
||||
# session (fresh server pairs, blocks ref,cand,cand,ref x 16 handoffs) -> FAIL if the
|
||||
# candidate median exceeds the reference median by more than max(25%, 200 ms). Two
|
||||
# production deploys were lost to arbiter changes that only showed up here.
|
||||
# --perf llama-bench, candidate vs reference, alternating ref,cand,cand,ref, on the 35B MoE
|
||||
# -> no prefill regression > 5% at depth 0/16k/32k (FAIL), decode tg256@d4096
|
||||
# reported; peak VRAM per arm (FAIL if > 95% of the card). Compares MEDIANS of every
|
||||
@@ -582,6 +586,64 @@ run_arbiter() {
|
||||
local bad_lines; bad_lines=$(scan_logs "$S/A.log" "$S/B.log")
|
||||
check arbiter/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) \
|
||||
"$([ -z "$bad_lines" ] && echo 'no asserts / Vulkan errors in A/B logs' || echo "$bad_lines" | head -3 | tr '\n' ' ')"
|
||||
|
||||
run_handoff_ab
|
||||
}
|
||||
|
||||
# handoff_block BUILD TAG N : a fresh pair of BUILD on the scratch arena (A with mmproj, B text-only,
|
||||
# the same config as case a), one warm-up request on A, then N+1 alternating requests starting with
|
||||
# B. Request 0 is not a handoff (B holds the token after its load); the latency of requests 1..N is
|
||||
# appended to $S/lat-TAG.txt. HB_BAD counts setup failures, non-200s and answers without the token.
|
||||
handoff_block() {
|
||||
local build=$1 tag=$2 n=$3 a b i port pid code t0 t1
|
||||
HB_BAD=0
|
||||
: > "$S/lat-$tag.txt"
|
||||
start_server "L$tag-A" "$build" "$PA" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --mmproj "$MMPROJ"; a=$SPID
|
||||
if ! wait_health "$PA" "$a" 180; then HB_BAD=1; stop_server "$a"; return 1; fi
|
||||
complete "$PA" "$S/L$tag-warm.json" "$(greedy_body 8 false)" > /dev/null
|
||||
start_server "L$tag-B" "$build" "$PB" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --no-mmproj; b=$SPID
|
||||
if ! wait_health "$PB" "$b" 180; then HB_BAD=1; stop_server "$b"; stop_server "$a"; return 1; fi
|
||||
for i in $(seq 0 "$n"); do
|
||||
if [ $((i % 2)) -eq 0 ]; then port=$PB pid=$b; else port=$PA pid=$a; fi
|
||||
t0=$(now_ms)
|
||||
code=$(complete "$port" "$S/L$tag-$i.json" "$(greedy_body 48 false)" 60)
|
||||
t1=$(now_ms)
|
||||
if [ "$code" != 200 ] || [ "$(token_holder)" != "$pid" ]; then HB_BAD=$((HB_BAD + 1)); continue; fi
|
||||
[ "$i" -gt 0 ] && echo $((t1 - t0)) >> "$S/lat-$tag.txt"
|
||||
done
|
||||
stop_server "$b"; stop_server "$a"
|
||||
}
|
||||
|
||||
# ---- (a') ordinary handoff latency, candidate vs reference, in one session
|
||||
# Two production deploys were lost to arbiter changes that passed the pathological cases but made
|
||||
# every ordinary handoff slower. Blocks run ref, cand, cand, ref (a fixed order once gave the later
|
||||
# slot about 9%); each build's median is over all its handoffs. FAIL if the candidate median exceeds
|
||||
# the reference median by more than max(25%, 200 ms): small absolute numbers are noisy.
|
||||
run_handoff_ab() {
|
||||
local n=16 blk k=0 bad_r=0 bad_c=0 tag
|
||||
log "arbiter: handoff latency A/B, blocks ref,cand,cand,ref x $n handoffs"
|
||||
: > "$S/lat-ref.txt"; : > "$S/lat-cand.txt"
|
||||
for blk in ref cand cand ref; do
|
||||
k=$((k + 1)); tag=$k-$blk
|
||||
local b; [ $blk = ref ] && b=$REF || b=$BUILD
|
||||
handoff_block "$b" "$tag" "$n"
|
||||
if [ $blk = ref ]; then bad_r=$((bad_r + HB_BAD)); else bad_c=$((bad_c + HB_BAD)); fi
|
||||
cat "$S/lat-$tag.txt" >> "$S/lat-$blk.txt"
|
||||
log "arbiter: block $tag median $(median < "$S/lat-$tag.txt") ms over $(wc -l < "$S/lat-$tag.txt") handoffs, $HB_BAD bad"
|
||||
done
|
||||
local mr mc nr nc maxr maxc limit
|
||||
mr=$(median < "$S/lat-ref.txt"); mc=$(median < "$S/lat-cand.txt")
|
||||
nr=$(wc -l < "$S/lat-ref.txt"); nc=$(wc -l < "$S/lat-cand.txt")
|
||||
maxr=$(sort -n "$S/lat-ref.txt" | tail -1); maxc=$(sort -n "$S/lat-cand.txt" | tail -1)
|
||||
if [ -z "$mr" ] || [ -z "$mc" ] || [ $bad_r -gt 0 ] || [ $bad_c -gt 0 ]; then
|
||||
fail arbiter/a-handoff-latency "blocks incomplete: reference $nr handoffs ($bad_r bad), candidate $nc ($bad_c bad); see $S/L*-*.log"
|
||||
return
|
||||
fi
|
||||
limit=$(awk -v r="$mr" 'BEGIN {d = r * 0.25; if (d < 200) d = 200; printf "%.0f", r + d}')
|
||||
check arbiter/a-handoff-latency "$(awk -v c="$mc" -v l="$limit" 'BEGIN {print (c > l) ? 1 : 0}')" \
|
||||
"median handoff latency candidate ${mc} ms (n $nc, max $maxc) vs reference ${mr} ms (n $nr, max $maxr); limit ${limit} ms = ref + max(25%, 200 ms)"
|
||||
local bad_lines; bad_lines=$(scan_logs "$S"/L*-A.log "$S"/L*-B.log)
|
||||
[ -z "$bad_lines" ] || fail arbiter/a-handoff-latency-logs "$(echo "$bad_lines" | head -3 | tr '\n' ' ')"
|
||||
}
|
||||
|
||||
# ================================================================ 4. perf
|
||||
|
||||
Reference in new issue
Block a user